103 changed files with 14900 additions and 42 deletions
+6
View File
@@ -18,3 +18,9 @@ __pycache__
node_modules
evidence
.DS_Store
research
student-tests
scripts
docs
requirements-research.txt
*.md
+32 -21
View File
@@ -1,30 +1,41 @@
# Copy to .env for your LOCAL synthetic test environment; never commit .env.
# These are logical interface suggestions; map actual framework names in REPORT.md.
# Секреты (пароль БД, DATABASE_URL) создаёт scripts/init-secrets.sh в каталоге secrets/ — не в .env.
APP_HOST=0.0.0.0
APP_PORT=8080
APP_ENV=lab
# Срок жизни сессии; для проверки expiry запускайте с SESSION_TTL_SECONDS=2 (см. student-tests/test_expiry.sh)
SESSION_TTL_SECONDS=3600
# Choose credentials locally. File paths contain no secret values.
DATABASE_URL_FILE=/run/secrets/database_url
# Optional on 3, required use on 4 in lab2; Redis OR Valkey:
# TTL записи в кэше Valkey (не больше остатка жизни сессии)
SESSION_CACHE_TTL_SECONDS=60
# Источник сессий: PostgreSQL (истина) + Valkey (кэш). Адрес кэша задан в compose.yaml (VALKEY_URL).
SESSION_STORE=postgres
CACHE_URL_FILE=/run/secrets/cache_url
DB_POOL_MAX=10
# Файлы секретов внутри контейнера (создаются scripts/init-secrets.sh)
DATABASE_URL_FILE=/run/secrets/database_url
# --- lab3 ---
# Число API-реплик (compose deploy.replicas); также можно: docker compose up -d --scale api=4
API_REPLICAS=2
IDEMPOTENCY_TTL_SECONDS=86400
S3_ENDPOINT=http://s3:9000
S3_REGION=us-east-1
API_CPUS=1.0
API_MEM=256m
POSTGRES_CPUS=1.0
LB_CPUS=1.0
# Пауза между SIGTERM и закрытием listener: LB успевает исключить реплику (rolling update без ошибок)
DRAIN_SECONDS=4
# --- lab4 ---
S3_BUCKET=messenger-attachments
S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id
S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key
MAX_UPLOAD_BYTES=10485760
MAX_IMAGE_PIXELS=40000000
BROKER_URL_FILE=/run/secrets/broker_url
S3_CPUS=1.0
UPLOAD_CONCURRENCY=4
# --- lab5 ---
RABBITMQ_CPUS=1.0
WORKER_CPUS=1.0
WORKER_MEM=768m
WORKER_REPLICAS=1
WORKER_CONCURRENCY=2
JOB_MAX_ATTEMPTS=5
JOB_TIMEOUT_SECONDS=60
STAGING_RETENTION_SECONDS=86400
OTEL_SERVICE_NAME=messenger-api
# Worker must set a distinct service.name.
OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4318
OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf
OTEL_RESOURCE_ATTRIBUTES=deployment.environment.name=lab
MAX_ATTEMPTS=5
LEASE_SECONDS=60
MAX_QUEUED_JOBS=200
# --- lab6: наблюдаемость ---
OTEL_ENABLED=1
# доля трасс при head-sampling в SDK (1.0 = все); профиль tail-sampling: COLLECTOR_CONFIG=/etc/otelcol/collector-tail.yaml
OTEL_TRACES_SAMPLER_ARG=1.0
COLLECTOR_CONFIG=/etc/otelcol/collector.yaml
+2
View File
@@ -0,0 +1,2 @@
# Keep shell hooks runnable after a Windows clone with core.autocrlf=true.
*.sh text eol=lf
+25
View File
@@ -0,0 +1,25 @@
name: Application smoke
on: [push, pull_request]
permissions:
contents: read
jobs:
application:
runs-on: ubuntu-latest
timeout-minutes: 15
steps:
- uses: actions/checkout@v4
- uses: actions/setup-python@v5
with:
python-version: '3.12'
- name: Prepare isolated lab credentials
run: |
sh scripts/init-secrets.sh
cp .env.example .env
docker compose --env-file .env config --quiet
- name: Build and wait for application
run: docker compose --env-file .env up -d --build --wait --wait-timeout 120
- name: Check contract and public scenario
run: make check test PYTHON=python BASE_URL=http://127.0.0.1:8080
- name: Stop disposable CI stack
if: always()
run: docker compose --env-file .env down
+32
View File
@@ -0,0 +1,32 @@
# syntax=docker/dockerfile:1
# Базовый образ закреплён тегом и digest (python 3.12.12, Debian bookworm).
ARG PYTHON_IMAGE=python:3.12.12-slim-bookworm@sha256:593bd06efe90efa80dc4eee3948be7c0fde4134606dd40d8dd8dbcade98e669c
# --- сборка зависимостей в отдельном слое (в runtime не попадают pip и кэш) ---
FROM ${PYTHON_IMAGE} AS builder
ENV PIP_DISABLE_PIP_VERSION_CHECK=1 PIP_NO_CACHE_DIR=1
WORKDIR /build
COPY requirements.txt .
RUN python -m venv /opt/venv && /opt/venv/bin/pip install -r requirements.txt
# --- runtime ---
FROM ${PYTHON_IMAGE}
ENV PATH=/opt/venv/bin:$PATH \
PYTHONUNBUFFERED=1 PYTHONDONTWRITEBYTECODE=1 PYTHONPATH=/srv/src \
APP_HOST=0.0.0.0 APP_PORT=8080 APP_ENV=lab LIMIT_CONCURRENCY=1000 MIGRATIONS_DIR=/srv/migrations
RUN groupadd --system --gid 10001 app && useradd --system --uid 10001 --gid app --no-create-home app
COPY --from=builder /opt/venv /opt/venv
WORKDIR /srv
COPY src/ /srv/src/
COPY migrations/ /srv/migrations/
# ревизия образа (меняется при rolling update; видна в журнале запуска)
ARG REV=dev
LABEL org.opencontainers.image.revision=$REV
ENV APP_REVISION=$REV
USER 10001:10001
EXPOSE 8080
HEALTHCHECK --interval=10s --timeout=3s --start-period=5s --retries=3 \
CMD ["python", "-c", "import urllib.request,os,sys; sys.exit(0 if urllib.request.urlopen('http://127.0.0.1:%s/health/ready' % os.environ.get('APP_PORT','8080'), timeout=2).status == 200 else 1)"]
# exec-форма: сервер = PID 1. Состояние хранится в PostgreSQL/Valkey, процесс stateless;
# масштабирование — репликами контейнера. SIGTERM: drain (readiness 503) -> graceful shutdown (см. app/serve.py).
CMD ["python", "-m", "app.serve"]
+17
View File
@@ -28,3 +28,20 @@ test-persistence:
test-failover:
@test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable stop-one hook"; exit 2)
$(PYTHON) tests/resilience.py --mode failover --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS)
# --- собственные цели студента ---
.PHONY: secrets up down student-test
secrets:
sh scripts/init-secrets.sh
up: secrets
docker compose --env-file .env up -d --build
down:
docker compose --env-file .env down
student-test:
$(PYTHON) -m unittest discover -s student-tests -p "test_api.py" -v
$(PYTHON) -m unittest discover -s student-tests -p "test_attachments.py" -v
$(PYTHON) -m unittest discover -s student-tests -p "test_pipeline.py" -v
$(PYTHON) -m unittest discover -s student-tests -p "test_observability.py" -v
+185 -21
View File
@@ -1,37 +1,201 @@
# Отчёт — лабораторная 6
- Автор, группа: TODO
- Целевая оценка; выбранный трек 5, если нужен: TODO
- Commit/PR: TODO после публикации
- Использованные источники, библиотеки и помощь ИИ: TODO
- Автор, группа: Кушнаренко Николай Вячеславович, БПИ2302
- Целевая оценка; выбранный трек 5, если нужен: **5**, трек 1 «Цена наблюдаемости»
- Commit/PR: ветка `solution/bpi2302-kushnarenko` в [своём форке](https://gitea.privcloud.tech/kolya/mtusi2026lab6/src/branch/solution/bpi2302-kushnarenko); текущие проверки от 12.11.2026
- Использованные источники, библиотеки и помощь ИИ: Python 3.12, OpenTelemetry Python SDK 1.45.0 (traces/metrics/logs, OTLP/HTTP), uvicorn, asyncpg, aiobotocore, aio-pika, Pillow; OpenTelemetry Collector contrib 0.161.0, Prometheus 3.15.0, Loki 3.7.8, Tempo 2.9.0, Grafana 13.2.3; RabbitMQ 4.1.8, SeaweedFS 4.48, PostgreSQL 17, Valkey 8, HAProxy 3.0. Код lab5 перенесён, бизнес-API не менялся. ИИ-ассистенты Claude/Codex использовались для помощи с кодом, тестами и оформлением; результаты ниже получены реальными запусками
## Запуск из чистого clone
TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных.
Prerequisites: Docker Compose v2+ (выделить Docker не менее 7 ГиБ RAM; сумма лимитов контейнеров ≈6.1 ГиБ), bash (Git Bash на Windows), Python ≥ 3.9 для публичных тестов, Python ≥ 3.11 для исследований; на Windows `PYTHONUTF8=1`.
Версии всех образов закреплены тегом и digest в `compose.yaml`; зависимости — `requirements.txt` (включая `opentelemetry-*==1.45.0`).
```sh
sh scripts/init-secrets.sh # secrets/*: БД, S3, RabbitMQ, пароль Grafana (secrets/grafana_admin_password)
cp .env.example .env
docker compose --env-file .env config --quiet
docker compose --env-file .env up -d --build --wait --wait-timeout 180
curl http://127.0.0.1:8080/health/ready # API
# Grafana: http://127.0.0.1:3000 пользователь admin, пароль — secrets/grafana_admin_password
python3 -m pip install -r research/requirements.txt -r requirements-dev.txt
python3 scripts/check_contract.py # make check
python3 scripts/validate_openapi.py # make validate
python3 tests/smoke.py --base-url http://127.0.0.1:8080 # make test (телеметрия включена)
python3 -m unittest discover -s student-tests -p test_observability.py -v # метрики, логи, трассы, связь API -> worker, отсутствие секретов
python3 student-tests/test_observability_retry.py # все попытки задания в одной трассе
mkdir -p evidence/my-run
python3 research/incidents.py --out evidence/my-run
python3 research/obs_cost.py --out evidence/my-run --profiles off on100 head10 tail --repeat 2
python3 research/plot_obs_cost.py --out evidence/my-run
```
Windows: `scripts/restart.cmd`, `scripts/stop-one.cmd`. Остановка с сохранением данных — `docker compose --env-file .env stop`; **намеренное удаление учебных данных** — `docker compose --env-file .env down -v` (volumes `pgdata`, `valkeydata`, `seaweeddata`, `rabbitmqdata`, `promdata`, `lokidata`, `tempodata`, `grafanadata`).
Профиль без телеметрии: `OTEL_ENABLED=0`; head-sampling: `OTEL_TRACES_SAMPLER_ARG=0.1`; tail-sampling в Collector: `COLLECTOR_CONFIG=/etc/otelcol/collector-tail.yaml`.
## Архитектура и решения
TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab5, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации.
```
API ×2 ─┐ ┌─▶ Prometheus (метрики, retention 7 д, volume promdata)
├─OTLP/HTTP─▶ OTel Collector ─┼─▶ Tempo (трассы, retention 7 д, volume tempodata)
worker ─┘ (memory_limiter, batch, └─▶ Loki (логи, retention 7 д, volume lokidata)
scrub, tail_sampling*) ▲
traceparent: API ─▶ outbox ─▶ RabbitMQ ─▶ worker Grafana (127.0.0.1:3000) читает все три источника
```
## Доказательства критериев
| Роль | Сервис | Версия | Порты | Volume | Зачем |
| --- | --- | --- | --- | --- | --- |
| Collector | `otel-collector` | contrib 0.161.0 | OTLP 4317/4318, метрики 8888/8889 — **только сеть backend** | — | единая точка приёма, ограничение RAM, удаление чувствительных атрибутов, маршрутизация сигналов; это не хранилище |
| Метрики | `prometheus` | 3.15.0 | не публикуется | `promdata` | временные ряды, SLI/recording rules, alert rules, firing/resolved |
| Логи | `loki` | 3.7.8 | не публикуется | `lokidata` | структурные логи API и worker (OTLP), поиск по `trace_id` |
| Трассы | `tempo` | 2.9.0 | не публикуется | `tempodata` | распределённые трассы, TraceQL |
| UI | `grafana` | 13.2.3 | `127.0.0.1:3000` | `grafanadata` | datasources/dashboard из Git (provisioning), анонимный доступ выключен, пароль — secret |
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
| --- | --- | --- | --- |
| Публичный smoke | make test | TODO | TODO |
| Остальные пункты уровня 3 | TODO | TODO | TODO |
| Все пункты уровня 4, если заявлен | TODO | TODO | TODO |
| Выбранный трек 5, если заявлен | TODO | TODO | TODO |
**Что хранит что.** Grafana ничего не хранит из сигналов: это интерфейс запросов (PromQL, LogQL, TraceQL) и provisioning. Collector — маршрутизатор/обработчик (не БД). Метрики — Prometheus, логи — Loki, трассы — Tempo. Конфигурации (`infra/otel`, `infra/prometheus`, `infra/loki`, `infra/tempo`, `infra/grafana`) и dashboard JSON лежат в Git; провижининг datasources и dashboard автоматический.
TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены.
### Сигналы и соответствие `contracts/observability.md`
## Эксперименты и отказы
| Сигнал контракта | Реализация (SDK) → имя в Prometheus / место |
| --- | --- |
| HTTP requests; HTTP duration (s) | `http.server.request.duration` (histogram, секунды, buckets 5 мс…10 с) → `http_server_request_duration_seconds_{bucket,sum,count}{http_request_method, http_route, http_response_status_code}`; число запросов = `_count` (отдельный счётчик не дублируется) |
| Jobs processed/failed; duration | `messenger.jobs.processed{job_type,outcome=ready|failed|retried|noop}` → `messenger_jobs_processed_total`; `messenger.job.duration` → `messenger_job_duration_seconds_*` |
| End-to-end time to ready | `messenger.job.time_to_ready` (от приёма до ready) → `messenger_job_time_to_ready_seconds_*` |
| Queue depth / oldest job age | `messenger.queue.depth`, `messenger.queue.oldest_job_age` → `messenger_queue_depth`, `messenger_queue_oldest_job_age_seconds` (считает API, т. е. метрика живёт и при остановленном worker) |
| DB pool used/waiting | `db.client.connections.usage{state}`, `db.client.connections.waiting` → `db_client_connections_usage`, `db_client_connections_waiting` |
| Worker busy/capacity | `messenger_worker_busy`, `messenger_worker_capacity` |
| CPU/RAM процессов | `process_memory_rss_bytes`, `process_cpu_time_seconds_total` (счётчик CPU-времени процесса) |
| Логи API и worker | структурный JSON в stdout **и** OTLP: `ts, level, service.name, msg(event), trace_id, span_id` (при активном спане), `request_id`; resource: `service.name` (`messenger-api`/`messenger-worker`), `service.version`, `service.instance.id`, `deployment.environment.name=lab` |
| Трассы API и worker | server-спан запроса (`HTTP POST /api/v1/…`), `db.connection`, `s3.put_object/get_object`, producer-спан `attachment.process publish`, consumer-спан `attachment.process process` — реальные, не статичные |
| Связь upload → publish → consume → S3/DB | W3C `traceparent/tracestate`: контекст запроса сохраняется в `outbox` (миграция 0006), relay кладёт его в поля сообщения (`jobs.schema.json`) и заголовки AMQP; worker продолжает ту же трассу (consumer — потомок producer; producer — потомок server-спана) |
TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения.
**Низкая кардинальность.** `user_id/chat_id/message_id/attachment_id/request_id/filename` не используются как labels метрик (проверяет `test_no_high_cardinality_or_secret_labels`); маршрут — шаблон `/api/v1/chats/{chat_id}/messages`. В логах/спанах допустимы технические ID синтетических данных. **Секреты не экспортируются**: токены, пароли, `Authorization`, тела сообщений, изображения, SQL-текст, query-строки — не попадают ни в атрибуты, ни в логи; дополнительно Collector удаляет `http.request.header.authorization`, `db.statement`, `url.query` и т. п. (`attributes/scrub`). Проверка `test_secrets_and_message_bodies_are_not_exported`: уникальный текст сообщения, токен и пароль не находятся ни в Loki, ни в трассе, ни в метриках.
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
| --- | --- | --- | --- |
| TODO | TODO | TODO | TODO |
**Health-проверки** LB (раз в секунду) не порождают спанов и access-логов (метрики остаются, исключены из SLI фильтром `http_route!~"/health.*"`); фоновые опросы БД (relay, sweeper) — без корневых спанов (`child_span`).
## Ограничения и следующие шаги
### Dashboard и SLI/SLO
TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза.
Dashboard `Messenger: API, очередь, worker` (`infra/grafana/dashboards/messenger.json`, генератор `build_dashboard.py`): RPS по маршрутам и кодам, **доля 5xx**, **p50/p95/p99** (`histogram_quantile` из `sum by (le)` гистограмм всех реплик — не среднее p95), задержки по маршрутам, DB pool (занято/простаивает/ожидают), задания по исходам, p50/p95 длительности обработки и **времени до ready**, глубина очереди, **возраст старейшей задачи**, saturation worker (busy/capacity), RSS/CPU, очередь и отказы экспорта Collector, логи ошибок (Loki), список алертов, последние трассы. Единицы и запросы описаны в подсказках панелей.
- **SLI доступности**: доля бизнес-запросов (все маршруты кроме `/health`) без ответа 5xx. Числитель — 5xx, знаменатель — все запросы без `/health`; 4xx (неверный пароль, чужой чат) в отказ сервиса не входят.
- **SLO**: 99.5 % за окно 30 суток (бюджет ошибок 0.5 %); для демонстрации на локальном стенде окна алертов короткие (rate по 1 мин, `for` 15–20 с).
- Дополнительные SLI: p95 задержки (порог 0.5 с), доля 5xx **на маршруте загрузки** (малообъёмный, но критичный — общий SLI его деградации не заметит), p95 времени до ready.
### Alert rules (`infra/prometheus/rules/messenger.yml`, runbook — `docs/runbooks.md`)
| Алерт | Условие / окно | Для чего |
| --- | --- | --- |
| `HighHttpErrorRate` | 5xx > 5 % при ≥ 0.5 RPS, `for 20s`, rate[1m] | общий отказ сервиса |
| `HighUploadErrorRate` | 5xx на маршруте загрузки > 20 %, `for 15s` | отказ S3/очереди |
| `HighLatencyP95` | p95 > 0.5 с, `for 20s` | медленная БД/обработчик |
| `QueueLagGrowing` | старейшая задача > 30 с, `for 20s` | остановка/перегрузка worker |
| `WorkerSilent` | `absent(messenger_worker_capacity)`, `for 30s` | worker не шлёт метрики |
| `TelemetryExportFailing` | отказы экспорта Collector > 0, `for 15s` | сбой хранилищ телеметрии |
Alerts вычисляет Prometheus; состояние (pending/firing/resolved) видно в Prometheus и на панели «Алерты» dashboard. Внешняя доставка не требуется и не настроена.
### Collector: ограничение ресурсов и поведение при backpressure
`memory_limiter` (limit 280 MiB при лимите контейнера 384 MiB, spike 60 MiB; при приближении данные отклоняются, SDK отбрасывает пакет, приложение продолжает работать), `batch` (1024/2 с), экспортёры с **ограниченной очередью** (`sending_queue` 500) и повторами до 30 с, затем отбрасывание; SDK: очередь спанов/логов 2048, таймаут экспорта 3 с, **отказ Tempo/Loki не нарушил работу API** в воспроизведённом сценарии ниже. `metric_expiration: 20s` настроен в exporter. В текущем опыте остановки worker серия capacity оставалась видимой, поэтому диагноз основан на росте возраста очереди и нулевой скорости ready, а не на `WorkerSilent`.
## Перенос предыдущей работы и воспроизводимость
Перенесён собственный мессенджер из ЛР5: PostgreSQL — пользователи, чаты, сообщения, сессии, идемпотентность, metadata/outbox; Valkey — вспомогательный кэш; приватный SeaweedFS S3 — оригиналы и варианты; RabbitMQ — durable очередь; worker — асинхронная обработка изображений; HAProxy и две API-реплики. Загрузка по-прежнему возвращает 202, текстовое сообщение без вложений допустимо. Бизнес-контракт ЛР6 не меняется.
| Компонент | CPU / RAM | Постоянные данные / готовность |
| --- | --- | --- |
| API ×2 | каждому 1 CPU / 256 MiB | stateless; HTTP `/health/ready` |
| worker | 1 CPU / 768 MiB | состояние в БД/S3/broker; heartbeat |
| PostgreSQL 17.11 | 1 CPU / 512 MiB | `pgdata`; `pg_isready`, миграции отдельным одноразовым сервисом |
| Valkey 8.1.10 | 0.5 CPU / 128 MiB | `valkeydata`; ping |
| SeaweedFS 4.48 | 1 CPU / 768 MiB | `seaweeddata`; bootstrap создаёт приватный bucket и проверяет ограниченные права API |
| RabbitMQ 4.1 (digest) | 1 CPU / 512 MiB | `rabbitmqdata`; diagnostics ping |
| HAProxy 3.0 (digest) | 1 CPU / 128 MiB | HTTP health; опубликован только `127.0.0.1:8080` |
| Collector / Prometheus / Loki / Tempo / Grafana | каждому 1 CPU; 384 / 512 / 512 / 1024 / 512 MiB | volumes и назначение выше; реальную доступность сигналов проверяют собственные тесты |
Секреты и `.env` создаются локально, в Git не входят. `init-secrets.sh` идемпотентен; `.gitattributes` сохраняет LF у shell-скриптов при Windows clone. Внешние зависимости не нужно поднимать отдельно. Workflow `.github/workflows/application.yml` собирает приложение и запускает публичный сценарий; его наличие не означает успешного запуска CI на сервере Gitea.
Выделенные исправления базы ЛР5: HAProxy не повторяет потенциально выполненный upload и сохраняет JSON-ответ приложения при отказе БД; reconciler защищает варианты, staging и незавершённые операции от удаления. Добавлены уже использованные в ЛР5 проверки TTL, зависимостей и reconciler; исправлено имя S3-сервиса в сценарии отказа. Версии и архитектура телеметрии сохранены.
## Доказательства критериев (12.11.2026)
| Проверка | Наблюдаемый результат | Доказательство |
| --- | --- | --- |
| Контракт, полная OpenAPI-валидация и public smoke при включённой телеметрии | PASS | [вывод](evidence/checks_2026-11-12.txt) |
| Собственные проверки | **66 уникальных тестов OK**: 12 API, 9 idempotency, 11 attachments, 6 topology, 9 pipeline, 5 observability, 1 retry trace, 2 reconcile, 7 resilience, 2 expiry, 2 dependencies | [команды](student-tests/README.md), [вывод](evidence/checks_2026-11-12.txt) |
| Реальные metrics/logs/traces, trace/span IDs, отсутствие секретов и высококардинальных labels | 5 observability-тестов OK; после исследования повторены | [вывод](evidence/recheck_2026-11-12/commands.txt) |
| Upload → producer → consumer → S3/DB; повторные попытки | 3 consumer-спана и 2 reschedule в одной трассе, parent-child проверен | [вывод](evidence/checks_2026-11-12.txt) |
| Перезапуск всего стенда | 13 контейнеров перезапущены, volumes сохранены, сессии и подтверждённые данные доступны | [lifecycle](evidence/recheck_2026-11-12/lifecycle.json), [повторная проверка](evidence/recheck_2026-11-12/recovery_checks.txt) |
| Отказ API-реплик по очереди | обе разные реплики остановлены; 120 и 119 успешных записей, 0 неуспешных попыток | [failover](evidence/recheck_2026-11-12/failover.json), [вывод](evidence/recheck_2026-11-12/recovery_checks.txt) |
| S3 bootstrap ×2 | повторный запуск успешен, anonymous 403, создание чужого bucket запрещено | [вывод](evidence/recheck_2026-11-12/recovery_checks.txt) |
| Grafana provisioning | 3 источника, dashboard `messenger-overview`, 22 панели, `provisioned=true` | [ответ API](evidence/recheck_2026-11-12/grafana_provisioning.json) |
| Чистый clone `6d87f54` | новые секреты и пустые volumes, build/up, public smoke и provisioning успешны; отдельный проект, порты 18080/13000 | [вывод](evidence/recheck_2026-11-12/clean_clone.txt), [override портов](evidence/recheck_2026-11-12/clean-ports.yaml) |
| Инциденты, alerts firing → resolved | все 4 воспроизведены | [JSON](evidence/recheck_2026-11-12/incidents.json), разбор ниже |
| Трек 1 | 4 профиля × 2 повтора | [исходные замеры](evidence/recheck_2026-11-12/obs_cost.json), [ресурсы](evidence/recheck_2026-11-12/research_runtime.json) |
На Windows установлен Borland make, поэтому выполнены точные Python-команды целей `make check`, `make validate`, `make test` из Makefile, а не GNU make. Публичные `tests/`, `contracts/`, README, COURSE и CONTRIBUTING совпадают с исходным skeleton `470d9d6`.
Первый persistence-запуск завершился таймаутом recovery hook после успешных 66 тестов. Полный обход S3 не успевал за 45 с на накопленных ~58 тыс. вложений даже при исправном backend. Hook теперь читает первый байт до 16 последних готовых оригиналов; это ограниченная проверка готовности, а не аудит всех исторических объектов. После исправления публичная проверка сохранности и оба failover пройдены. Исходный неуспешный вывод сохранён, повторный — отдельно в `recovery_checks.txt`.
## Контролируемые инциденты
Нагрузка: 4 цикла отправки сообщений с паузой 0.1 с и 10 циклов загрузки PNG с паузой 0.3 с. Реальная интенсивность зависит от задержки/отказа. `research/incidents.py` записывает метрики каждые ~3 с и локальные состояния Prometheus. Времена включают окно rate[1m] и `for` правила.
| Воздействие | Алерт | До firing, с | От команды восстановления до resolved, с |
| --- | --- | ---: | ---: |
| Остановлен worker | `QueueLagGrowing` | 68.3 | 21.8 |
| DB: pg_sleep(0.8) | `HighLatencyP95` | 67.7 | 43.5 |
| Остановлен S3 | `HighUploadErrorRate` | 84.1 | 39.3 |
| Остановлены Tempo и Loki | `TelemetryExportFailing` | 69.1 | 67.5 |
Диагностика по сохранённым наблюдениям (строки `diagnosis` в JSON — шаблоны скрипта; ниже учтены фактические поля, в том числе присутствующая capacity и нулевое ожидание DB pool):
1. **Worker.** Возраст старейшего задания 60.7 с, скорость завершения ready 0.0/с; серия capacity присутствует: `True`. Сочетание роста возраста и отсутствия обработки указывает на worker. После восстановления алерт снят.
2. **Медленная БД.** p95 = 782.8 мс; ожидающих в DB pool: 0.0. Поиск обнаружил 20 трасс с `db.connection >300ms` и 20 медленных HTTP POST сообщений (лимит поиска 20). Задержка видна в спане БД; рост ожидания пула не требуется для этого диагноза.
3. **S3.** 5xx на upload = 100.0 %, общая доля = 1.1 %. Найдены 20 ошибочных трасс и 37 строк `s3_put_failed`. Локализация по маршруту, операциям S3 и логам отделяет этот отказ от общего отказа API.
4. **Tempo/Loki.** API: 62.6 RPS, 5xx = 0.0 %; очередь Collector = 32.0, send-failed spans ≈62.9/с, RSS = 239.0 MiB при лимите 384 MiB. Продолжающиеся бизнес-запросы и ошибки экспортёра показывают отказ пути телеметрии. Prometheus в этом сценарии доступен. Счётчик неуспешных экспортов включает повторы и не равен числу окончательно потерянных уникальных спанов; точное число потерянных логов/трасс этим опытом не установлено.
![Метрики и переходы алертов](evidence/recheck_2026-11-12/incidents.png)
## Трек 1: цена наблюдаемости
**Гипотеза:** инструментирование добавляет CPU/RAM; head 10 % уменьшает ingest, но теряет большинство редких проблемных трасс. Tail policy сохраняет ошибки и задержки >500 мс, отправляя в Tempo лишь 10 % остальных трасс.
**Методика:** 8 процессов-клиентов, 64 одновременных запроса; общие 40 синтетических пользователей и 40 чатов на всю серию, 90 % POST сообщений и 10 % GET. Каждый профиль: прогрев 5 с, замер 30 с, 25 с дозаписи, **2 повтора**; перед профилем пересозданы API/worker/Collector. Во всех профилях одинаковая инъекция: 2 % запросов → 500, 2 % → +0.6 с. RPS, ошибки и latency в таблице относятся к POST-сообщениям; успешные RPS исключают внедрённые ошибки. Для off отключены сигналы API/worker, но контейнеры хранилищ остаются запущены.
Машина: i9-12900KS (16 ядер / 24 потока), 95.8 ГиБ RAM, Windows/Docker Desktop (WSL2); Docker доступны 24 CPU и 46.9 ГиБ RAM. Клиент Python 3.11.9, приложение Python 3.12.12. Лимиты контейнеров одинаковы во всех профилях (таблица выше); другой Docker-проект оставался в фоне. Накопленные данные не очищались. Размеры БД после серии: {'users': 739, 'chats': 425, 'messages': 1610549, 'attachments': 68898}; отдельный счётчик сообщений до серии не снимался. [Размер набора](evidence/recheck_2026-11-12/dataset.json), [хэши исходников и среда](evidence/recheck_2026-11-12/source_metadata.json).
Среднее двух прогонов; RAM — максимальный пик из двух. CPU API — **на одну реплику**, 100 % соответствует одному ядру.
| Профиль | Успешные RPS (min–max) | Ошибки, % | p50 / p95 / p99, мс | CPU API, % | RAM API, MiB | CPU / RAM Collector |
| --- | ---: | ---: | --- | ---: | ---: | --- |
| Без телеметрии | 1368 (1348–1388) | 2.05 | 16.7 / 73.0 / 618.1 | 50.2 | 76.4 | 0.1 % / 45.1 MiB |
| Head 100 % | 1245 (1239–1251) | 2.01 | 24.6 / 100.3 / 629.7 | 79.5 | 83.2 | 3.3 % / 75.0 MiB |
| Head 10 % | 1220 (1137–1303) | 1.98 | 23.9 / 101.5 / 626.8 | 68.5 | 83.7 | 1.9 % / 68.9 MiB |
| SDK 100 % + tail | 1291 (1282–1300) | 2.07 | 23.5 / 90.2 / 628.6 | 77.4 | 75.5 | 4.0 % / 108.4 MiB |
Объём — среднее за измерительное окно с дозаписью; logs и metric points считаются отдельно от spans.
| Профиль | Принято spans | Отправлено spans | Logs | Metric points | Найдено error / внедрено 500 | Найдено slow / медленных у клиента |
| --- | ---: | ---: | ---: | ---: | --- | --- |
| Без телеметрии | 0 | 0 | 0 | 0 | 0 / 873 | 0 / 856 |
| Head 100 % | 89886 | 89886 | 44324 | 396 | 864 / 780 | 876 / 772 |
| Head 10 % | 8898 | 8982 | 43176 | 396 | 85 / 750 | 86 / 736 |
| SDK 100 % + tail | 91882 | 13138 | 45226 | 396 | 916 / 828 | 858 / 768 |
**Результат.** Средний CPU API: off 50.2 %, on100 79.5 %, head10 68.5 %, tail 77.4 %. Изменение on100 относительно off: 29.3 п. п. Head sampling сокращает число спанов, но логи и метрики остаются; поэтому снижение CPU не обязано быть пропорционально sampling rate. При tail SDK продолжает формировать 100 % спанов, а Collector отбирает их перед Tempo: отправлено на 85.7 % меньше, чем принято. Это экономия ingest спанов, не прямое измерение дискового объёма.
**Потери и выбор конфигурации.** Head 10 % сохраняет около 11.3 % ошибочных и 11.8 % медленных трасс по приближённому сопоставлению окон. Tail сохраняет значительно больше проблемных трасс при меньшем ingest, поэтому выбран как проверенная сбалансированная конфигурация: SDK 100 %, `collector-tail.yaml`, все ERROR, все >500 мс, 10 % остальных. Цена — CPU SDK и память Collector для ожидания решения. Для демонстрации/тестов по умолчанию восстановлен on100 без инъекции отказов.
Сумма `receiver_refused_spans` / `exporter_send_failed_spans` по двум повторам: off 0/0, on100 0/0, head10 0/0, tail 0/0. Tempo `discarded_spans_total` до/после серии: null / null. `null` означает отсутствие ряда, а не измеренный ноль. Эти счётчики описывают наблюдавшиеся отказы приёма/экспорта; сами по себе не доказывают отсутствие любых потерь на SDK.
![Сравнение профилей](evidence/recheck_2026-11-12/obs_cost.png)
На графике RPS усреднены по двум прогонам, CPU/ingest/отношения взяты из второго; отношения визуально ограничены 100 %. В таблицах выше приведены средние и исходные количества, подробные результаты каждого повтора — в JSON.
**Ограничения методики:** два коротких повтора на одном ПК с фоновыми задачами, без доверительных интервалов; различия RPS могут быть обычным разбросом. Данные накапливаются, профили идут в фиксированном порядке. Tempo-поиск включает 5 с прогрева, поэтому числитель иногда больше знаменателя: это приближённое сопоставление, не точная вероятность сохранения или доказательство 100 % сохранности. Поиск ограничен 5000 трассами (для всех HTTP лимит достижим); точное число потерянных уникальных спанов SDK не измерено. Размер volumes не используется для вывода: compaction и flush искажают короткие замеры.
## Ограничения решения
- Один Docker-host; БД, LB, broker, worker и каждое хранилище телеметрии — по одному экземпляру. Отказ host/volume не покрывается.
- Проверена остановка Tempo/Loki; отдельная остановка Collector/Prometheus в текущем исследовании не выполнялась. Ограниченные очереди допускают потери телеметрии при длительном простое.
- Recovery hook проверяет недавние готовые оригиналы; публичная persistence-проверка отдельно проверяет созданные ею данные. Это не проверка каждого исторического объекта S3.
- SLO 99.5 % за 30 суток задан как цель; короткие эксперименты не доказывают выполнение месячного SLO. Алерты локальные, без внешней доставки.
- Исторические `evidence/incidents.json`, `evidence/obs_cost.json` и графики от октября оставлены как архив; все выводы текущей сдачи опираются на `evidence/recheck_2026-11-12/`.
+337
View File
@@ -0,0 +1,337 @@
# Стенд lab6: HAProxy -> API x N -> PostgreSQL (метаданные) + SeaweedFS (приватный S3, байты) + Valkey (кэш сессий).
# Секреты: scripts/init-secrets.sh создаёт secrets/* (в Git не попадают). Запуск и проверки — см. REPORT.md.
name: mtusi-lab6
x-common: &common
restart: unless-stopped
logging:
driver: json-file
options: {max-size: "10m", max-file: "3"}
services:
postgres:
<<: *common
image: postgres:17.11-alpine@sha256:b0f9560a2de083e2cc7382e75f808c7381a32852a7ec49117deedb300e552b24
environment:
POSTGRES_USER: messenger
POSTGRES_DB: messenger
POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password
secrets: [postgres_password]
volumes:
- pgdata:/var/lib/postgresql/data
command: ["postgres", "-c", "max_connections=200", "-c", "shared_buffers=128MB", "-c", "log_min_duration_statement=500"]
healthcheck:
test: ["CMD-SHELL", "pg_isready -U messenger -d messenger"]
interval: 5s
timeout: 3s
retries: 12
mem_limit: 512m
cpus: ${POSTGRES_CPUS:-1.0}
networks: [backend]
valkey:
<<: *common
image: valkey/valkey:8.1.10-alpine@sha256:081c2f5cb575efc901aa80ff9cdbd1ec6a301682fd35e1ebb4b0990a4a4a8507
command: ["valkey-server", "--appendonly", "yes", "--appendfsync", "everysec",
"--maxmemory", "64mb", "--maxmemory-policy", "volatile-ttl", "--save", ""]
volumes:
- valkeydata:/data
healthcheck:
test: ["CMD", "valkey-cli", "ping"]
interval: 5s
timeout: 3s
retries: 12
mem_limit: 128m
cpus: 0.5
networks: [backend]
# Приватное S3-совместимое хранилище SeaweedFS (all-in-one: master + volume + filer + S3-шлюз).
# Порты наружу не публикуются: доступ только из сети backend и только по ключам из s3.json (identities).
# Анонимный доступ запрещён (при заданных identities запрос без подписи получает 403).
seaweedfs:
<<: *common
image: chrislusf/seaweedfs:4.48@sha256:4e61d15fd35994cb1e43e1e553dff106794841fd9a99ade2fc8c8bfce4d7872d
entrypoint: ["/bin/sh", "-c"]
command: ["exec weed server -dir=/data -ip=seaweedfs -master.volumeSizeLimitMB=512 -volume.max=30 -filer -s3 -s3.port=8333 -s3.config=/run/secrets/seaweed_s3_config"]
secrets: [seaweed_s3_config]
volumes:
- seaweeddata:/data
healthcheck:
test: ["CMD-SHELL", "wget -q -O /dev/null http://127.0.0.1:9333/cluster/status && nc -z 127.0.0.1 8333"]
interval: 5s
timeout: 3s
retries: 24
start_period: 10s
mem_limit: 768m
cpus: ${S3_CPUS:-1.0}
networks: [backend]
# Идемпотентный bootstrap: bucket создаётся, если его нет; проверяется, что анонимный доступ запрещён и что ключ API работает.
s3-init:
build: .
image: mtusi-messenger:lab6
command: ["python", "-m", "app.s3_bootstrap"]
environment:
S3_ENDPOINT: http://seaweedfs:8333
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
S3_ADMIN_ACCESS_KEY_FILE: /run/secrets/s3_admin_access_key
S3_ADMIN_SECRET_KEY_FILE: /run/secrets/s3_admin_secret_key
S3_ACCESS_KEY_FILE: /run/secrets/s3_access_key
S3_SECRET_KEY_FILE: /run/secrets/s3_secret_key
secrets: [s3_admin_access_key, s3_admin_secret_key, s3_access_key, s3_secret_key]
depends_on:
seaweedfs: {condition: service_healthy}
restart: "no"
networks: [backend]
# Брокер: durable quorum-очередь, persistent-сообщения, publisher confirms; данные на volume rabbitmqdata.
# hostname фиксирован: имя узла RabbitMQ определяет каталог данных, иначе очередь не переживёт пересоздание контейнера.
rabbitmq:
<<: *common
image: rabbitmq:4.1-alpine@sha256:6a0f7984185cd2cdbf78a184b4c17abe88ecbbda6006a7acf17dbc7e4307f4e2
hostname: rabbitmq
# RabbitMQ 4.x: RABBITMQ_DEFAULT_USER/PASS объявлены устаревшими -> учётные данные в конфигурационном файле (secret)
secrets:
- source: rabbitmq_conf
target: /etc/rabbitmq/conf.d/20-credentials.conf
mode: 0444
volumes:
- rabbitmqdata:/var/lib/rabbitmq
healthcheck:
test: ["CMD", "rabbitmq-diagnostics", "-q", "ping"]
interval: 10s
timeout: 8s
retries: 12
mem_limit: 512m
cpus: ${RABBITMQ_CPUS:-1.0}
networks: [backend]
migrate:
build: .
image: mtusi-messenger:lab6
command: ["python", "-m", "app.migrate"]
environment:
DATABASE_URL_FILE: /run/secrets/database_url
secrets: [database_url]
depends_on:
postgres: {condition: service_healthy}
restart: "no"
networks: [backend]
api:
<<: *common
build: .
image: mtusi-messenger:lab6
environment:
APP_ENV: ${APP_ENV:-lab}
DATABASE_URL_FILE: /run/secrets/database_url
VALKEY_URL: redis://valkey:6379/0
S3_ENDPOINT: http://seaweedfs:8333
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
S3_ACCESS_KEY_FILE: /run/secrets/s3_access_key
S3_SECRET_KEY_FILE: /run/secrets/s3_secret_key
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600}
SESSION_CACHE_TTL_SECONDS: ${SESSION_CACHE_TTL_SECONDS:-60}
DB_POOL_MAX: ${DB_POOL_MAX:-10}
DRAIN_SECONDS: ${DRAIN_SECONDS:-4}
MAX_INFLIGHT: ${MAX_INFLIGHT:-64}
ACCESS_LOG: ${ACCESS_LOG:-1}
UPLOAD_CONCURRENCY: ${UPLOAD_CONCURRENCY:-4}
FAULT_INJECTION: ${API_FAULT_INJECTION:-}
BROKER_URL_FILE: /run/secrets/broker_url
MAX_QUEUED_JOBS: ${MAX_QUEUED_JOBS:-200}
SERVICE_NAME: messenger-api
OTEL_ENABLED: ${OTEL_ENABLED:-1}
OTEL_EXPORTER_OTLP_ENDPOINT: http://otel-collector:4318
OTEL_TRACES_SAMPLER_ARG: ${OTEL_TRACES_SAMPLER_ARG:-1.0}
OTEL_QUEUE_SIZE: ${OTEL_QUEUE_SIZE:-2048}
OTEL_BATCH_SIZE: ${OTEL_BATCH_SIZE:-512}
secrets: [database_url, s3_access_key, s3_secret_key, broker_url]
depends_on:
migrate: {condition: service_completed_successfully}
s3-init: {condition: service_completed_successfully}
valkey: {condition: service_healthy}
stop_grace_period: 15s
mem_limit: ${API_MEM:-256m}
cpus: ${API_CPUS:-1.0}
deploy:
replicas: ${API_REPLICAS:-2}
networks: [backend]
# Worker: отдельный процесс (тот же образ). Берёт задания из RabbitMQ, обрабатывает изображения, пишет результат в S3 и БД.
worker:
<<: *common
build: .
image: mtusi-messenger:lab6
command: ["python", "-m", "app.worker"]
environment:
SERVICE_NAME: messenger-worker
DATABASE_URL_FILE: /run/secrets/database_url
S3_ENDPOINT: http://seaweedfs:8333
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
S3_ACCESS_KEY_FILE: /run/secrets/s3_access_key
S3_SECRET_KEY_FILE: /run/secrets/s3_secret_key
BROKER_URL_FILE: /run/secrets/broker_url
WORKER_CONCURRENCY: ${WORKER_CONCURRENCY:-2}
MAX_ATTEMPTS: ${MAX_ATTEMPTS:-5}
LEASE_SECONDS: ${LEASE_SECONDS:-60}
RETRY_BASE_DELAY: ${RETRY_BASE_DELAY:-2}
FAULT_INJECTION: ${WORKER_FAULT_INJECTION:-}
OTEL_ENABLED: ${OTEL_ENABLED:-1}
OTEL_EXPORTER_OTLP_ENDPOINT: http://otel-collector:4318
OTEL_TRACES_SAMPLER_ARG: ${OTEL_TRACES_SAMPLER_ARG:-1.0}
OTEL_QUEUE_SIZE: ${OTEL_QUEUE_SIZE:-2048}
OTEL_BATCH_SIZE: ${OTEL_BATCH_SIZE:-512}
ACCESS_LOG: "0"
secrets: [database_url, s3_access_key, s3_secret_key, broker_url]
depends_on:
migrate: {condition: service_completed_successfully}
s3-init: {condition: service_completed_successfully}
rabbitmq: {condition: service_healthy}
healthcheck:
test: ["CMD", "python", "-c", "import os,time,sys; sys.exit(0 if time.time()-os.path.getmtime('/tmp/worker.alive') < 30 else 1)"]
interval: 10s
timeout: 5s
retries: 3
start_period: 20s
stop_grace_period: 30s
mem_limit: ${WORKER_MEM:-768m}
cpus: ${WORKER_CPUS:-1.0}
deploy:
replicas: ${WORKER_REPLICAS:-1}
networks: [backend]
# ------------------------------ наблюдаемость (lab6) ------------------------------
# OTLP-порты 4317/4318 и метрики Collector 8888/8889 доступны только внутри сети backend (наружу не публикуются).
otel-collector:
<<: *common
image: otel/opentelemetry-collector-contrib:0.161.0@sha256:fd328de2552466ad78385e1b1289c3f2402b1c45f265b252aab1955b42845ac1
command: ["--config=${COLLECTOR_CONFIG:-/etc/otelcol/collector.yaml}"]
volumes:
- ./infra/otel:/etc/otelcol:ro
mem_limit: 384m
cpus: ${COLLECTOR_CPUS:-1.0}
networks: [backend]
prometheus:
<<: *common
image: prom/prometheus:v3.15.0@sha256:efd719c99d83b060d9daefdcf00360461adf279f45ef5391f8d111892118753e
command: ["--config.file=/etc/prometheus/prometheus.yml", "--storage.tsdb.path=/prometheus", "--storage.tsdb.retention.time=7d",
"--web.enable-lifecycle"]
volumes:
- ./infra/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro
- ./infra/prometheus/rules:/etc/prometheus/rules:ro
- promdata:/prometheus
healthcheck:
test: ["CMD", "wget", "-q", "-O", "/dev/null", "http://127.0.0.1:9090/-/ready"]
interval: 10s
timeout: 3s
retries: 6
mem_limit: 512m
cpus: 1.0
networks: [backend]
loki:
<<: *common
image: grafana/loki:3.7.8@sha256:1107dd5274e0ada47e42472b7a7e71f3b2a2fe878878108f3e2f9e51528f0193
command: ["-config.file=/etc/loki/loki.yaml"]
volumes:
- ./infra/loki/loki.yaml:/etc/loki/loki.yaml:ro
- lokidata:/loki
mem_limit: 512m
cpus: 1.0
networks: [backend]
tempo:
<<: *common
image: grafana/tempo:2.9.0@sha256:65a5789759435f1ef696f1953258b9bbdb18eb571d5ce711ff812d2e128288a4
command: ["-config.file=/etc/tempo/tempo.yaml"]
volumes:
- ./infra/tempo/tempo.yaml:/etc/tempo/tempo.yaml:ro
- tempodata:/var/tempo
mem_limit: 1g
cpus: 1.0
networks: [backend]
# Grafana: единственный опубликованный сервис наблюдаемости, только 127.0.0.1:3000. Datasources, dashboard и alert-вью
# подключаются provisioning-ом из Git (infra/grafana). Анонимный доступ выключен; пароль admin — secret.
grafana:
<<: *common
image: grafana/grafana:13.2.3@sha256:b28bae15e219c998fb0e0424ed724930cc61b1f61fb404d47c862f9a23f9e572
environment:
GF_SECURITY_ADMIN_USER: admin
GF_SECURITY_ADMIN_PASSWORD__FILE: /run/secrets/grafana_admin_password
GF_AUTH_ANONYMOUS_ENABLED: "false"
GF_USERS_ALLOW_SIGN_UP: "false"
GF_ANALYTICS_REPORTING_ENABLED: "false"
GF_DASHBOARDS_DEFAULT_HOME_DASHBOARD_PATH: /var/lib/grafana/dashboards/messenger.json
secrets: [grafana_admin_password]
volumes:
- ./infra/grafana/provisioning:/etc/grafana/provisioning:ro
- ./infra/grafana/dashboards:/var/lib/grafana/dashboards:ro
- grafanadata:/var/lib/grafana
depends_on: [prometheus, loki, tempo]
ports:
- "127.0.0.1:3000:3000"
healthcheck:
test: ["CMD-SHELL", "wget -q -O /dev/null http://127.0.0.1:3000/api/health"]
interval: 10s
timeout: 3s
retries: 12
mem_limit: 512m
cpus: 1.0
networks: [backend]
lb:
<<: *common
image: haproxy:3.0-alpine@sha256:56b887da77428b7a6621e59e480cdbd330cc805c22d3cedb66ceea76ffdea2c6
volumes:
- ./infra/haproxy/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro
depends_on:
- api
ports:
- "127.0.0.1:8080:8080"
healthcheck:
test: ["CMD-SHELL", "wget -q -O /dev/null http://127.0.0.1:8081/healthz"]
interval: 5s
timeout: 3s
retries: 6
mem_limit: 128m
cpus: ${LB_CPUS:-1.0}
stop_grace_period: 10s
networks: [backend]
secrets:
postgres_password:
file: ./secrets/postgres_password
database_url:
file: ./secrets/database_url
seaweed_s3_config:
file: ./secrets/seaweed_s3.json
s3_admin_access_key:
file: ./secrets/s3_admin_access_key
s3_admin_secret_key:
file: ./secrets/s3_admin_secret_key
s3_access_key:
file: ./secrets/s3_access_key
s3_secret_key:
file: ./secrets/s3_secret_key
rabbitmq_conf:
file: ./secrets/rabbitmq.conf
grafana_admin_password:
file: ./secrets/grafana_admin_password
broker_url:
file: ./secrets/broker_url
volumes:
pgdata:
valkeydata:
seaweeddata:
rabbitmqdata:
promdata:
lokidata:
tempodata:
grafanadata:
networks:
backend:
+41
View File
@@ -0,0 +1,41 @@
## Лабораторная 6
Автор / группа: Кушнаренко Николай Вячеславович, БПИ2302
Целевая оценка (3/4/5): 5
Трек 5, если заявлен: Трек 1 — цена наблюдаемости
Что реализовано и что перенесено:
Собственный мессенджер из ЛР5: две API-реплики, HAProxy, PostgreSQL, Valkey, приватный S3, RabbitMQ и worker. Бизнес-API сохранён. Добавлены OpenTelemetry SDK/Collector, Prometheus, Loki, Tempo и Grafana; provisioning трёх источников и dashboard из 22 панелей, гистограммы, SLI/SLO и шесть alert rules. Контекст W3C проходит API → outbox → broker → worker → S3/DB, включая retry. Есть memory limiter, batch, ограниченные очереди, удаление чувствительных атрибутов. Исправления базы ЛР5 (HAProxy/reconciler), Windows LF и ограниченный S3 recovery hook выделены в REPORT.md.
Команды запуска из чистого clone:
```sh
sh scripts/init-secrets.sh
cp .env.example .env
docker compose --env-file .env config --quiet
docker compose --env-file .env up -d --build --wait --wait-timeout 180
python3 scripts/check_contract.py
python3 tests/smoke.py --base-url http://127.0.0.1:8080
```
Полные команды своих тестов и исследований — [student-tests/README.md](https://gitea.privcloud.tech/kolya/mtusi2026lab6/src/branch/solution/bpi2302-kushnarenko/student-tests/README.md).
Результаты проверок и ссылка на REPORT.md:
[REPORT.md](https://gitea.privcloud.tech/kolya/mtusi2026lab6/src/branch/solution/bpi2302-kushnarenko/REPORT.md), [Word](https://gitea.privcloud.tech/kolya/mtusi2026lab6/src/branch/solution/bpi2302-kushnarenko/docs/report_lab6.docx), [вывод 66 тестов](https://gitea.privcloud.tech/kolya/mtusi2026lab6/src/branch/solution/bpi2302-kushnarenko/evidence/checks_2026-11-12.txt), [текущие доказательства](https://gitea.privcloud.tech/kolya/mtusi2026lab6/src/branch/solution/bpi2302-kushnarenko/evidence/recheck_2026-11-12).
- контракт, полная OpenAPI-валидация, публичный smoke — PASS; 66 собственных тестов — OK;
- отдельный чистый clone: build/up на новых секретах и пустых volumes, public smoke и provisioning Grafana — PASS;
- перезапуск 13 контейнеров сохранил volumes, сессии и данные; обе API-реплики проверены отдельными failover;
- четыре инцидента (worker, медленная БД, S3, Tempo/Loki): alerts firing → resolved; отказ Tempo/Loki не нарушил бизнес-запросы;
- трек 1: off/on100/head10/tail, одинаковая нагрузка, два повтора; latency, CPU/RAM, ingest и сохранение редких трасс измерены. Tail уменьшил отправку спанов относительно приёма на 85.7 %;
- на Windows выполнены Python-эквиваленты целей make check/validate/test. Серверный CI Gitea не заявляется как пройденный.
Ограничения / известные ошибки:
Один Docker-host и по одному экземпляру stateful-компонентов; возможны потери телеметрии при простое backend. Два коротких повтора, фоновые процессы и накопленная БД ограничивают точность сравнения. TraceQL-окно включает прогрев, поэтому отношения сохранённых трасс приближённые. Первый полный S3 recovery probe превысил таймаут на ~58 тыс. вложений; после ограничения до 16 последних оригиналов persistence/failover прошли, оба вывода сохранены.
Источники и использование ИИ:
Собственный код ЛР5, Python/uvicorn, asyncpg, aiobotocore, aio-pika, Pillow, OpenTelemetry и указанные хранилища. Первичные материалы — ссылки README; версии закреплены в репозитории. Claude/Codex помогали с кодом, тестами и оформлением; результаты проверены реальными запусками.
- [x] Весь обязательный минимум текущей и предыдущих работ сохранён с объявленными переходами.
- [x] Контракт и публичные тесты не ослаблены.
- [x] make check и make test выполнены эквивалентными Python-командами; приложен реальный вывод.
- [x] Сценарии отказов/безопасности своего уровня воспроизведены.
- [x] Все секреты и приватные ключи находятся вне Git; отчёт не содержит tokens.
- [x] README задания сохранён, REPORT.md содержит инструкцию именно моего решения.
Binary file not shown.
+34
View File
@@ -0,0 +1,34 @@
# Runbooks (локальный стенд lab6)
Grafana — `http://127.0.0.1:3000` (пользователь `admin`, пароль — `secrets/grafana_admin_password`), dashboard «Messenger: API, очередь, worker».
Prometheus/Loki/Tempo доступны только внутри сети стенда: запросы — через Grafana (Explore) либо `docker compose exec prometheus wget -qO- http://loki:3100/...`.
Общий порядок диагностики: **метрика → трасса → лог**. Метрика показывает *что* и *когда* (алерт, dashboard), трасса — *где* потрачено время/где ошибка
(по `trace_id` из лога или из Tempo), лог — *почему* (событие и безопасные поля без тел сообщений и токенов).
Связка лог → трасса: в Loki у каждой записи есть `trace_id`/`span_id` (в Grafana — ссылка «Tempo»); трасса → логи: кнопка «Logs for this span».
## HighErrorRate
Алерт `HighHttpErrorRate`: доля ответов 5xx среди бизнес-запросов (без `/health`) > 5 % при ≥ 0.5 запроса/с, 20 с. 4xx (неверный пароль, 404) в SLI не входят.
1. Dashboard → «RPS по статус-кодам» и «Доля ошибок»: какой маршрут даёт 5xx (`http_route` в `sum by (http_route)`).
2. Tempo: TraceQL `{ status = error && resource.service.name = "messenger-api" }` — в трассе виден спан с ошибкой (`s3.put_object`, `db.connection`, …) и `http.response.status_code`.
3. Loki: `{service_name="messenger-api"} | status >= 500` или поиск событий `s3_put_failed`, `db_unavailable`, `unhandled_exception` (поле `error_type`).
4. Действия: S3 недоступен → `docker compose ps seaweedfs`, `docker compose start seaweedfs`; БД → `docker compose ps postgres`, readiness реплик; затем алерт возвращается в resolved через ~1 мин (окно `rate[1m]`).
## HighLatency
Алерт `HighLatencyP95`: p95 всех бизнес-запросов (из суммарной гистограммы всех реплик) > 0.5 с.
1. Dashboard «Latency p50/p95/p99» и «Latency p95 по маршрутам» → какой маршрут.
2. Отличить **медленную БД** от **медленного обработчика**: «DB pool: занято/ожидают» (рост `waiting`) и длительность спанов `db.connection` в медленной трассе (`{ name = "db.connection" && duration > 300ms }`). Если почти всё время запроса внутри `db.connection` — БД; если время в самом серверном спане вне дочерних — код приложения/CPU (`process_cpu_time`).
3. БД: `log_min_duration_statement=500` пишет медленные запросы в лог `postgres`; проверить блокировки/триггеры (`pg_stat_activity`).
## QueueLag
Алерт `QueueLagGrowing`: возраст старейшего задания (`queued`/`processing`) > 30 с, 20 с.
1. Dashboard «Возраст старейшей задачи», «Глубина очереди», «Worker: занято/ёмкость», «Задания по исходам».
2. Нет метрик worker / `busy=0` и растёт возраст → worker остановлен или не подключён к брокеру: `docker compose ps worker rabbitmq`, `docker compose logs worker`. `busy == capacity` при росте очереди → не хватает мощности (увеличить `WORKER_CONCURRENCY`/реплики, см. отчёт lab5).
3. Много `retried`/`failed` → причина в логах worker (`job_rescheduled`, `reason`) и в трассе consumer-спана (S3/БД). Реестр: `docker compose run --rm -T worker python -m app.replay list`.
4. После устранения причины очередь рассасывается, алерт переходит в resolved.
## Telemetry
Алерт `TelemetryExportFailing`: Collector не может отправить данные в Tempo/Loki.
1. Это сбой наблюдаемости, а не приложения: бизнес-метрики API (ошибки, RPS) не меняются, приложение работает (SDK и Collector ограничивают очередь/RAM и отбрасывают данные).
2. Проверить: `docker compose ps tempo loki`; метрики Collector `otelcol_exporter_queue_size`, `otelcol_exporter_send_failed_*`, `otelcol_process_memory_rss`.
3. Запустить хранилища: `docker compose start tempo loki`. Часть трасс/логов может потеряться после исчерпания очереди или срока повторов. `send_failed` считает неуспешные попытки, включая повторы, и не даёт точного числа потерянных уникальных спанов. При остановке только Tempo/Loki метрики продолжают идти по отдельному пути Collector → Prometheus.
+66
View File
@@ -0,0 +1,66 @@
$ python3 scripts/check_contract.py
PASS: offline skeleton structure, local refs and Python syntax; application NOT tested.
$ python3 tests/smoke.py (телеметрия включена)
PASS: lab6 public API scenario; infrastructure and grading evidence remain separate.
# контейнеры до перезапуска:
/mtusi-lab6-api-2 StartedAt=2026-10-03T02:04:24.831646035Z
/mtusi-lab6-api-1 StartedAt=2026-10-03T02:04:24.660308554Z
/mtusi-lab6-worker-1 StartedAt=2026-10-03T02:04:23.350829139Z
/mtusi-lab6-rabbitmq-1 StartedAt=2026-10-03T02:04:19.141968204Z
/mtusi-lab6-seaweedfs-1 StartedAt=2026-10-03T02:04:19.613928237Z
/mtusi-lab6-postgres-1 StartedAt=2026-10-03T02:04:18.665413808Z
/mtusi-lab6-valkey-1 StartedAt=2026-10-03T02:04:18.435301589Z
/mtusi-lab6-otel-collector-1 StartedAt=2026-10-03T02:04:19.399662403Z
/mtusi-lab6-prometheus-1 StartedAt=2026-10-03T02:04:16.212964886Z
/mtusi-lab6-loki-1 StartedAt=2026-10-03T02:04:19.815321273Z
/mtusi-lab6-tempo-1 StartedAt=2026-10-03T02:04:18.87934529Z
/mtusi-lab6-grafana-1 StartedAt=2026-10-03T02:04:18.083530623Z
$ python3 tests/resilience.py --mode persistence --action-script scripts/restart.sh (Windows: restart.cmd; перезапускаются API, worker, брокер, S3, БД, кэш и стек наблюдаемости)
Executing explicit persistence hook; hook output suppressed to avoid leaking secrets.
PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence.
# контейнеры после перезапуска:
/mtusi-lab6-api-2 StartedAt=2026-10-03T02:05:18.916843054Z
/mtusi-lab6-api-1 StartedAt=2026-10-03T02:05:20.476426854Z
/mtusi-lab6-worker-1 StartedAt=2026-10-03T02:05:19.075989142Z
/mtusi-lab6-rabbitmq-1 StartedAt=2026-10-03T02:05:15.668495976Z
/mtusi-lab6-seaweedfs-1 StartedAt=2026-10-03T02:05:15.807584366Z
/mtusi-lab6-postgres-1 StartedAt=2026-10-03T02:05:14.948208653Z
/mtusi-lab6-valkey-1 StartedAt=2026-10-03T02:05:14.108577454Z
/mtusi-lab6-otel-collector-1 StartedAt=2026-10-03T02:05:13.357193004Z
/mtusi-lab6-prometheus-1 StartedAt=2026-10-03T02:05:14.678083736Z
/mtusi-lab6-loki-1 StartedAt=2026-10-03T02:05:15.497739002Z
/mtusi-lab6-tempo-1 StartedAt=2026-10-03T02:05:15.099095895Z
/mtusi-lab6-grafana-1 StartedAt=2026-10-03T02:05:15.320432594Z
$ failover (1-я реплика)
Executing explicit failover hook; hook output suppressed to avoid leaking secrets.
{"observed_instances_before": 2, "successful_writes": 112, "failed_attempts": 0, "first_recovery_seconds": 2.36}
PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.
$ failover (2-я реплика)
Executing explicit failover hook; hook output suppressed to avoid leaking secrets.
{"observed_instances_before": 2, "successful_writes": 109, "failed_attempts": 0, "first_recovery_seconds": 2.375}
PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.
$ python -m unittest test_api.py
Ran 12 tests in 2.950s
OK
$ python -m unittest test_idempotency.py
Ran 9 tests in 2.070s
OK
$ python -m unittest test_attachments.py
Ran 11 tests in 6.104s
OK
$ python student-tests/test_pipeline.py (Pillow)
Ran 9 tests in 11.272s
OK
$ python student-tests/test_topology.py
Ran 6 tests in 2.857s
OK
$ python student-tests/test_observability.py (метрики, логи, трассы, связь API->worker, нет секретов)
Ran 5 tests in 7.658s
OK
$ python student-tests/test_observability_retry.py (все попытки задания — одна трасса)
Ran 1 test in 18.312s
OK
$ python student-tests/test_resilience5.py (окна отказа конвейера, телеметрия включена)
Ran 7 tests in 139.292s
OK
CHECKS-DONE
+182
View File
@@ -0,0 +1,182 @@
$ python scripts/check_contract.py
PASS: offline skeleton structure, local refs and Python syntax; application NOT tested.
exit=0
$ python scripts/validate_openapi.py
PASS: full OpenAPI 3.1 and JSON Schema validation; application NOT tested.
exit=0
$ python tests/smoke.py --base-url http://127.0.0.1:8080
PASS: lab6 public API scenario; infrastructure and grading evidence remain separate.
exit=0
$ python -m unittest discover -s student-tests -p test_api.py -v
test_bearer_rules (test_api.AuthTest.test_bearer_rules) ... ok
test_concurrent_registration_of_same_username (test_api.AuthTest.test_concurrent_registration_of_same_username) ... ok
test_logout_revokes_in_database_and_cache (test_api.AuthTest.test_logout_revokes_in_database_and_cache) ... ok
test_password_is_not_exposed_or_stored_plain (test_api.AuthTest.test_password_is_not_exposed_or_stored_plain) ... ok
test_register_validation (test_api.AuthTest.test_register_validation) ... ok
test_session_shape_entropy_and_logout (test_api.AuthTest.test_session_shape_entropy_and_logout) ... ok
test_token_only_stored_as_hash (test_api.AuthTest.test_token_only_stored_as_hash) ... ok
test_wrong_password_and_unknown_user_are_indistinguishable (test_api.AuthTest.test_wrong_password_and_unknown_user_are_indistinguishable) ... ok
test_chat_creation_is_atomic (test_api.DataTest.test_chat_creation_is_atomic) ... ok
test_foreign_chat_empty_message_bad_json (test_api.DataTest.test_foreign_chat_empty_message_bad_json) ... ok
test_multi_page_history_and_cursor_binding (test_api.DataTest.test_multi_page_history_and_cursor_binding) ... ok
test_parallel_writes_from_many_clients (test_api.DataTest.test_parallel_writes_from_many_clients) ... ok
----------------------------------------------------------------------
Ran 12 tests in 2.581s
OK
exit=0
$ python -m unittest discover -s student-tests -p test_idempotency.py -v
test_concurrent_same_key_creates_exactly_one_message_across_replicas (test_idempotency.IdempotencyTest.test_concurrent_same_key_creates_exactly_one_message_across_replicas) ... ok
test_different_body_same_key_is_409 (test_idempotency.IdempotencyTest.test_different_body_same_key_is_409) ... ok
test_expired_key_is_reusable_after_ttl (test_idempotency.IdempotencyTest.test_expired_key_is_reusable_after_ttl) ... ok
test_key_validation (test_idempotency.IdempotencyTest.test_key_validation) ... ok
test_replay_rechecks_session_and_membership (test_idempotency.IdempotencyTest.test_replay_rechecks_session_and_membership) ... ok
test_replay_returns_same_message_and_creates_no_duplicate (test_idempotency.IdempotencyTest.test_replay_returns_same_message_and_creates_no_duplicate) ... ok
test_scope_is_user_and_chat (test_idempotency.IdempotencyTest.test_scope_is_user_and_chat) ... ok
test_semantically_equal_body_is_replayed (test_idempotency.IdempotencyTest.test_semantically_equal_body_is_replayed) ... ok
test_session_is_shared_between_replicas_without_relogin (test_idempotency.IdempotencyTest.test_session_is_shared_between_replicas_without_relogin) ... ok
----------------------------------------------------------------------
Ran 9 tests in 2.087s
OK
exit=0
$ python -m unittest discover -s student-tests -p test_attachments.py -v
test_aborted_upload_leaves_nothing (test_attachments.AttachmentTest.test_aborted_upload_leaves_nothing) ... ok
test_bucket_is_private_outside_the_api (test_attachments.AttachmentTest.test_bucket_is_private_outside_the_api) ... ok
test_corrupted_and_spoofed_files_are_415_not_500 (test_attachments.AttachmentTest.test_corrupted_and_spoofed_files_are_415_not_500) ... ok
test_dangerous_filenames_are_400 (test_attachments.AttachmentTest.test_dangerous_filenames_are_400) ... ok
test_decompression_bomb_header_is_413 (test_attachments.AttachmentTest.test_decompression_bomb_header_is_413) ... ok
test_empty_body_is_400 (test_attachments.AttachmentTest.test_empty_body_is_400) ... ok
test_foreign_user_cannot_upload_read_or_download (test_attachments.AttachmentTest.test_foreign_user_cannot_upload_read_or_download) ... ok
test_idempotent_replay_with_attachments (test_attachments.AttachmentTest.test_idempotent_replay_with_attachments) ... ok
test_message_with_attachments_and_chat_binding (test_attachments.AttachmentTest.test_message_with_attachments_and_chat_binding) ... ok
test_png_and_jpeg_roundtrip_with_checksum (test_attachments.AttachmentTest.test_png_and_jpeg_roundtrip_with_checksum) ... ok
test_too_large_is_413_and_nothing_is_stored (test_attachments.AttachmentTest.test_too_large_is_413_and_nothing_is_stored) ... ok
----------------------------------------------------------------------
Ran 11 tests in 5.449s
OK
exit=0
$ python -m unittest discover -s student-tests -p test_topology.py -v
test_api_runs_as_non_root (test_topology.TopologyTest.test_api_runs_as_non_root) ... ok
test_at_least_two_api_replicas_without_published_ports (test_topology.TopologyTest.test_at_least_two_api_replicas_without_published_ports) ... ok
test_no_fixed_container_names (test_topology.TopologyTest.test_no_fixed_container_names) ... ok
test_only_lb_is_published_and_only_on_localhost (test_topology.TopologyTest.test_only_lb_is_published_and_only_on_localhost) ... ok
test_resource_limits_are_set (test_topology.TopologyTest.test_resource_limits_are_set) ... ok
test_stop_grace_period_covers_drain (test_topology.TopologyTest.test_stop_grace_period_covers_drain) ... ok
----------------------------------------------------------------------
Ran 6 tests in 2.739s
OK
exit=0
$ python -m unittest discover -s student-tests -p test_pipeline.py -v
test_alpha_is_composited_on_white (test_pipeline.PipelineTest.test_alpha_is_composited_on_white) ... C:\Users\kolya5544\source\repos\vmdMTUCI\mtusi2026lab6\student-tests\test_pipeline.py:25: ResourceWarning: unclosed file <_io.BufferedReader name='C:\\Users\\kolya5544\\source\\repos\\vmdMTUCI\\mtusi2026lab6\\student-tests\\fixtures\\alpha.png'>
return open(os.path.join(FIX, name), "rb").read()
ResourceWarning: Enable tracemalloc to get the object allocation traceback
ok
test_content_is_409_while_not_ready_and_original_is_untouched (test_pipeline.PipelineTest.test_content_is_409_while_not_ready_and_original_is_untouched) ... ok
test_exif_orientation_is_applied (test_pipeline.PipelineTest.test_exif_orientation_is_applied) ... C:\Users\kolya5544\source\repos\vmdMTUCI\mtusi2026lab6\student-tests\test_pipeline.py:25: ResourceWarning: unclosed file <_io.BufferedReader name='C:\\Users\\kolya5544\\source\\repos\\vmdMTUCI\\mtusi2026lab6\\student-tests\\fixtures\\exif_rot6.jpg'>
return open(os.path.join(FIX, name), "rb").read()
ResourceWarning: Enable tracemalloc to get the object allocation traceback
ok
test_message_can_reference_attachment_before_ready_and_failed_cannot (test_pipeline.PipelineTest.test_message_can_reference_attachment_before_ready_and_failed_cannot) ... C:\Users\kolya5544\source\repos\vmdMTUCI\mtusi2026lab6\student-tests\test_pipeline.py:25: ResourceWarning: unclosed file <_io.BufferedReader name='C:\\Users\\kolya5544\\source\\repos\\vmdMTUCI\\mtusi2026lab6\\student-tests\\fixtures\\poison.png'>
return open(os.path.join(FIX, name), "rb").read()
ResourceWarning: Enable tracemalloc to get the object allocation traceback
ok
test_other_variants_visible_only_to_chat_members (test_pipeline.PipelineTest.test_other_variants_visible_only_to_chat_members) ... ok
test_ready_job_redelivery_does_not_duplicate_variants (test_pipeline.PipelineTest.test_ready_job_redelivery_does_not_duplicate_variants) ... ok
test_sample_png_produces_all_variants_per_contract (test_pipeline.PipelineTest.test_sample_png_produces_all_variants_per_contract) ... ok
test_small_image_is_not_upscaled_but_crop_is_128 (test_pipeline.PipelineTest.test_small_image_is_not_upscaled_but_crop_is_128) ... C:\Users\kolya5544\source\repos\vmdMTUCI\mtusi2026lab6\student-tests\test_pipeline.py:25: ResourceWarning: unclosed file <_io.BufferedReader name='C:\\Users\\kolya5544\\source\\repos\\vmdMTUCI\\mtusi2026lab6\\student-tests\\fixtures\\small.png'>
return open(os.path.join(FIX, name), "rb").read()
ResourceWarning: Enable tracemalloc to get the object allocation traceback
ok
test_staging_is_cleaned_after_ready_and_kept_for_failed (test_pipeline.PipelineTest.test_staging_is_cleaned_after_ready_and_kept_for_failed) ... ok
----------------------------------------------------------------------
Ran 9 tests in 10.668s
OK
exit=0
$ python -m unittest discover -s student-tests -p test_observability.py -v
test_logs_are_structured_and_linked_by_trace_id (test_observability.ObservabilityTest.test_logs_are_structured_and_linked_by_trace_id) ... ok
test_metrics_for_http_jobs_queue_pool_worker (test_observability.ObservabilityTest.test_metrics_for_http_jobs_queue_pool_worker) ... ok
test_no_high_cardinality_or_secret_labels (test_observability.ObservabilityTest.test_no_high_cardinality_or_secret_labels) ... ok
test_one_trace_spans_api_outbox_broker_and_worker (test_observability.ObservabilityTest.test_one_trace_spans_api_outbox_broker_and_worker) ... ok
test_secrets_and_message_bodies_are_not_exported (test_observability.ObservabilityTest.test_secrets_and_message_bodies_are_not_exported) ... ok
----------------------------------------------------------------------
Ran 5 tests in 9.932s
OK
exit=0
$ python -m unittest discover -s student-tests -p test_observability_retry.py -v
test_all_attempts_in_one_trace_and_logs (test_observability_retry.RetryTraceTest.test_all_attempts_in_one_trace_and_logs) ... ok
----------------------------------------------------------------------
Ran 1 test in 19.482s
OK
exit=0
$ python -m unittest discover -s student-tests -p test_reconcile5.py -v
test_ready_variants_and_uncommitted_side_effect_are_preserved (test_reconcile5.ReconcilePipelineTest.test_ready_variants_and_uncommitted_side_effect_are_preserved) ... ok
test_staging_is_protected_and_unowned_orphan_obeys_grace (test_reconcile5.ReconcilePipelineTest.test_staging_is_protected_and_unowned_orphan_obeys_grace) ... ok
----------------------------------------------------------------------
Ran 2 tests in 34.423s
OK
exit=0
$ python -m unittest discover -s student-tests -p test_resilience5.py -v
test_1_worker_down_upload_api_restart_worker_up (test_resilience5.ResilienceTest.test_1_worker_down_upload_api_restart_worker_up) ... ok
test_2_broker_restart_keeps_accepted_jobs (test_resilience5.ResilienceTest.test_2_broker_restart_keeps_accepted_jobs) ... ok
test_3_broker_down_during_acceptance_outbox_delivers_later (test_resilience5.ResilienceTest.test_3_broker_down_during_acceptance_outbox_delivers_later) ... ok
test_4_api_crash_between_commit_and_publish (test_resilience5.ResilienceTest.test_4_api_crash_between_commit_and_publish) ... ok
test_5_worker_crash_after_s3_before_commit_lease_recovery (test_resilience5.ResilienceTest.test_5_worker_crash_after_s3_before_commit_lease_recovery) ... ok
test_6_worker_crash_after_commit_before_ack_redelivery_is_noop (test_resilience5.ResilienceTest.test_6_worker_crash_after_commit_before_ack_redelivery_is_noop) ... ok
test_7_retry_exhaustion_registry_and_operator_replay (test_resilience5.ResilienceTest.test_7_retry_exhaustion_registry_and_operator_replay) ... ok
----------------------------------------------------------------------
Ran 7 tests in 146.730s
OK
exit=0
$ python -m unittest discover -s student-tests -p test_expiry.py -v
test_expiry_is_not_extended_by_use (test_expiry.ExpiryTest.test_expiry_is_not_extended_by_use) ... ok
test_session_expires_after_ttl (test_expiry.ExpiryTest.test_session_expires_after_ttl) ... ok
----------------------------------------------------------------------
Ran 2 tests in 21.421s
OK
exit=0
$ python -m unittest discover -s student-tests -p test_dependencies.py -v
test_postgres_down_gives_bounded_503_never_anonymous_access (test_dependencies.DependencyTest.test_postgres_down_gives_bounded_503_never_anonymous_access) ... ok
test_valkey_down_is_bypassed (test_dependencies.DependencyTest.test_valkey_down_is_bypassed) ... ok
----------------------------------------------------------------------
Ran 2 tests in 21.620s
OK
exit=0
$ python tests/resilience.py --mode persistence --action-script scripts/restart.cmd --base-url http://127.0.0.1:8080
Executing explicit persistence hook; hook output suppressed to avoid leaking secrets.
FAIL: restart hook failed (inspect your script locally)
exit=1
+35
View File
@@ -0,0 +1,35 @@
$ curl -u admin:*** http://127.0.0.1:3000/api/search (dashboard импортирован provisioning-ом, а не вручную)
dash-folder Messenger eg028qble26m8e
dash-db Messenger: API, очередь, worker messenger-overview
$ curl -u admin:*** .../api/datasources (datasources из infra/grafana/provisioning)
Loki loki loki provisioned= True
Prometheus prometheus prometheus provisioned= True
Tempo tempo tempo provisioned= True
health prometheus: OK
health loki: OK
health tempo: OK
$ curl -u admin:*** .../api/dashboards/uid/messenger-overview (панели)
provisioned: True | �������: 22
- RPS по маршрутам
- Доля ошибок 5xx (SLI)
- Latency p50 / p95 / p99
- RPS по статус-кодам
- Latency p95 по маршрутам
- DB pool: занято / простаивает / ожидают
- Задания по исходам
- Длительность обработки p50 / p95 (worker)
- Время до ready p50 / p95
- Глубина очереди (queued + processing)
- Возраст старейшей задачи
- Worker: занято / ёмкость
- RSS по сервисам
- CPU по сервисам (ядра)
- Collector: очередь экспорта и отказы
- Структурные логи (API и worker), ошибки и предупреждения
- Алерты (firing / pending)
- Последние трассы API (Tempo)
$ анонимный доступ выключен:
GET /api/search ��� ������ -> HTTP 401
$ порты наружу (docker compose ps):
grafana 127.0.0.1:3000->3000/tcp
lb 127.0.0.1:8080->8080/tcp
File diff suppressed because it is too large Load Diff
Binary file not shown.

After

Width:  |  Height:  |  Size: 101 KiB

File diff suppressed because it is too large Load Diff
Binary file not shown.

After

Width:  |  Height:  |  Size: 67 KiB

@@ -0,0 +1,5 @@
services:
lb:
ports: !override ["127.0.0.1:18080:8080"]
grafana:
ports: !override ["127.0.0.1:13000:3000"]
+345
View File
@@ -0,0 +1,345 @@
$ git rev-parse HEAD
6d87f5426c555ea996d97f1415f50cd3bf7b1f17
exit=0
$ git status --short
exit=0
$ C:/Program Files/Git/bin/bash.exe --noprofile --norc -c sh scripts/init-secrets.sh
exit=0
$ docker compose -p mtusi-lab6-clean-check -f compose.yaml -f C:\Users\kolya5544\source\repos\vmdMTUCI\tools\lab6-qa\clean-ports.yaml config --quiet
exit=0
$ docker compose -p mtusi-lab6-clean-check -f compose.yaml -f C:\Users\kolya5544\source\repos\vmdMTUCI\tools\lab6-qa\clean-ports.yaml up -d --build --wait --wait-timeout 180
Image mtusi-messenger:lab6 Building
Image mtusi-messenger:lab6 Building
Image mtusi-messenger:lab6 Building
Image mtusi-messenger:lab6 Building
#1 [internal] load local bake definitions
#1 reading from stdin 2.19kB 0.0s done
#1 DONE 0.0s
#2 [worker internal] load build definition from Dockerfile
#2 transferring dockerfile: 1.91kB done
#2 DONE 0.1s
#2 [s3-init internal] load build definition from Dockerfile
#2 DONE 0.1s
#3 [s3-init] resolve image config for docker-image://docker.io/docker/dockerfile:1
#3 DONE 3.9s
#4 [migrate] docker-image://docker.io/docker/dockerfile:1@sha256:4edf897a3ffa55b89f906fc8cc78afdb3f1834cc9c7083565e611a8a7d5fe99e
#4 resolve docker.io/docker/dockerfile:1@sha256:4edf897a3ffa55b89f906fc8cc78afdb3f1834cc9c7083565e611a8a7d5fe99e 0.1s done
#4 CACHED
#5 [s3-init internal] load metadata for docker.io/library/python:3.12.12-slim-bookworm@sha256:593bd06efe90efa80dc4eee3948be7c0fde4134606dd40d8dd8dbcade98e669c
#5 DONE 1.9s
#6 [s3-init internal] load .dockerignore
#6 transferring context: 289B done
#6 DONE 0.1s
#7 [api builder 1/4] FROM docker.io/library/python:3.12.12-slim-bookworm@sha256:593bd06efe90efa80dc4eee3948be7c0fde4134606dd40d8dd8dbcade98e669c
#7 resolve docker.io/library/python:3.12.12-slim-bookworm@sha256:593bd06efe90efa80dc4eee3948be7c0fde4134606dd40d8dd8dbcade98e669c 0.1s done
#7 DONE 0.1s
#8 [api internal] load build context
#8 transferring context: 159.42kB 0.0s done
#8 DONE 0.2s
#9 [migrate builder 4/4] RUN python -m venv /opt/venv && /opt/venv/bin/pip install -r requirements.txt
#9 CACHED
#10 [migrate stage-1 3/6] COPY --from=builder /opt/venv /opt/venv
#10 CACHED
#11 [migrate stage-1 2/6] RUN groupadd --system --gid 10001 app && useradd --system --uid 10001 --gid app --no-create-home app
#11 CACHED
#12 [migrate builder 2/4] WORKDIR /build
#12 CACHED
#13 [migrate builder 3/4] COPY requirements.txt .
#13 CACHED
#14 [s3-init stage-1 4/6] WORKDIR /srv
#14 CACHED
#15 [api stage-1 5/6] COPY src/ /srv/src/
#15 DONE 0.1s
#15 [worker stage-1 5/6] COPY src/ /srv/src/
#15 DONE 0.1s
#16 [api stage-1 6/6] COPY migrations/ /srv/migrations/
#16 DONE 0.3s
#16 [worker stage-1 6/6] COPY migrations/ /srv/migrations/
#16 DONE 0.3s
#17 [api] exporting to image
#17 exporting layers
#17 exporting layers 0.4s done
#17 exporting manifest sha256:a44a3db4f954b2d839b7fe29cbb71c7f7674f8ca74a7fa09e656d6ca54c5764c
#17 exporting manifest sha256:a44a3db4f954b2d839b7fe29cbb71c7f7674f8ca74a7fa09e656d6ca54c5764c 0.3s done
#17 exporting config sha256:7d05c42b9fa6c5dde578b5fe27fcd0b08b41833510fe211417d886af50eff697
#17 exporting config sha256:7d05c42b9fa6c5dde578b5fe27fcd0b08b41833510fe211417d886af50eff697 0.2s done
#17 exporting attestation manifest sha256:b20893808f6cfa06f171dbcbc4bea0298f024d1011fec1f869b4c2ec35796cf6
#17 exporting attestation manifest sha256:b20893808f6cfa06f171dbcbc4bea0298f024d1011fec1f869b4c2ec35796cf6 0.5s done
#17 exporting manifest list sha256:bfc4140d1931f7f6fc6ab531b42b5cafbacdc8bd5f242c7ba7048342ae0d792f
#17 exporting manifest list sha256:bfc4140d1931f7f6fc6ab531b42b5cafbacdc8bd5f242c7ba7048342ae0d792f 0.2s done
#17 naming to docker.io/library/mtusi-messenger:lab6 0.0s done
#17 unpacking to docker.io/library/mtusi-messenger:lab6
#17 unpacking to docker.io/library/mtusi-messenger:lab6 0.6s done
#17 DONE 2.5s
#18 [migrate] exporting to image
#18 exporting layers 0.4s done
#18 exporting manifest sha256:5308c23deb5b344ce251b0b2bca861d6bd2bc58fc324fb5da4edd2fa3987922c 0.2s done
#18 exporting config sha256:44c24ac50125b68dca2d8cf40743ae2bc78fd95835a54519d19de53dbc8458da 0.2s done
#18 exporting attestation manifest sha256:6ca67fd654b1854f407e84829e0363921d0ee4749e7072f7524c8547426e4ec4 0.5s done
#18 exporting manifest list sha256:f9ccfe31b60f1164f9f0c5a6502e502b785d096ebe59aaa7502191d8eaae6407 0.2s done
#18 naming to docker.io/library/mtusi-messenger:lab6 0.1s done
#18 unpacking to docker.io/library/mtusi-messenger:lab6 0.4s done
#18 DONE 2.4s
#19 [s3-init] exporting to image
#19 exporting layers 0.5s done
#19 exporting manifest sha256:a1a52ab91fb42571d42357271ff4f44839686f1e9a9e5cab34e3fd45ddaabe77 0.1s done
#19 exporting config sha256:469a339ae5debdb4c78f89c63756e83b237176eda4b84c070a4c3b4bc7206f76 0.2s done
#19 exporting attestation manifest sha256:9849897140fe8d411579f10ec3b967c386258d69a347d282774217cb3ea5d4e4 0.5s done
#19 exporting manifest list sha256:e0ecdb5c24bc281b6861cbb6a7861794e8b9692f22ac3383b9725a2a68c385de 0.2s done
#19 naming to docker.io/library/mtusi-messenger:lab6 0.1s done
#19 unpacking to docker.io/library/mtusi-messenger:lab6 0.5s done
#19 DONE 2.4s
#20 [worker] exporting to image
#20 exporting layers 0.5s done
#20 exporting manifest sha256:9dccceba30cebf826f796751a00139cadc366f85fa6575ca7e5dbeaaa07ac69f 0.2s done
#20 exporting config sha256:fe76586a0b246816ab5ea92a6e2f7040c19c8132a975a8a4d76a956ac08b6b85 0.2s done
#20 exporting attestation manifest sha256:5c98251a245b9dda793e1c6841b23918675605ae0962a14e0b6699f66bcb7bec 0.5s done
#20 exporting manifest list sha256:7f326220eb027a78f09705d6f233b029bd58f56e6da48d22c8acaad5102d781c 0.3s done
#20 naming to docker.io/library/mtusi-messenger:lab6 0.1s done
#20 unpacking to docker.io/library/mtusi-messenger:lab6 0.6s done
#20 DONE 2.5s
#21 [migrate] resolving provenance for metadata file
#21 DONE 0.4s
#22 [worker] resolving provenance for metadata file
#22 DONE 0.2s
#23 [s3-init] resolving provenance for metadata file
#23 DONE 0.4s
#24 [api] resolving provenance for metadata file
#24 DONE 0.0s
Image mtusi-messenger:lab6 Built
Image mtusi-messenger:lab6 Built
Image mtusi-messenger:lab6 Built
Image mtusi-messenger:lab6 Built
Network mtusi-lab6-clean-check_backend Creating
Network mtusi-lab6-clean-check_backend Created
Volume mtusi-lab6-clean-check_rabbitmqdata Creating
Volume mtusi-lab6-clean-check_rabbitmqdata Created
Volume mtusi-lab6-clean-check_valkeydata Creating
Volume mtusi-lab6-clean-check_valkeydata Created
Volume mtusi-lab6-clean-check_pgdata Creating
Volume mtusi-lab6-clean-check_pgdata Created
Volume mtusi-lab6-clean-check_lokidata Creating
Volume mtusi-lab6-clean-check_lokidata Created
Volume mtusi-lab6-clean-check_seaweeddata Creating
Volume mtusi-lab6-clean-check_seaweeddata Created
Volume mtusi-lab6-clean-check_promdata Creating
Volume mtusi-lab6-clean-check_promdata Created
Volume mtusi-lab6-clean-check_tempodata Creating
Volume mtusi-lab6-clean-check_tempodata Created
Volume mtusi-lab6-clean-check_grafanadata Creating
Volume mtusi-lab6-clean-check_grafanadata Created
Container mtusi-lab6-clean-check-valkey-1 Creating
Container mtusi-lab6-clean-check-rabbitmq-1 Creating
Container mtusi-lab6-clean-check-otel-collector-1 Creating
Container mtusi-lab6-clean-check-postgres-1 Creating
Container mtusi-lab6-clean-check-seaweedfs-1 Creating
Container mtusi-lab6-clean-check-tempo-1 Creating
Container mtusi-lab6-clean-check-prometheus-1 Creating
Container mtusi-lab6-clean-check-loki-1 Creating
time="2026-11-12T04:51:24+03:00" level=warning msg="secrets `uid`, `gid` and `mode` are not supported, they will be ignored"
Container mtusi-lab6-clean-check-valkey-1 Created
Container mtusi-lab6-clean-check-prometheus-1 Created
Container mtusi-lab6-clean-check-tempo-1 Created
Container mtusi-lab6-clean-check-rabbitmq-1 Created
Container mtusi-lab6-clean-check-loki-1 Created
Container mtusi-lab6-clean-check-grafana-1 Creating
Container mtusi-lab6-clean-check-otel-collector-1 Created
Container mtusi-lab6-clean-check-postgres-1 Created
Container mtusi-lab6-clean-check-migrate-1 Creating
Container mtusi-lab6-clean-check-seaweedfs-1 Created
Container mtusi-lab6-clean-check-s3-init-1 Creating
Container mtusi-lab6-clean-check-grafana-1 Created
Container mtusi-lab6-clean-check-migrate-1 Created
Container mtusi-lab6-clean-check-s3-init-1 Created
Container mtusi-lab6-clean-check-worker-1 Creating
Container mtusi-lab6-clean-check-api-2 Creating
Container mtusi-lab6-clean-check-api-1 Creating
Container mtusi-lab6-clean-check-api-2 Created
Container mtusi-lab6-clean-check-worker-1 Created
Container mtusi-lab6-clean-check-api-1 Created
Container mtusi-lab6-clean-check-lb-1 Creating
Container mtusi-lab6-clean-check-lb-1 Created
Container mtusi-lab6-clean-check-seaweedfs-1 Starting
Container mtusi-lab6-clean-check-loki-1 Starting
Container mtusi-lab6-clean-check-tempo-1 Starting
Container mtusi-lab6-clean-check-valkey-1 Starting
Container mtusi-lab6-clean-check-rabbitmq-1 Starting
Container mtusi-lab6-clean-check-postgres-1 Starting
Container mtusi-lab6-clean-check-prometheus-1 Starting
Container mtusi-lab6-clean-check-otel-collector-1 Starting
Container mtusi-lab6-clean-check-loki-1 Started
Container mtusi-lab6-clean-check-seaweedfs-1 Started
Container mtusi-lab6-clean-check-seaweedfs-1 Waiting
Container mtusi-lab6-clean-check-rabbitmq-1 Started
Container mtusi-lab6-clean-check-valkey-1 Started
Container mtusi-lab6-clean-check-tempo-1 Started
Container mtusi-lab6-clean-check-postgres-1 Started
Container mtusi-lab6-clean-check-postgres-1 Waiting
Container mtusi-lab6-clean-check-otel-collector-1 Started
Container mtusi-lab6-clean-check-prometheus-1 Started
Container mtusi-lab6-clean-check-grafana-1 Starting
Container mtusi-lab6-clean-check-grafana-1 Started
Container mtusi-lab6-clean-check-seaweedfs-1 Healthy
Container mtusi-lab6-clean-check-s3-init-1 Starting
Container mtusi-lab6-clean-check-s3-init-1 Started
Container mtusi-lab6-clean-check-postgres-1 Healthy
Container mtusi-lab6-clean-check-migrate-1 Starting
Container mtusi-lab6-clean-check-migrate-1 Started
Container mtusi-lab6-clean-check-s3-init-1 Waiting
Container mtusi-lab6-clean-check-valkey-1 Waiting
Container mtusi-lab6-clean-check-rabbitmq-1 Waiting
Container mtusi-lab6-clean-check-migrate-1 Waiting
Container mtusi-lab6-clean-check-migrate-1 Waiting
Container mtusi-lab6-clean-check-s3-init-1 Waiting
Container mtusi-lab6-clean-check-s3-init-1 Exited
Container mtusi-lab6-clean-check-s3-init-1 Exited
Container mtusi-lab6-clean-check-rabbitmq-1 Healthy
Container mtusi-lab6-clean-check-valkey-1 Healthy
Container mtusi-lab6-clean-check-migrate-1 Exited
Container mtusi-lab6-clean-check-worker-1 Starting
Container mtusi-lab6-clean-check-migrate-1 Exited
Container mtusi-lab6-clean-check-api-1 Starting
Container mtusi-lab6-clean-check-api-1 Started
Container mtusi-lab6-clean-check-api-2 Starting
Container mtusi-lab6-clean-check-worker-1 Started
Container mtusi-lab6-clean-check-api-2 Started
Container mtusi-lab6-clean-check-lb-1 Starting
Container mtusi-lab6-clean-check-lb-1 Started
Container mtusi-lab6-clean-check-grafana-1 Waiting
Container mtusi-lab6-clean-check-valkey-1 Waiting
Container mtusi-lab6-clean-check-api-1 Waiting
Container mtusi-lab6-clean-check-api-2 Waiting
Container mtusi-lab6-clean-check-postgres-1 Waiting
Container mtusi-lab6-clean-check-loki-1 Waiting
Container mtusi-lab6-clean-check-migrate-1 Waiting
Container mtusi-lab6-clean-check-otel-collector-1 Waiting
Container mtusi-lab6-clean-check-tempo-1 Waiting
Container mtusi-lab6-clean-check-worker-1 Waiting
Container mtusi-lab6-clean-check-s3-init-1 Waiting
Container mtusi-lab6-clean-check-rabbitmq-1 Waiting
Container mtusi-lab6-clean-check-seaweedfs-1 Waiting
Container mtusi-lab6-clean-check-prometheus-1 Waiting
Container mtusi-lab6-clean-check-lb-1 Waiting
Container mtusi-lab6-clean-check-otel-collector-1 Healthy
Container mtusi-lab6-clean-check-rabbitmq-1 Healthy
Container mtusi-lab6-clean-check-seaweedfs-1 Healthy
Container mtusi-lab6-clean-check-postgres-1 Healthy
Container mtusi-lab6-clean-check-tempo-1 Healthy
Container mtusi-lab6-clean-check-prometheus-1 Healthy
Container mtusi-lab6-clean-check-loki-1 Healthy
Container mtusi-lab6-clean-check-valkey-1 Healthy
Container mtusi-lab6-clean-check-migrate-1 Exited
Container mtusi-lab6-clean-check-s3-init-1 Exited
Container mtusi-lab6-clean-check-worker-1 Healthy
Container mtusi-lab6-clean-check-api-1 Healthy
Container mtusi-lab6-clean-check-api-2 Healthy
Container mtusi-lab6-clean-check-lb-1 Healthy
Container mtusi-lab6-clean-check-grafana-1 Healthy
exit=0
$ python scripts/check_contract.py
PASS: offline skeleton structure, local refs and Python syntax; application NOT tested.
exit=0
$ python tests/smoke.py --base-url http://127.0.0.1:18080
PASS: lab6 public API scenario; infrastructure and grading evidence remain separate.
exit=0
PASS: fresh secrets/volumes, public API, 3 datasources, provisioned dashboard with 22 panels
$ docker compose -p mtusi-lab6-clean-check -f compose.yaml -f C:\Users\kolya5544\source\repos\vmdMTUCI\tools\lab6-qa\clean-ports.yaml down
Container mtusi-lab6-clean-check-otel-collector-1 Stopping
Container mtusi-lab6-clean-check-lb-1 Stopping
Container mtusi-lab6-clean-check-grafana-1 Stopping
Container mtusi-lab6-clean-check-worker-1 Stopping
Container mtusi-lab6-clean-check-otel-collector-1 Stopped
Container mtusi-lab6-clean-check-otel-collector-1 Removing
Container mtusi-lab6-clean-check-otel-collector-1 Removed
Container mtusi-lab6-clean-check-lb-1 Stopped
Container mtusi-lab6-clean-check-lb-1 Removing
Container mtusi-lab6-clean-check-lb-1 Removed
Container mtusi-lab6-clean-check-api-2 Stopping
Container mtusi-lab6-clean-check-api-1 Stopping
Container mtusi-lab6-clean-check-grafana-1 Stopped
Container mtusi-lab6-clean-check-grafana-1 Removing
Container mtusi-lab6-clean-check-grafana-1 Removed
Container mtusi-lab6-clean-check-loki-1 Stopping
Container mtusi-lab6-clean-check-tempo-1 Stopping
Container mtusi-lab6-clean-check-prometheus-1 Stopping
Container mtusi-lab6-clean-check-prometheus-1 Stopped
Container mtusi-lab6-clean-check-prometheus-1 Removing
Container mtusi-lab6-clean-check-prometheus-1 Removed
Container mtusi-lab6-clean-check-tempo-1 Stopped
Container mtusi-lab6-clean-check-tempo-1 Removing
Container mtusi-lab6-clean-check-tempo-1 Removed
Container mtusi-lab6-clean-check-loki-1 Stopped
Container mtusi-lab6-clean-check-loki-1 Removing
Container mtusi-lab6-clean-check-loki-1 Removed
Container mtusi-lab6-clean-check-worker-1 Stopped
Container mtusi-lab6-clean-check-worker-1 Removing
Container mtusi-lab6-clean-check-worker-1 Removed
Container mtusi-lab6-clean-check-rabbitmq-1 Stopping
Container mtusi-lab6-clean-check-api-2 Stopped
Container mtusi-lab6-clean-check-api-2 Removing
Container mtusi-lab6-clean-check-api-2 Removed
Container mtusi-lab6-clean-check-rabbitmq-1 Stopped
Container mtusi-lab6-clean-check-rabbitmq-1 Removing
Container mtusi-lab6-clean-check-rabbitmq-1 Removed
Container mtusi-lab6-clean-check-api-1 Stopped
Container mtusi-lab6-clean-check-api-1 Removing
Container mtusi-lab6-clean-check-api-1 Removed
Container mtusi-lab6-clean-check-migrate-1 Stopping
Container mtusi-lab6-clean-check-valkey-1 Stopping
Container mtusi-lab6-clean-check-s3-init-1 Stopping
Container mtusi-lab6-clean-check-migrate-1 Stopped
Container mtusi-lab6-clean-check-migrate-1 Removing
Container mtusi-lab6-clean-check-s3-init-1 Stopped
Container mtusi-lab6-clean-check-s3-init-1 Removing
Container mtusi-lab6-clean-check-s3-init-1 Removed
Container mtusi-lab6-clean-check-seaweedfs-1 Stopping
Container mtusi-lab6-clean-check-migrate-1 Removed
Container mtusi-lab6-clean-check-postgres-1 Stopping
Container mtusi-lab6-clean-check-valkey-1 Stopped
Container mtusi-lab6-clean-check-valkey-1 Removing
Container mtusi-lab6-clean-check-postgres-1 Stopped
Container mtusi-lab6-clean-check-postgres-1 Removing
Container mtusi-lab6-clean-check-valkey-1 Removed
Container mtusi-lab6-clean-check-postgres-1 Removed
Container mtusi-lab6-clean-check-seaweedfs-1 Stopped
Container mtusi-lab6-clean-check-seaweedfs-1 Removing
Container mtusi-lab6-clean-check-seaweedfs-1 Removed
Network mtusi-lab6-clean-check_backend Removing
Network mtusi-lab6-clean-check_backend Removed
exit=0
+136
View File
@@ -0,0 +1,136 @@
$ python -u research/incidents.py --out evidence\recheck_2026-11-12
{
"incidents": [
{
"incident": "worker_down",
"alert": "QueueLagGrowing",
"injected_at_s": 25.0,
"firing_at_s": 93.3,
"detection_seconds": 68.3,
"recovered_cmd_at_s": 96.5,
"resolved_at_s": 118.3,
"fired": true,
"resolved": true,
"evidence": {
"queue_oldest_age_s": 60.743133,
"worker_capacity_series_present": true,
"jobs_ready_rate_1m": 0.0,
"diagnosis": "возраст старейшей задачи растёт при нулевой скорости обработки jobs (worker не отдаёт метрики) -> остановлен worker, а не медленная БД",
"alerts_in_state_firing": [
"HighHttpErrorRate",
"HighUploadErrorRate",
"QueueLagGrowing"
]
},
"time_to_resolve_after_recovery_s": 21.8
},
{
"incident": "slow_db",
"alert": "HighLatencyP95",
"injected_at_s": 124.3,
"firing_at_s": 192.0,
"detection_seconds": 67.7,
"recovered_cmd_at_s": 193.7,
"resolved_at_s": 237.2,
"fired": true,
"resolved": true,
"evidence": {
"p95_s": 0.7828124999999998,
"db_pool_waiting": 0.0,
"slow_traces_with_long_db_span": 20,
"slow_http_post_messages_traces": 20,
"diagnosis": "p95 высокий, а время в спанах db.connection сопоставимо с длительностью запроса и растёт ожидание пула -> медленная БД, а не обработчик",
"alerts_in_state_firing": [
"HighLatencyP95"
]
},
"time_to_resolve_after_recovery_s": 43.5
},
{
"incident": "s3_down",
"alert": "HighUploadErrorRate",
"injected_at_s": 243.2,
"firing_at_s": 327.3,
"detection_seconds": 84.1,
"recovered_cmd_at_s": 328.6,
"resolved_at_s": 367.9,
"fired": true,
"resolved": true,
"evidence": {
"upload_error_ratio_1m": 1.0,
"global_error_ratio_1m": 0.011166945840312675,
"error_traces": 20,
"s3_put_failed_log_lines": 37,
"diagnosis": "рост доли 5xx на маршруте загрузки при нормальных остальных; спаны s3.put_object с ошибкой и логи s3_put_failed -> недоступен S3",
"alerts_in_state_firing": [
"HighUploadErrorRate",
"QueueLagGrowing"
]
},
"time_to_resolve_after_recovery_s": 39.3
},
{
"incident": "telemetry_backend",
"alert": "TelemetryExportFailing",
"injected_at_s": 373.9,
"firing_at_s": 442.9,
"detection_seconds": 69.1,
"recovered_cmd_at_s": 444.7,
"resolved_at_s": 512.2,
"fired": true,
"resolved": true,
"evidence": {
"api_error_ratio_1m": 0.0,
"api_rps": 62.62045892577912,
"collector_queue_size": 32.0,
"collector_send_failed_spans_rate": 62.94195947803441,
"collector_rss_mib": 239.0,
"diagnosis": "ошибки экспорта растут, очередь ограничена; бизнес-метрики API без роста ошибок -> сбой хранилища телеметрии, не приложения",
"alerts_in_state_firing": [
"TelemetryExportFailing"
]
},
"time_to_resolve_after_recovery_s": 67.5
}
],
"traffic": {
"msg_2": 15471,
"up_2": 10598,
"up_5": 2175,
"up_err": 45
}
}
exit=0
$ python -u research/obs_cost.py --out evidence\recheck_2026-11-12 --profiles off on100 head10 tail --repeat 2
{"profile": "off", "post": {"requests": 41303, "ok": 40440, "errors": {"500": 863}, "seconds": 30, "ok_rps": 1348.0, "error_ratio": 0.0209, "p50_ms": 17.39, "p95_ms": 74.05, "p99_ms": 618.63}, "docker_stats": {"worker": {"avg_cpu_percent": 1.6, "peak_mem_mib": 54.6}, "api": {"avg_cpu_percent": 50.6, "peak_mem_mib": 76.4}, "otel-collector": {"avg_cpu_percent": 0.1, "peak_mem_mib": 45.1}, "tempo": {"avg_cpu_percent": 0.3, "peak_mem_mib": 426.2}, "lb": {"avg_cpu_percent": 23.0, "peak_mem_mib": 18.3}, "grafana": {"avg_cpu_percent": 1.1, "peak_mem_mib": 314.4}, "rabbitmq": {"avg_cpu_percent": 14.5, "peak_mem_mib": 177.9}, "postgres": {"avg_cpu_percent": 95.9, "peak_mem_mib": 264.3}, "loki": {"avg_cpu_percent": 0.5, "peak_mem_mib": 119.8}, "seaweedfs": {"avg_cpu_percent": 1.3, "peak_mem_mib": 524.7}, "prometheus": {"avg_cpu_percent": 0.7, "peak_mem_mib": 75.6}, "valkey": {"avg_cpu_percent": 4.7, "peak_mem_mib": 10.7}}, "collector_delta": {"receiver_accepted_spans": 0, "receiver_accepted_log_records": 0, "receiver_accepted_metric_points": 0, "receiver_refused_spans": 0, "exporter_sent_spans": 0, "exporter_send_failed_spans": 0}, "tempo_bytes_delta": 0, "loki_bytes_delta": 0, "errors_injected_500": 873, "slow_requests_gt500ms": 823, "traces_in_tempo": {"all_http": 0, "error": 0, "slow_gt500ms": 0}, "kept_ratio": {"errors": 0.0, "slow": 0.0}}
{"profile": "off", "post": {"requests": 42499, "ok": 41641, "errors": {"500": 858}, "seconds": 30, "ok_rps": 1388.0, "error_ratio": 0.0202, "p50_ms": 16.03, "p95_ms": 72.03, "p99_ms": 617.51}, "docker_stats": {"worker": {"avg_cpu_percent": 1.4, "peak_mem_mib": 54.6}, "api": {"avg_cpu_percent": 49.7, "peak_mem_mib": 64.0}, "otel-collector": {"avg_cpu_percent": 0.1, "peak_mem_mib": 43.7}, "tempo": {"avg_cpu_percent": 0.3, "peak_mem_mib": 388.5}, "lb": {"avg_cpu_percent": 22.6, "peak_mem_mib": 21.8}, "grafana": {"avg_cpu_percent": 1.4, "peak_mem_mib": 315.2}, "rabbitmq": {"avg_cpu_percent": 8.3, "peak_mem_mib": 178.2}, "postgres": {"avg_cpu_percent": 94.0, "peak_mem_mib": 263.0}, "loki": {"avg_cpu_percent": 0.5, "peak_mem_mib": 121.8}, "seaweedfs": {"avg_cpu_percent": 1.4, "peak_mem_mib": 505.8}, "prometheus": {"avg_cpu_percent": 0.6, "peak_mem_mib": 60.9}, "valkey": {"avg_cpu_percent": 4.5, "peak_mem_mib": 10.7}}, "collector_delta": {"receiver_accepted_spans": 0, "receiver_accepted_log_records": 0, "receiver_accepted_metric_points": 0, "receiver_refused_spans": 0, "exporter_sent_spans": 0, "exporter_send_failed_spans": 0}, "tempo_bytes_delta": 0, "loki_bytes_delta": 0, "errors_injected_500": 873, "slow_requests_gt500ms": 888, "traces_in_tempo": {"all_http": 0, "error": 0, "slow_gt500ms": 0}, "kept_ratio": {"errors": 0.0, "slow": 0.0}}
{"profile": "on100", "post": {"requests": 38267, "ok": 37530, "errors": {"500": 737}, "seconds": 30, "ok_rps": 1251.0, "error_ratio": 0.0193, "p50_ms": 24.79, "p95_ms": 97.91, "p99_ms": 631.54}, "docker_stats": {"worker": {"avg_cpu_percent": 2.2, "peak_mem_mib": 64.5}, "api": {"avg_cpu_percent": 80.6, "peak_mem_mib": 83.2}, "otel-collector": {"avg_cpu_percent": 3.4, "peak_mem_mib": 75.0}, "tempo": {"avg_cpu_percent": 11.2, "peak_mem_mib": 745.7}, "lb": {"avg_cpu_percent": 20.2, "peak_mem_mib": 21.5}, "grafana": {"avg_cpu_percent": 1.1, "peak_mem_mib": 320.0}, "rabbitmq": {"avg_cpu_percent": 12.1, "peak_mem_mib": 157.0}, "postgres": {"avg_cpu_percent": 92.6, "peak_mem_mib": 282.2}, "loki": {"avg_cpu_percent": 4.4, "peak_mem_mib": 194.8}, "seaweedfs": {"avg_cpu_percent": 1.4, "peak_mem_mib": 494.7}, "prometheus": {"avg_cpu_percent": 0.5, "peak_mem_mib": 60.8}, "valkey": {"avg_cpu_percent": 5.4, "peak_mem_mib": 11.5}}, "collector_delta": {"receiver_accepted_spans": 90462, "receiver_accepted_log_records": 44825, "receiver_accepted_metric_points": 396, "receiver_refused_spans": 0, "exporter_sent_spans": 90462, "exporter_send_failed_spans": 0}, "tempo_bytes_delta": 47091712, "loki_bytes_delta": 30777344, "errors_injected_500": 761, "slow_requests_gt500ms": 808, "traces_in_tempo": {"all_http": 5000, "error": 848, "slow_gt500ms": 902}, "kept_ratio": {"errors": 1.114, "slow": 1.116}}
{"profile": "on100", "post": {"requests": 37970, "ok": 37178, "errors": {"500": 792}, "seconds": 30, "ok_rps": 1239.3, "error_ratio": 0.0209, "p50_ms": 24.47, "p95_ms": 102.73, "p99_ms": 627.88}, "docker_stats": {"worker": {"avg_cpu_percent": 1.7, "peak_mem_mib": 60.3}, "api": {"avg_cpu_percent": 78.4, "peak_mem_mib": 75.4}, "otel-collector": {"avg_cpu_percent": 3.3, "peak_mem_mib": 72.0}, "tempo": {"avg_cpu_percent": 7.9, "peak_mem_mib": 526.0}, "lb": {"avg_cpu_percent": 19.5, "peak_mem_mib": 21.7}, "grafana": {"avg_cpu_percent": 0.8, "peak_mem_mib": 321.6}, "rabbitmq": {"avg_cpu_percent": 12.0, "peak_mem_mib": 188.4}, "postgres": {"avg_cpu_percent": 89.5, "peak_mem_mib": 284.4}, "loki": {"avg_cpu_percent": 4.8, "peak_mem_mib": 216.5}, "seaweedfs": {"avg_cpu_percent": 1.3, "peak_mem_mib": 494.4}, "prometheus": {"avg_cpu_percent": 0.7, "peak_mem_mib": 61.0}, "valkey": {"avg_cpu_percent": 5.0, "peak_mem_mib": 11.5}}, "collector_delta": {"receiver_accepted_spans": 89310, "receiver_accepted_log_records": 43824, "receiver_accepted_metric_points": 396, "receiver_refused_spans": 0, "exporter_sent_spans": 89310, "exporter_send_failed_spans": 0}, "tempo_bytes_delta": 22323200, "loki_bytes_delta": 30490624, "errors_injected_500": 800, "slow_requests_gt500ms": 737, "traces_in_tempo": {"all_http": 5000, "error": 881, "slow_gt500ms": 850}, "kept_ratio": {"errors": 1.101, "slow": 1.153}}
{"profile": "head10", "post": {"requests": 34821, "ok": 34102, "errors": {"500": 719}, "seconds": 30, "ok_rps": 1136.7, "error_ratio": 0.0206, "p50_ms": 26.35, "p95_ms": 123.14, "p99_ms": 630.88}, "docker_stats": {"worker": {"avg_cpu_percent": 2.0, "peak_mem_mib": 62.5}, "api": {"avg_cpu_percent": 67.6, "peak_mem_mib": 83.7}, "otel-collector": {"avg_cpu_percent": 1.9, "peak_mem_mib": 68.4}, "tempo": {"avg_cpu_percent": 0.6, "peak_mem_mib": 472.9}, "lb": {"avg_cpu_percent": 19.2, "peak_mem_mib": 23.2}, "grafana": {"avg_cpu_percent": 0.9, "peak_mem_mib": 322.9}, "rabbitmq": {"avg_cpu_percent": 10.5, "peak_mem_mib": 194.3}, "postgres": {"avg_cpu_percent": 89.7, "peak_mem_mib": 302.1}, "loki": {"avg_cpu_percent": 4.8, "peak_mem_mib": 239.9}, "seaweedfs": {"avg_cpu_percent": 1.5, "peak_mem_mib": 494.4}, "prometheus": {"avg_cpu_percent": 0.6, "peak_mem_mib": 60.9}, "valkey": {"avg_cpu_percent": 5.1, "peak_mem_mib": 12.8}}, "collector_delta": {"receiver_accepted_spans": 8506, "receiver_accepted_log_records": 40586, "receiver_accepted_metric_points": 396, "receiver_refused_spans": 0, "exporter_sent_spans": 8506, "exporter_send_failed_spans": 0}, "tempo_bytes_delta": 1200128, "loki_bytes_delta": 4431872, "errors_injected_500": 727, "slow_requests_gt500ms": 671, "traces_in_tempo": {"all_http": 3919, "error": 89, "slow_gt500ms": 80}, "kept_ratio": {"errors": 0.122, "slow": 0.119}}
{"profile": "head10", "post": {"requests": 39840, "ok": 39078, "errors": {"500": 762}, "seconds": 30, "ok_rps": 1302.6, "error_ratio": 0.0191, "p50_ms": 21.48, "p95_ms": 79.8, "p99_ms": 622.79}, "docker_stats": {"worker": {"avg_cpu_percent": 1.3, "peak_mem_mib": 60.2}, "api": {"avg_cpu_percent": 69.4, "peak_mem_mib": 74.0}, "otel-collector": {"avg_cpu_percent": 1.9, "peak_mem_mib": 68.9}, "tempo": {"avg_cpu_percent": 0.6, "peak_mem_mib": 755.1}, "lb": {"avg_cpu_percent": 19.9, "peak_mem_mib": 21.7}, "grafana": {"avg_cpu_percent": 1.5, "peak_mem_mib": 325.1}, "rabbitmq": {"avg_cpu_percent": 10.9, "peak_mem_mib": 194.5}, "postgres": {"avg_cpu_percent": 91.7, "peak_mem_mib": 303.6}, "loki": {"avg_cpu_percent": 4.6, "peak_mem_mib": 281.0}, "seaweedfs": {"avg_cpu_percent": 1.2, "peak_mem_mib": 494.4}, "prometheus": {"avg_cpu_percent": 0.7, "peak_mem_mib": 61.0}, "valkey": {"avg_cpu_percent": 5.0, "peak_mem_mib": 11.4}}, "collector_delta": {"receiver_accepted_spans": 9291, "receiver_accepted_log_records": 45766, "receiver_accepted_metric_points": 396, "receiver_refused_spans": 0, "exporter_sent_spans": 9458, "exporter_send_failed_spans": 0}, "tempo_bytes_delta": 32768, "loki_bytes_delta": 32833536, "errors_injected_500": 774, "slow_requests_gt500ms": 802, "traces_in_tempo": {"all_http": 4411, "error": 81, "slow_gt500ms": 93}, "kept_ratio": {"errors": 0.105, "slow": 0.116}}
{"profile": "tail", "post": {"requests": 39275, "ok": 38469, "errors": {"500": 806}, "seconds": 30, "ok_rps": 1282.3, "error_ratio": 0.0205, "p50_ms": 23.74, "p95_ms": 86.59, "p99_ms": 631.13}, "docker_stats": {"worker": {"avg_cpu_percent": 2.1, "peak_mem_mib": 59.8}, "api": {"avg_cpu_percent": 77.1, "peak_mem_mib": 75.4}, "otel-collector": {"avg_cpu_percent": 4.6, "peak_mem_mib": 105.8}, "tempo": {"avg_cpu_percent": 0.8, "peak_mem_mib": 719.9}, "lb": {"avg_cpu_percent": 19.7, "peak_mem_mib": 22.0}, "grafana": {"avg_cpu_percent": 1.0, "peak_mem_mib": 325.2}, "rabbitmq": {"avg_cpu_percent": 12.5, "peak_mem_mib": 185.8}, "postgres": {"avg_cpu_percent": 93.0, "peak_mem_mib": 302.7}, "loki": {"avg_cpu_percent": 5.8, "peak_mem_mib": 312.2}, "seaweedfs": {"avg_cpu_percent": 1.4, "peak_mem_mib": 494.4}, "prometheus": {"avg_cpu_percent": 0.6, "peak_mem_mib": 60.9}, "valkey": {"avg_cpu_percent": 5.1, "peak_mem_mib": 11.4}}, "collector_delta": {"receiver_accepted_spans": 91490, "receiver_accepted_log_records": 45024, "receiver_accepted_metric_points": 396, "receiver_refused_spans": 0, "exporter_sent_spans": 13023, "exporter_send_failed_spans": 0}, "tempo_bytes_delta": 401408, "loki_bytes_delta": 34607104, "errors_injected_500": 818, "slow_requests_gt500ms": 793, "traces_in_tempo": {"all_http": 5000, "error": 907, "slow_gt500ms": 883}, "kept_ratio": {"errors": 1.109, "slow": 1.113}}
{"profile": "tail", "post": {"requests": 39821, "ok": 38989, "errors": {"500": 832}, "seconds": 30, "ok_rps": 1299.6, "error_ratio": 0.0209, "p50_ms": 23.35, "p95_ms": 93.86, "p99_ms": 626.16}, "docker_stats": {"worker": {"avg_cpu_percent": 1.3, "peak_mem_mib": 64.2}, "api": {"avg_cpu_percent": 77.7, "peak_mem_mib": 75.5}, "otel-collector": {"avg_cpu_percent": 3.4, "peak_mem_mib": 108.4}, "tempo": {"avg_cpu_percent": 1.9, "peak_mem_mib": 443.0}, "lb": {"avg_cpu_percent": 18.6, "peak_mem_mib": 21.6}, "grafana": {"avg_cpu_percent": 0.7, "peak_mem_mib": 325.1}, "rabbitmq": {"avg_cpu_percent": 9.5, "peak_mem_mib": 175.5}, "postgres": {"avg_cpu_percent": 88.4, "peak_mem_mib": 303.5}, "loki": {"avg_cpu_percent": 4.3, "peak_mem_mib": 317.3}, "seaweedfs": {"avg_cpu_percent": 4.2, "peak_mem_mib": 692.3}, "prometheus": {"avg_cpu_percent": 0.5, "peak_mem_mib": 61.1}, "valkey": {"avg_cpu_percent": 4.9, "peak_mem_mib": 11.7}}, "collector_delta": {"receiver_accepted_spans": 92273, "receiver_accepted_log_records": 45427, "receiver_accepted_metric_points": 396, "receiver_refused_spans": 0, "exporter_sent_spans": 13252, "exporter_send_failed_spans": 0}, "tempo_bytes_delta": 1703936, "loki_bytes_delta": 32804864, "errors_injected_500": 837, "slow_requests_gt500ms": 743, "traces_in_tempo": {"all_http": 5000, "error": 925, "slow_gt500ms": 834}, "kept_ratio": {"errors": 1.105, "slow": 1.122}}
saved
exit=0
$ python research/plot_obs_cost.py --out evidence\recheck_2026-11-12
saved
exit=0
$ python tests/smoke.py --base-url http://127.0.0.1:8080
PASS: lab6 public API scenario; infrastructure and grading evidence remain separate.
exit=0
$ python -m unittest discover -s student-tests -p test_observability.py -v
test_logs_are_structured_and_linked_by_trace_id (test_observability.ObservabilityTest.test_logs_are_structured_and_linked_by_trace_id) ... ok
test_metrics_for_http_jobs_queue_pool_worker (test_observability.ObservabilityTest.test_metrics_for_http_jobs_queue_pool_worker) ... ok
test_no_high_cardinality_or_secret_labels (test_observability.ObservabilityTest.test_no_high_cardinality_or_secret_labels) ... ok
test_one_trace_spans_api_outbox_broker_and_worker (test_observability.ObservabilityTest.test_one_trace_spans_api_outbox_broker_and_worker) ... ok
test_secrets_and_message_bodies_are_not_exported (test_observability.ObservabilityTest.test_secrets_and_message_bodies_are_not_exported) ... ok
----------------------------------------------------------------------
Ran 5 tests in 12.547s
OK
exit=0
+9
View File
@@ -0,0 +1,9 @@
{
"when": "after experiments and final smoke, 2026-11-12",
"rows": {
"users": 739,
"chats": 425,
"messages": 1610549,
"attachments": 68898
}
}
+10
View File
@@ -0,0 +1,10 @@
{
"stopped_in_order": [
"mtusi-lab6-api-1",
"mtusi-lab6-api-2"
],
"distinct_replicas": [
"mtusi-lab6-api-1",
"mtusi-lab6-api-2"
]
}
@@ -0,0 +1,25 @@
{
"datasources": [
{
"name": "Loki",
"type": "loki",
"url": "http://loki:3100"
},
{
"name": "Prometheus",
"type": "prometheus",
"url": "http://prometheus:9090"
},
{
"name": "Tempo",
"type": "tempo",
"url": "http://tempo:3200"
}
],
"dashboard": {
"uid": "messenger-overview",
"title": "Messenger: API, очередь, worker",
"provisioned": true,
"panels": 22
}
}
File diff suppressed because it is too large Load Diff
Binary file not shown.

After

Width:  |  Height:  |  Size: 105 KiB

+274
View File
@@ -0,0 +1,274 @@
{
"before": {
"mtusi-lab6-api-1": {
"service": "api",
"started_at": "2026-11-12T01:25:27.883010338Z",
"running": true,
"health": "healthy",
"volumes": [],
"memory_bytes": 268435456,
"nano_cpus": 1000000000
},
"mtusi-lab6-api-2": {
"service": "api",
"started_at": "2026-11-12T01:25:28.11093455Z",
"running": true,
"health": "healthy",
"volumes": [],
"memory_bytes": 268435456,
"nano_cpus": 1000000000
},
"mtusi-lab6-grafana-1": {
"service": "grafana",
"started_at": "2026-11-12T01:21:50.391698676Z",
"running": true,
"health": "healthy",
"volumes": [
"mtusi-lab6_grafanadata"
],
"memory_bytes": 536870912,
"nano_cpus": 1000000000
},
"mtusi-lab6-lb-1": {
"service": "lb",
"started_at": "2026-11-12T01:21:48.668160902Z",
"running": true,
"health": "healthy",
"volumes": [],
"memory_bytes": 134217728,
"nano_cpus": 1000000000
},
"mtusi-lab6-loki-1": {
"service": "loki",
"started_at": "2026-11-12T01:21:50.190375514Z",
"running": true,
"health": null,
"volumes": [
"mtusi-lab6_lokidata"
],
"memory_bytes": 536870912,
"nano_cpus": 1000000000
},
"mtusi-lab6-otel-collector-1": {
"service": "otel-collector",
"started_at": "2026-11-12T01:21:51.528830632Z",
"running": true,
"health": null,
"volumes": [],
"memory_bytes": 402653184,
"nano_cpus": 1000000000
},
"mtusi-lab6-postgres-1": {
"service": "postgres",
"started_at": "2026-11-12T01:21:50.009875096Z",
"running": true,
"health": "healthy",
"volumes": [
"mtusi-lab6_pgdata"
],
"memory_bytes": 536870912,
"nano_cpus": 1000000000
},
"mtusi-lab6-prometheus-1": {
"service": "prometheus",
"started_at": "2026-11-12T01:21:49.578549023Z",
"running": true,
"health": "healthy",
"volumes": [
"mtusi-lab6_promdata"
],
"memory_bytes": 536870912,
"nano_cpus": 1000000000
},
"mtusi-lab6-rabbitmq-1": {
"service": "rabbitmq",
"started_at": "2026-11-12T01:21:51.306616597Z",
"running": true,
"health": "healthy",
"volumes": [
"mtusi-lab6_rabbitmqdata"
],
"memory_bytes": 536870912,
"nano_cpus": 1000000000
},
"mtusi-lab6-seaweedfs-1": {
"service": "seaweedfs",
"started_at": "2026-11-12T01:21:50.815349111Z",
"running": true,
"health": "healthy",
"volumes": [
"mtusi-lab6_seaweeddata"
],
"memory_bytes": 805306368,
"nano_cpus": 1000000000
},
"mtusi-lab6-tempo-1": {
"service": "tempo",
"started_at": "2026-11-12T01:21:51.015757118Z",
"running": true,
"health": null,
"volumes": [
"mtusi-lab6_tempodata"
],
"memory_bytes": 1073741824,
"nano_cpus": 1000000000
},
"mtusi-lab6-valkey-1": {
"service": "valkey",
"started_at": "2026-11-12T01:21:49.795208905Z",
"running": true,
"health": "healthy",
"volumes": [
"mtusi-lab6_valkeydata"
],
"memory_bytes": 134217728,
"nano_cpus": 500000000
},
"mtusi-lab6-worker-1": {
"service": "worker",
"started_at": "2026-11-12T01:25:27.886972662Z",
"running": true,
"health": "healthy",
"volumes": [],
"memory_bytes": 805306368,
"nano_cpus": 1000000000
}
},
"after": {
"mtusi-lab6-api-1": {
"service": "api",
"started_at": "2026-11-12T01:25:43.517609672Z",
"running": true,
"health": "healthy",
"volumes": [],
"memory_bytes": 268435456,
"nano_cpus": 1000000000
},
"mtusi-lab6-api-2": {
"service": "api",
"started_at": "2026-11-12T01:25:43.743154951Z",
"running": true,
"health": "healthy",
"volumes": [],
"memory_bytes": 268435456,
"nano_cpus": 1000000000
},
"mtusi-lab6-grafana-1": {
"service": "grafana",
"started_at": "2026-11-12T01:25:37.85991836Z",
"running": true,
"health": "healthy",
"volumes": [
"mtusi-lab6_grafanadata"
],
"memory_bytes": 536870912,
"nano_cpus": 1000000000
},
"mtusi-lab6-lb-1": {
"service": "lb",
"started_at": "2026-11-12T01:25:36.297842763Z",
"running": true,
"health": "healthy",
"volumes": [],
"memory_bytes": 134217728,
"nano_cpus": 1000000000
},
"mtusi-lab6-loki-1": {
"service": "loki",
"started_at": "2026-11-12T01:25:37.451077579Z",
"running": true,
"health": null,
"volumes": [
"mtusi-lab6_lokidata"
],
"memory_bytes": 536870912,
"nano_cpus": 1000000000
},
"mtusi-lab6-otel-collector-1": {
"service": "otel-collector",
"started_at": "2026-11-12T01:25:39.004879376Z",
"running": true,
"health": null,
"volumes": [],
"memory_bytes": 402653184,
"nano_cpus": 1000000000
},
"mtusi-lab6-postgres-1": {
"service": "postgres",
"started_at": "2026-11-12T01:25:37.693692485Z",
"running": true,
"health": "healthy",
"volumes": [
"mtusi-lab6_pgdata"
],
"memory_bytes": 536870912,
"nano_cpus": 1000000000
},
"mtusi-lab6-prometheus-1": {
"service": "prometheus",
"started_at": "2026-11-12T01:25:37.087818096Z",
"running": true,
"health": "healthy",
"volumes": [
"mtusi-lab6_promdata"
],
"memory_bytes": 536870912,
"nano_cpus": 1000000000
},
"mtusi-lab6-rabbitmq-1": {
"service": "rabbitmq",
"started_at": "2026-11-12T01:25:38.135027155Z",
"running": true,
"health": "healthy",
"volumes": [
"mtusi-lab6_rabbitmqdata"
],
"memory_bytes": 536870912,
"nano_cpus": 1000000000
},
"mtusi-lab6-seaweedfs-1": {
"service": "seaweedfs",
"started_at": "2026-11-12T01:25:38.293452184Z",
"running": true,
"health": "healthy",
"volumes": [
"mtusi-lab6_seaweeddata"
],
"memory_bytes": 805306368,
"nano_cpus": 1000000000
},
"mtusi-lab6-tempo-1": {
"service": "tempo",
"started_at": "2026-11-12T01:25:37.986295937Z",
"running": true,
"health": null,
"volumes": [
"mtusi-lab6_tempodata"
],
"memory_bytes": 1073741824,
"nano_cpus": 1000000000
},
"mtusi-lab6-valkey-1": {
"service": "valkey",
"started_at": "2026-11-12T01:25:37.254699807Z",
"running": true,
"health": "healthy",
"volumes": [
"mtusi-lab6_valkeydata"
],
"memory_bytes": 134217728,
"nano_cpus": 500000000
},
"mtusi-lab6-worker-1": {
"service": "worker",
"started_at": "2026-11-12T01:25:42.656281441Z",
"running": true,
"health": "healthy",
"volumes": [],
"memory_bytes": 805306368,
"nano_cpus": 1000000000
}
},
"all_services_restarted": true,
"volumes_preserved": true
}
File diff suppressed because it is too large Load Diff
Binary file not shown.

After

Width:  |  Height:  |  Size: 62 KiB

@@ -0,0 +1,64 @@
$ python tests/resilience.py --mode persistence --action-script scripts/restart.cmd --base-url http://127.0.0.1:8080
Executing explicit persistence hook; hook output suppressed to avoid leaking secrets.
PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence.
exit=0
$ python tests/resilience.py --mode failover --action-script scripts/stop-one.cmd --base-url http://127.0.0.1:8080
Executing explicit failover hook; hook output suppressed to avoid leaking secrets.
{"observed_instances_before": 2, "successful_writes": 120, "failed_attempts": 0, "first_recovery_seconds": 2.344}
PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.
exit=0
$ docker compose up -d --wait --no-recreate --no-deps api
Container mtusi-lab6-api-2 Running
Container mtusi-lab6-api-1 Starting
Container mtusi-lab6-api-1 Started
Container mtusi-lab6-api-1 Waiting
Container mtusi-lab6-api-2 Waiting
Container mtusi-lab6-api-1 Healthy
Container mtusi-lab6-api-2 Healthy
exit=0
$ python tests/resilience.py --mode failover --action-script scripts/stop-one.cmd --base-url http://127.0.0.1:8080
Executing explicit failover hook; hook output suppressed to avoid leaking secrets.
{"observed_instances_before": 2, "successful_writes": 119, "failed_attempts": 0, "first_recovery_seconds": 2.344}
PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.
exit=0
$ docker compose up -d --wait --no-recreate --no-deps api
Container mtusi-lab6-api-1 Running
Container mtusi-lab6-api-2 Starting
Container mtusi-lab6-api-2 Started
Container mtusi-lab6-api-1 Waiting
Container mtusi-lab6-api-2 Waiting
Container mtusi-lab6-api-1 Healthy
Container mtusi-lab6-api-2 Healthy
exit=0
$ docker compose run --rm -T s3-init
Container mtusi-lab6-seaweedfs-1 Running
Container mtusi-lab6-seaweedfs-1 Waiting
Container mtusi-lab6-seaweedfs-1 Healthy
Container mtusi-lab6-s3-init-run-ed2a1968cae4 Creating
Container mtusi-lab6-s3-init-run-ed2a1968cae4 Created
bucket messenger-attachments уже существует
анонимный доступ запрещён (HTTP 403)
ключ API ограничен одним bucket (create_bucket отклонён: AccessDenied)
bootstrap завершён
exit=0
$ docker compose run --rm -T s3-init
Container mtusi-lab6-seaweedfs-1 Running
Container mtusi-lab6-seaweedfs-1 Waiting
Container mtusi-lab6-seaweedfs-1 Healthy
Container mtusi-lab6-s3-init-run-4edec659c8c0 Creating
Container mtusi-lab6-s3-init-run-4edec659c8c0 Created
bucket messenger-attachments уже существует
анонимный доступ запрещён (HTTP 403)
ключ API ограничен одним bucket (create_bucket отклонён: AccessDenied)
bootstrap завершён
exit=0
$ python tests/smoke.py --base-url http://127.0.0.1:8080
PASS: lab6 public API scenario; infrastructure and grading evidence remain separate.
exit=0
@@ -0,0 +1,84 @@
{
"tempo_discarded_spans_before": null,
"tempo_discarded_spans_after": null,
"containers": {
"mtusi-lab6-api-1": {
"running": true,
"oom_killed": false,
"memory_bytes": 268435456,
"nano_cpus": 1000000000
},
"mtusi-lab6-api-2": {
"running": true,
"oom_killed": false,
"memory_bytes": 268435456,
"nano_cpus": 1000000000
},
"mtusi-lab6-grafana-1": {
"running": true,
"oom_killed": false,
"memory_bytes": 536870912,
"nano_cpus": 1000000000
},
"mtusi-lab6-lb-1": {
"running": true,
"oom_killed": false,
"memory_bytes": 134217728,
"nano_cpus": 1000000000
},
"mtusi-lab6-loki-1": {
"running": true,
"oom_killed": false,
"memory_bytes": 536870912,
"nano_cpus": 1000000000
},
"mtusi-lab6-otel-collector-1": {
"running": true,
"oom_killed": false,
"memory_bytes": 402653184,
"nano_cpus": 1000000000
},
"mtusi-lab6-postgres-1": {
"running": true,
"oom_killed": false,
"memory_bytes": 536870912,
"nano_cpus": 1000000000
},
"mtusi-lab6-prometheus-1": {
"running": true,
"oom_killed": false,
"memory_bytes": 536870912,
"nano_cpus": 1000000000
},
"mtusi-lab6-rabbitmq-1": {
"running": true,
"oom_killed": false,
"memory_bytes": 536870912,
"nano_cpus": 1000000000
},
"mtusi-lab6-seaweedfs-1": {
"running": true,
"oom_killed": false,
"memory_bytes": 805306368,
"nano_cpus": 1000000000
},
"mtusi-lab6-tempo-1": {
"running": true,
"oom_killed": false,
"memory_bytes": 1073741824,
"nano_cpus": 1000000000
},
"mtusi-lab6-valkey-1": {
"running": true,
"oom_killed": false,
"memory_bytes": 134217728,
"nano_cpus": 500000000
},
"mtusi-lab6-worker-1": {
"running": true,
"oom_killed": false,
"memory_bytes": 805306368,
"nano_cpus": 1000000000
}
}
}
@@ -0,0 +1,89 @@
{
"base_commit": "1d908578e3ccdec8440efdf2e5135580170159aa",
"branch": "solution/bpi2302-kushnarenko",
"source_sha256": {
"research/requirements.txt": "0a0f9ffaa9b56d590cb38c77f9da8dbc64f2d503b69fc27b0116b9a9594b2591",
"src/app/s3_recovery.py": "582b56ebd579c94670f28e801b7d66aa6dab2915bcbdc57f129f18651f6aa228",
"student-tests/README.md": "ab6c3c99ef61e14ba30037d16d51b3759d0ee02b71ecbf84b51ae941dfb3b90d",
"student-tests/test_dependencies.py": "e152b87441d1374ce9b69d3d441d262e47890313d2ccc798f5e2cff2e3719399",
"student-tests/test_expiry.py": "16a374ebe3df15c50c07a9fc3bc801c6c532dde3c4185b164c6aea4605923f83",
"student-tests/test_reconcile5.py": "7df51d06fc6da180fada10d3d6e1233c9462870ec18f6d55753a36e30dca458b",
"Dockerfile": "f1dc711cacf00e3e2bd6b4a0616c925e357446ebda370e7e5fdb3170eaecd4cb",
"compose.yaml": "acb6f5bb7a96221a077cff4f5311125cfafb16a98e0478a48c6c47b3b974b77d",
"infra/README.md": "33a4f7b870fd0b2bed98d46daf26d45dd7436176739672925246f8e3fe7c2278",
"infra/grafana/build_dashboard.py": "8a1eb1a037c8aeb473cfce9610682b15c1b92f4ed85efe95e61b8f293c87debf",
"infra/grafana/dashboards/messenger.json": "d897fa6a9ac06b05520906773cce0cba3bb5118576263bc82596d462b9b9ab94",
"infra/grafana/provisioning/dashboards/dashboards.yaml": "f2f11fb3a1d5c78745ededf551c187994bef0d3e0fbadc1c7bae3ae4bd76c52e",
"infra/grafana/provisioning/datasources/datasources.yaml": "fe956b90867757e930ba87022a3e77584b42949ddd814ed058d6502d64200e74",
"infra/haproxy/haproxy.cfg": "511fb24b32740c332cd7b6ea42d3f1cf252e1452de754ff97c8e6886f3692061",
"infra/loki/loki.yaml": "9069174e115badf13e559f2c9e8d3593603b451416737d9fa7d2724113da0bd3",
"infra/otel/README.md": "709332a3e910f9d42cb2a44c8d24b13aff6ce0ac7b671840e3cc273ddb82d3d6",
"infra/otel/collector-tail.yaml": "3e3b5c9a371a2f9f88d4f96a6da11630e652eeea42da7dc8386c7062c7887723",
"infra/otel/collector.yaml": "c0931a19c3b65aa893cff72aa4528aa5bf814b64ffac563506aa0ce59cc52d78",
"infra/prometheus/prometheus.yml": "b3c5272057d171c10f01674cc02e71f62d30c6abd4a560589cb3bc106de66294",
"infra/prometheus/rules/messenger.yml": "370cf706c19eed8acca5e310f3b036c88ef209e30cffde019d124fad35bd3252",
"infra/tempo/tempo.yaml": "f87d6dbc22c25e4007024ade5429a8e05a2176873da69004a8f234baa7877668",
"requirements.txt": "3538783836e3ba370df10d9335409a77fe1576f0269fea1917d54e37926e47a7",
"research/failover_sequence.py": "9d5dec92d067f467dc7842f9c698b39de28fb5c91902668917abaf1cceae5935",
"research/incidents.py": "4b2cd148b3b86e3e85565d4d7a5a11ad0d798f53a1709c077bb89952d4c2a98d",
"research/loadgen.py": "e4bee7ea46ad97822d440ea0bed35e36615ae60158ab4bc770179079a7558303",
"research/obs_cost.py": "88ecc651d0ca5d51eb5e0afafea09f12b4cd3eb91aa3400ef04eff88b502bfd0",
"research/plot_obs_cost.py": "67e491695321a43882d9ae32645914bfe02650ee022858a43049781f3a92c1ba",
"research/queue_overload.py": "6d0a3fdb92899a04fb1d98f5d1f4fbbb27396284dc8186266522b83c85565dbc",
"research/rolling_update.py": "447da4a9e2161bb232669f0558d6cb15ae3f0ef181fdf563e19cd56135b58240",
"scripts/check_contract.py": "61e300fe5c1074360418e24b4653b12f96808247a2beaf2a64859aa360f81493",
"scripts/init-secrets.sh": "3e238e5066f323e424f26cea9e360792d8a3d55d3755742e81d130707bfe967d",
"scripts/restart.cmd": "b4f0ada9f8b0d009a3fbaa84aecef04b6d4d0f0229c938a3a8c68ad3ba76a7f8",
"scripts/restart.example.sh": "0328246ba75defbced992104e986fde973f63e3830d83251eced5953263e944b",
"scripts/restart.sh": "7a567573bf3510a7d4ab5a10f656a9c3ef2f0ead1ae0802f6b4a55b800d235d0",
"scripts/restore-replicas.sh": "4e89f3f27ab5e9c958d3716c3bb2a8323133d773617bfec4b137f1c0b40d1e97",
"scripts/rolling-update.sh": "9663b1fe8eeda5dafdc4338105e77cf7696c117d0f4652950ae8283d50439d4d",
"scripts/stop-one.cmd": "a12b882f0f580aa10591d167e69e01d4a5fc8c8e0520773a542b21b1aac43d39",
"scripts/stop-one.example.sh": "1bcf34394520b89d95b7605ec56675be799eea920e83cfbba306d5dc0a6d9811",
"scripts/stop-one.sh": "9ae4a3d75e7d857ac3c62c2d97b95282b33a30b1affbf83df6de8f88c2ffded1",
"scripts/validate_openapi.py": "cd7a33221b1860c5d2330e52afa6cb222d375db671ae9ab379ba51a0d3a77946",
"src/app/attachments.py": "ba4f48e2220e39e4d3dd4d04f571afc1b3bf68f1b809b2eaeff240a1da3b3329",
"src/app/broker.py": "b68f297acd17bbd4d63b140cef06b89f73048da6d35fc6027717d54c5cf3733b",
"src/app/config.py": "f9483cb31afd5a9be0f00e2ea71a32dd1216d5d48b4c0595ee43d31ffbfa0a61",
"src/app/db.py": "fc6a3872f977550b2fef7cd9db9b36e1fdd8b1fc95abe77f36916d2c8422222f",
"src/app/images.py": "3dc7434cbfbc9d4c01b95eb897c1332b2a2748dd9df88c99a7086f3e4cdf4c41",
"src/app/main.py": "1ec3323d1edfcad9a54f8f7b62e8064daaa69555cb5cd2b87da20f7313bae954",
"src/app/migrate.py": "cabc192ca4182af780636415a008d2dd4636dad7d3cf0f4a8a2b52a48aedaa7e",
"src/app/outbox.py": "4255f981c059fad0dee7d355d746f53a3a1c86a8d2e7cbd9aed3e12bedb895f0",
"src/app/passwords.py": "836fc282093058f6717eaa6374775251ed655dcd517f325dc60323a88154f261",
"src/app/pipeline.py": "222ac87f275e4607ecdc98cdc5f68d8957fd7f68603af2d2643f3026445a3fe1",
"src/app/reconcile.py": "c7dd35bf40e672ffe15f0e878913e17f6c6d9de3f69922b4edb38d1802330dc4",
"src/app/replay.py": "2669b9e5fa2f6fb83dbe457f0241db166eb3e370e21d81c3e32f922060f52fa5",
"src/app/s3.py": "86cb37980023d02e56468a355793b69a8c5f80246d36460e17f49c102385f0de",
"src/app/s3_bootstrap.py": "216d8121d6e7b25d95b845f4deaff18fd31eeb28484e51fe4c463c646a3af0d1",
"src/app/serve.py": "3ebe12cf8cb2dddfc7d7234bba9bd190d868cc88d02e61fe640ba51dcf115b55",
"src/app/sessions.py": "f975eb27c50c024f3e562f755ee97bb7bfe21cf5e01cc1d347bf107971c30670",
"src/app/telemetry.py": "16a789f79fa185598d9de9915d89e6b5275fdec819c4df78a092a7c47e952686",
"src/app/web.py": "0c5aa4ab059a1dac64733217e143c9f96fce1f321d2727cf5580b62e04941bfd",
"src/app/worker.py": "23024debdd22076e586d5490874de0de6167e21e472e0a83fbbc86f001e53570",
"student-tests/fixtures/alpha.png": "1f9d6ee26fa7beb4ba179c7c38ae97f5f31e95ad9d4108267352225e6705882f",
"student-tests/fixtures/exif_rot6.jpg": "62386265960512fd51279e40723331c6c52f81ba2325d82afc3fb14f3b332ddc",
"student-tests/fixtures/poison.png": "2be4bea6355e83da4733b1d7f59dc985dfe22cf5d8283eea9b20871d1f43c252",
"student-tests/fixtures/sample.jpg": "33a614b40fe6ae25fdbd99af22e7d9d3e5dbea455b36aac750de112c034a6311",
"student-tests/fixtures/sample.png": "99ea7533c441f79cd630d15ce4d58d1ec4e06f6f0efb3b77a5c088305cd1bd75",
"student-tests/fixtures/small.png": "001e6b4538ce86cbb0c88a34a2b88c111336b8dc8825af81e0f7d9630b731e32",
"student-tests/test_api.py": "2635113842a5b26ec1fdd98ea42fc861edb1d10895e8be6c2eff46ed73fdba9d",
"student-tests/test_attachments.py": "441dc4c5a2d212c135a7d4326a127c95d20bdce6107a8e58fd7d3551415d59c1",
"student-tests/test_idempotency.py": "4ee985a3ef26fefa816fdb136fa2b0644cca13d73fe3a226d1ba975a55698b5b",
"student-tests/test_observability.py": "de61bdb52ca55c361b2394ea82e35039eca38509d3508d9918c4307ddc2336d7",
"student-tests/test_observability_retry.py": "e38e8ab3528479ff3570af1959896571a15d18cfdf8f3761703bd6640feac999",
"student-tests/test_pipeline.py": "1f6b031b0b67a81d4db1e09e8e334efa65a123cefd49c2fd8db05e9505834ad0",
"student-tests/test_resilience5.py": "8906a99d9b3dbc0e47f2e963e95c5ef3c3aa7577767b6c3575eec4617ec7e23b",
"student-tests/test_topology.py": "4401a2db10c65c41430cc0c47c37bee6aefc902880735fb489acad7ad0ff779f"
},
"hardware": {
"cpu": "Intel Core i9-12900KS",
"cores": 16,
"logical_cpus": 24,
"ram_bytes": 102830256128,
"docker_cpu_count": 24,
"docker_memory_bytes": 50392047616,
"background_other_docker_project": true
},
"client_python": "3.11.9 (tags/v3.11.9:de54cf5, Apr 2 2024, 10:12:12) [MSC v.1938 64 bit (AMD64)]"
}
+115
View File
@@ -0,0 +1,115 @@
#!/usr/bin/env python3
"""Генератор dashboard JSON (infra/grafana/dashboards/messenger.json). Запуск: python infra/grafana/build_dashboard.py
Dashboard хранится в Git как JSON и импортируется provisioning-ом автоматически; генератор — только удобство правки."""
import json
import os
PROM = {"type": "prometheus", "uid": "prometheus"}
LOKI = {"type": "loki", "uid": "loki"}
TEMPO = {"type": "tempo", "uid": "tempo"}
panel_id = [0]
def _id():
panel_id[0] += 1
return panel_id[0]
def stat_or_ts(title, exprs, x, y, w=8, h=7, unit="short", kind="timeseries", desc="", thresholds=None, ds=PROM, legends=None):
p = {"id": _id(), "type": kind, "title": title, "description": desc, "datasource": ds,
"gridPos": {"x": x, "y": y, "w": w, "h": h},
"targets": [{"refId": chr(65 + i), "expr": e, "datasource": ds, "legendFormat": (legends or [""] * len(exprs))[i]}
for i, e in enumerate(exprs)],
"fieldConfig": {"defaults": {"unit": unit, "custom": {"lineWidth": 1, "fillOpacity": 10}}, "overrides": []},
"options": {"legend": {"displayMode": "list", "placement": "bottom"}}}
if thresholds:
p["fieldConfig"]["defaults"]["thresholds"] = {"mode": "absolute", "steps": thresholds}
return p
def row(title, y):
return {"id": _id(), "type": "row", "title": title, "gridPos": {"x": 0, "y": y, "w": 24, "h": 1}, "collapsed": False, "panels": []}
NB = '{http_route!~"/health.*"}'
panels = []
y = 0
panels.append(row("API (бизнес-запросы без /health) — запросы, ошибки, задержки", y)); y += 1
panels += [
stat_or_ts("RPS по маршрутам", ['sum by (http_route) (rate(http_server_request_duration_seconds_count%s[1m]))' % NB], 0, y,
unit="reqps", legends=["{{http_route}}"], desc="Единицы: запросов в секунду. Источник: http_server_request_duration_seconds_count."),
stat_or_ts("Доля ошибок 5xx (SLI)", ['sli:http_error_ratio:1m'], 8, y, unit="percentunit",
desc="Числитель: 5xx; знаменатель: все запросы кроме /health. 4xx (в т.ч. неверный пароль) не считаются отказом сервиса.",
thresholds=[{"color": "green", "value": None}, {"color": "red", "value": 0.05}]),
stat_or_ts("Latency p50 / p95 / p99", [
'histogram_quantile(0.50, sum by (le) (rate(http_server_request_duration_seconds_bucket%s[1m])))' % NB,
'histogram_quantile(0.95, sum by (le) (rate(http_server_request_duration_seconds_bucket%s[1m])))' % NB,
'histogram_quantile(0.99, sum by (le) (rate(http_server_request_duration_seconds_bucket%s[1m])))' % NB],
16, y, unit="s", legends=["p50", "p95", "p99"],
desc="Процентили считаются из суммарной гистограммы всех реплик (sum by le), а не усреднением p95 реплик."),
]
y += 7
panels += [
stat_or_ts("RPS по статус-кодам", ['sum by (http_response_status_code) (rate(http_server_request_duration_seconds_count%s[1m]))' % NB],
0, y, unit="reqps", legends=["{{http_response_status_code}}"]),
stat_or_ts("Latency p95 по маршрутам", ['histogram_quantile(0.95, sum by (le, http_route) (rate(http_server_request_duration_seconds_bucket%s[1m])))' % NB],
8, y, unit="s", legends=["{{http_route}}"]),
stat_or_ts("DB pool: занято / простаивает / ожидают", ['sum(db_client_connections_usage{state="used"})', 'sum(db_client_connections_usage{state="idle"})',
'sum(db_client_connections_waiting)'], 16, y, unit="short",
legends=["used", "idle", "waiting"], desc="Пул соединений PostgreSQL по всем репликам API. Рост waiting — насыщение пула."),
]
y += 7
panels.append(row("Задания (worker) и очередь", y)); y += 1
panels += [
stat_or_ts("Задания по исходам", ['sum by (outcome) (rate(messenger_jobs_processed_total[1m]))'], 0, y, unit="ops", legends=["{{outcome}}"],
desc="ready — успешно; failed — финальная ошибка; retried — временная ошибка, запланирован повтор; noop — повторная доставка."),
stat_or_ts("Длительность обработки p50 / p95 (worker)", [
'histogram_quantile(0.50, sum by (le) (rate(messenger_job_duration_seconds_bucket[1m])))',
'histogram_quantile(0.95, sum by (le) (rate(messenger_job_duration_seconds_bucket[1m])))'], 8, y, unit="s", legends=["p50", "p95"]),
stat_or_ts("Время до ready p50 / p95", [
'histogram_quantile(0.50, sum by (le) (rate(messenger_job_time_to_ready_seconds_bucket[1m])))',
'histogram_quantile(0.95, sum by (le) (rate(messenger_job_time_to_ready_seconds_bucket[1m])))'], 16, y, unit="s", legends=["p50", "p95"],
desc="От устойчивого приёма загрузки (202) до ready. Включает ожидание в очереди."),
]
y += 7
panels += [
stat_or_ts("Глубина очереди (queued + processing)", ['max(messenger_queue_depth)'], 0, y, unit="short", legends=["jobs"]),
stat_or_ts("Возраст старейшей задачи", ['max(messenger_queue_oldest_job_age_seconds)'], 8, y, unit="s", legends=["oldest"],
thresholds=[{"color": "green", "value": None}, {"color": "red", "value": 30}],
desc="Backlog — не задержка обработки: показывает, как долго ждёт самое старое задание."),
stat_or_ts("Worker: занято / ёмкость", ['sum(messenger_worker_busy)', 'sum(messenger_worker_capacity)'], 16, y, legends=["busy", "capacity"],
desc="Насыщение worker: busy == capacity при растущей очереди означает нехватку мощности."),
]
y += 7
panels.append(row("Ресурсы процессов", y)); y += 1
panels += [
stat_or_ts("RSS по сервисам", ['sum by (service_name) (process_memory_rss_bytes)'], 0, y, unit="bytes", legends=["{{service_name}}"]),
stat_or_ts("CPU по сервисам (ядра)", ['sum by (service_name) (rate(process_cpu_time_seconds_total[1m]))'], 8, y, unit="short", legends=["{{service_name}}"],
desc="rate накопленного CPU-времени процесса: 1.0 = одно ядро."),
stat_or_ts("Collector: очередь экспорта и отказы", ['sum by (exporter) (otelcol_exporter_queue_size)',
'sum by (exporter) (rate(otelcol_exporter_send_failed_spans[1m]))',
'sum(rate(otelcol_processor_memory_limiter_refused_spans[1m]))'], 16, y, legends=["queue {{exporter}}", "failed spans {{exporter}}", "refused (memory_limiter)"],
desc="Поведение Collector при недоступном backend: ограниченная очередь, отказы, memory_limiter."),
]
y += 7
panels.append(row("Логи и алерты", y)); y += 1
panels.append({"id": _id(), "type": "logs", "title": "Структурные логи (API и worker), ошибки и предупреждения", "datasource": LOKI,
"gridPos": {"x": 0, "y": y, "w": 16, "h": 10},
"targets": [{"refId": "A", "datasource": LOKI, "expr": '{service_name=~"messenger-.*"} | detected_level=~"warn|warning|error|ERROR|WARN"'}],
"options": {"showTime": True, "wrapLogMessage": True, "enableLogDetails": True, "sortOrder": "Descending"}})
panels.append({"id": _id(), "type": "alertlist", "title": "Алерты (firing / pending)", "datasource": PROM,
"gridPos": {"x": 16, "y": y, "w": 8, "h": 10},
"options": {"alertInstanceLabelFilter": "", "alertName": "", "dashboardAlerts": False, "groupBy": [], "groupMode": "default",
"maxItems": 20, "sortOrder": 1, "stateFilter": {"firing": True, "pending": True, "noData": True, "normal": False, "error": True}}})
y += 10
panels.append({"id": _id(), "type": "traces", "title": "Последние трассы API (Tempo)", "datasource": TEMPO,
"gridPos": {"x": 0, "y": y, "w": 24, "h": 8},
"targets": [{"refId": "A", "datasource": TEMPO, "queryType": "traceql", "query": '{ resource.service.name = "messenger-api" }', "limit": 20}]})
dashboard = {"uid": "messenger-overview", "title": "Messenger: API, очередь, worker", "tags": ["messenger", "lab6"], "timezone": "browser",
"schemaVersion": 39, "version": 1, "refresh": "5s", "time": {"from": "now-15m", "to": "now"}, "panels": panels,
"annotations": {"list": []}, "templating": {"list": []}}
out = os.path.join(os.path.dirname(os.path.abspath(__file__)), "dashboards", "messenger.json")
with open(out, "w", encoding="utf-8") as fh:
json.dump(dashboard, fh, ensure_ascii=False, indent=1)
print("saved", out)
+917
View File
@@ -0,0 +1,917 @@
{
"uid": "messenger-overview",
"title": "Messenger: API, очередь, worker",
"tags": [
"messenger",
"lab6"
],
"timezone": "browser",
"schemaVersion": 39,
"version": 1,
"refresh": "5s",
"time": {
"from": "now-15m",
"to": "now"
},
"panels": [
{
"id": 1,
"type": "row",
"title": "API (бизнес-запросы без /health) — запросы, ошибки, задержки",
"gridPos": {
"x": 0,
"y": 0,
"w": 24,
"h": 1
},
"collapsed": false,
"panels": []
},
{
"id": 2,
"type": "timeseries",
"title": "RPS по маршрутам",
"description": "Единицы: запросов в секунду. Источник: http_server_request_duration_seconds_count.",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"x": 0,
"y": 1,
"w": 8,
"h": 7
},
"targets": [
{
"refId": "A",
"expr": "sum by (http_route) (rate(http_server_request_duration_seconds_count{http_route!~\"/health.*\"}[1m]))",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"legendFormat": "{{http_route}}"
}
],
"fieldConfig": {
"defaults": {
"unit": "reqps",
"custom": {
"lineWidth": 1,
"fillOpacity": 10
}
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
}
}
},
{
"id": 3,
"type": "timeseries",
"title": "Доля ошибок 5xx (SLI)",
"description": "Числитель: 5xx; знаменатель: все запросы кроме /health. 4xx (в т.ч. неверный пароль) не считаются отказом сервиса.",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"x": 8,
"y": 1,
"w": 8,
"h": 7
},
"targets": [
{
"refId": "A",
"expr": "sli:http_error_ratio:1m",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"legendFormat": ""
}
],
"fieldConfig": {
"defaults": {
"unit": "percentunit",
"custom": {
"lineWidth": 1,
"fillOpacity": 10
},
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 0.05
}
]
}
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
}
}
},
{
"id": 4,
"type": "timeseries",
"title": "Latency p50 / p95 / p99",
"description": "Процентили считаются из суммарной гистограммы всех реплик (sum by le), а не усреднением p95 реплик.",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"x": 16,
"y": 1,
"w": 8,
"h": 7
},
"targets": [
{
"refId": "A",
"expr": "histogram_quantile(0.50, sum by (le) (rate(http_server_request_duration_seconds_bucket{http_route!~\"/health.*\"}[1m])))",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"legendFormat": "p50"
},
{
"refId": "B",
"expr": "histogram_quantile(0.95, sum by (le) (rate(http_server_request_duration_seconds_bucket{http_route!~\"/health.*\"}[1m])))",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"legendFormat": "p95"
},
{
"refId": "C",
"expr": "histogram_quantile(0.99, sum by (le) (rate(http_server_request_duration_seconds_bucket{http_route!~\"/health.*\"}[1m])))",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"legendFormat": "p99"
}
],
"fieldConfig": {
"defaults": {
"unit": "s",
"custom": {
"lineWidth": 1,
"fillOpacity": 10
}
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
}
}
},
{
"id": 5,
"type": "timeseries",
"title": "RPS по статус-кодам",
"description": "",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"x": 0,
"y": 8,
"w": 8,
"h": 7
},
"targets": [
{
"refId": "A",
"expr": "sum by (http_response_status_code) (rate(http_server_request_duration_seconds_count{http_route!~\"/health.*\"}[1m]))",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"legendFormat": "{{http_response_status_code}}"
}
],
"fieldConfig": {
"defaults": {
"unit": "reqps",
"custom": {
"lineWidth": 1,
"fillOpacity": 10
}
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
}
}
},
{
"id": 6,
"type": "timeseries",
"title": "Latency p95 по маршрутам",
"description": "",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"x": 8,
"y": 8,
"w": 8,
"h": 7
},
"targets": [
{
"refId": "A",
"expr": "histogram_quantile(0.95, sum by (le, http_route) (rate(http_server_request_duration_seconds_bucket{http_route!~\"/health.*\"}[1m])))",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"legendFormat": "{{http_route}}"
}
],
"fieldConfig": {
"defaults": {
"unit": "s",
"custom": {
"lineWidth": 1,
"fillOpacity": 10
}
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
}
}
},
{
"id": 7,
"type": "timeseries",
"title": "DB pool: занято / простаивает / ожидают",
"description": "Пул соединений PostgreSQL по всем репликам API. Рост waiting — насыщение пула.",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"x": 16,
"y": 8,
"w": 8,
"h": 7
},
"targets": [
{
"refId": "A",
"expr": "sum(db_client_connections_usage{state=\"used\"})",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"legendFormat": "used"
},
{
"refId": "B",
"expr": "sum(db_client_connections_usage{state=\"idle\"})",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"legendFormat": "idle"
},
{
"refId": "C",
"expr": "sum(db_client_connections_waiting)",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"legendFormat": "waiting"
}
],
"fieldConfig": {
"defaults": {
"unit": "short",
"custom": {
"lineWidth": 1,
"fillOpacity": 10
}
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
}
}
},
{
"id": 8,
"type": "row",
"title": "Задания (worker) и очередь",
"gridPos": {
"x": 0,
"y": 15,
"w": 24,
"h": 1
},
"collapsed": false,
"panels": []
},
{
"id": 9,
"type": "timeseries",
"title": "Задания по исходам",
"description": "ready — успешно; failed — финальная ошибка; retried — временная ошибка, запланирован повтор; noop — повторная доставка.",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"x": 0,
"y": 16,
"w": 8,
"h": 7
},
"targets": [
{
"refId": "A",
"expr": "sum by (outcome) (rate(messenger_jobs_processed_total[1m]))",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"legendFormat": "{{outcome}}"
}
],
"fieldConfig": {
"defaults": {
"unit": "ops",
"custom": {
"lineWidth": 1,
"fillOpacity": 10
}
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
}
}
},
{
"id": 10,
"type": "timeseries",
"title": "Длительность обработки p50 / p95 (worker)",
"description": "",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"x": 8,
"y": 16,
"w": 8,
"h": 7
},
"targets": [
{
"refId": "A",
"expr": "histogram_quantile(0.50, sum by (le) (rate(messenger_job_duration_seconds_bucket[1m])))",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"legendFormat": "p50"
},
{
"refId": "B",
"expr": "histogram_quantile(0.95, sum by (le) (rate(messenger_job_duration_seconds_bucket[1m])))",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"legendFormat": "p95"
}
],
"fieldConfig": {
"defaults": {
"unit": "s",
"custom": {
"lineWidth": 1,
"fillOpacity": 10
}
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
}
}
},
{
"id": 11,
"type": "timeseries",
"title": "Время до ready p50 / p95",
"description": "От устойчивого приёма загрузки (202) до ready. Включает ожидание в очереди.",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"x": 16,
"y": 16,
"w": 8,
"h": 7
},
"targets": [
{
"refId": "A",
"expr": "histogram_quantile(0.50, sum by (le) (rate(messenger_job_time_to_ready_seconds_bucket[1m])))",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"legendFormat": "p50"
},
{
"refId": "B",
"expr": "histogram_quantile(0.95, sum by (le) (rate(messenger_job_time_to_ready_seconds_bucket[1m])))",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"legendFormat": "p95"
}
],
"fieldConfig": {
"defaults": {
"unit": "s",
"custom": {
"lineWidth": 1,
"fillOpacity": 10
}
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
}
}
},
{
"id": 12,
"type": "timeseries",
"title": "Глубина очереди (queued + processing)",
"description": "",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"x": 0,
"y": 23,
"w": 8,
"h": 7
},
"targets": [
{
"refId": "A",
"expr": "max(messenger_queue_depth)",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"legendFormat": "jobs"
}
],
"fieldConfig": {
"defaults": {
"unit": "short",
"custom": {
"lineWidth": 1,
"fillOpacity": 10
}
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
}
}
},
{
"id": 13,
"type": "timeseries",
"title": "Возраст старейшей задачи",
"description": "Backlog — не задержка обработки: показывает, как долго ждёт самое старое задание.",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"x": 8,
"y": 23,
"w": 8,
"h": 7
},
"targets": [
{
"refId": "A",
"expr": "max(messenger_queue_oldest_job_age_seconds)",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"legendFormat": "oldest"
}
],
"fieldConfig": {
"defaults": {
"unit": "s",
"custom": {
"lineWidth": 1,
"fillOpacity": 10
},
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 30
}
]
}
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
}
}
},
{
"id": 14,
"type": "timeseries",
"title": "Worker: занято / ёмкость",
"description": "Насыщение worker: busy == capacity при растущей очереди означает нехватку мощности.",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"x": 16,
"y": 23,
"w": 8,
"h": 7
},
"targets": [
{
"refId": "A",
"expr": "sum(messenger_worker_busy)",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"legendFormat": "busy"
},
{
"refId": "B",
"expr": "sum(messenger_worker_capacity)",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"legendFormat": "capacity"
}
],
"fieldConfig": {
"defaults": {
"unit": "short",
"custom": {
"lineWidth": 1,
"fillOpacity": 10
}
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
}
}
},
{
"id": 15,
"type": "row",
"title": "Ресурсы процессов",
"gridPos": {
"x": 0,
"y": 30,
"w": 24,
"h": 1
},
"collapsed": false,
"panels": []
},
{
"id": 16,
"type": "timeseries",
"title": "RSS по сервисам",
"description": "",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"x": 0,
"y": 31,
"w": 8,
"h": 7
},
"targets": [
{
"refId": "A",
"expr": "sum by (service_name) (process_memory_rss_bytes)",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"legendFormat": "{{service_name}}"
}
],
"fieldConfig": {
"defaults": {
"unit": "bytes",
"custom": {
"lineWidth": 1,
"fillOpacity": 10
}
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
}
}
},
{
"id": 17,
"type": "timeseries",
"title": "CPU по сервисам (ядра)",
"description": "rate накопленного CPU-времени процесса: 1.0 = одно ядро.",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"x": 8,
"y": 31,
"w": 8,
"h": 7
},
"targets": [
{
"refId": "A",
"expr": "sum by (service_name) (rate(process_cpu_time_seconds_total[1m]))",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"legendFormat": "{{service_name}}"
}
],
"fieldConfig": {
"defaults": {
"unit": "short",
"custom": {
"lineWidth": 1,
"fillOpacity": 10
}
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
}
}
},
{
"id": 18,
"type": "timeseries",
"title": "Collector: очередь экспорта и отказы",
"description": "Поведение Collector при недоступном backend: ограниченная очередь, отказы, memory_limiter.",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"x": 16,
"y": 31,
"w": 8,
"h": 7
},
"targets": [
{
"refId": "A",
"expr": "sum by (exporter) (otelcol_exporter_queue_size)",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"legendFormat": "queue {{exporter}}"
},
{
"refId": "B",
"expr": "sum by (exporter) (rate(otelcol_exporter_send_failed_spans[1m]))",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"legendFormat": "failed spans {{exporter}}"
},
{
"refId": "C",
"expr": "sum(rate(otelcol_processor_memory_limiter_refused_spans[1m]))",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"legendFormat": "refused (memory_limiter)"
}
],
"fieldConfig": {
"defaults": {
"unit": "short",
"custom": {
"lineWidth": 1,
"fillOpacity": 10
}
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
}
}
},
{
"id": 19,
"type": "row",
"title": "Логи и алерты",
"gridPos": {
"x": 0,
"y": 38,
"w": 24,
"h": 1
},
"collapsed": false,
"panels": []
},
{
"id": 20,
"type": "logs",
"title": "Структурные логи (API и worker), ошибки и предупреждения",
"datasource": {
"type": "loki",
"uid": "loki"
},
"gridPos": {
"x": 0,
"y": 39,
"w": 16,
"h": 10
},
"targets": [
{
"refId": "A",
"datasource": {
"type": "loki",
"uid": "loki"
},
"expr": "{service_name=~\"messenger-.*\"} | detected_level=~\"warn|warning|error|ERROR|WARN\""
}
],
"options": {
"showTime": true,
"wrapLogMessage": true,
"enableLogDetails": true,
"sortOrder": "Descending"
}
},
{
"id": 21,
"type": "alertlist",
"title": "Алерты (firing / pending)",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"x": 16,
"y": 39,
"w": 8,
"h": 10
},
"options": {
"alertInstanceLabelFilter": "",
"alertName": "",
"dashboardAlerts": false,
"groupBy": [],
"groupMode": "default",
"maxItems": 20,
"sortOrder": 1,
"stateFilter": {
"firing": true,
"pending": true,
"noData": true,
"normal": false,
"error": true
}
}
},
{
"id": 22,
"type": "traces",
"title": "Последние трассы API (Tempo)",
"datasource": {
"type": "tempo",
"uid": "tempo"
},
"gridPos": {
"x": 0,
"y": 49,
"w": 24,
"h": 8
},
"targets": [
{
"refId": "A",
"datasource": {
"type": "tempo",
"uid": "tempo"
},
"queryType": "traceql",
"query": "{ resource.service.name = \"messenger-api\" }",
"limit": 20
}
]
}
],
"annotations": {
"list": []
},
"templating": {
"list": []
}
}
@@ -0,0 +1,11 @@
apiVersion: 1
providers:
- name: messenger
orgId: 1
folder: Messenger
type: file
disableDeletion: false
allowUiUpdates: false
updateIntervalSeconds: 10
options:
path: /var/lib/grafana/dashboards
@@ -0,0 +1,41 @@
apiVersion: 1
# Источники данных Grafana: Prometheus (метрики), Loki (логи), Tempo (трассы). Связи: лог -> трасса (derivedFields),
# трасса -> логи (tracesToLogsV2), трасса -> метрики.
datasources:
- name: Prometheus
uid: prometheus
type: prometheus
access: proxy
url: http://prometheus:9090
isDefault: true
jsonData:
timeInterval: 5s
- name: Loki
uid: loki
type: loki
access: proxy
url: http://loki:3100
jsonData:
derivedFields:
- name: trace_id
matcherType: label
matcherRegex: trace_id
url: "$${__value.raw}"
datasourceUid: tempo
- name: Tempo
uid: tempo
type: tempo
access: proxy
url: http://tempo:3200
jsonData:
tracesToLogsV2:
datasourceUid: loki
spanStartTimeShift: "-1m"
spanEndTimeShift: "1m"
filterByTraceID: true
customQuery: true
query: '{service_name="$${__span.tags["service.name"]}"} | trace_id="$${__span.traceId}"'
tracesToMetrics:
datasourceUid: prometheus
serviceMap:
datasourceUid: prometheus
+62
View File
@@ -0,0 +1,62 @@
# Балансировщик lab3: HAProxy 3.0. Единственная публичная точка входа; API-реплики наружу не опубликованы.
global
log stdout format raw local0 notice
maxconn 4096
defaults
mode http
log global
option dontlognull
option forwardfor
# connect короткий: недоступная реплика быстро заменяется другой (redispatch)
timeout connect 1s
timeout client 30s
timeout server 30s
timeout http-request 10s
timeout http-keep-alive 5s
timeout queue 5s
retries 3
option redispatch 1
# Only retry before a connection is established: upload POST has no idempotency key.
retry-on conn-failure
# Docker DNS (127.0.0.11): адреса реплик API обновляются без перезапуска LB (scale up/down, пересоздание контейнеров).
resolvers docker
nameserver dns 127.0.0.11:53
accepted_payload_size 8192
resolve_retries 3
timeout resolve 1s
timeout retry 1s
hold valid 2s
hold nx 2s
hold other 2s
hold obsolete 2s
frontend public
bind :8080
# If the shared DB is down, live API processes still produce contract JSON,
# validate missing credentials, and serve /health/live independently of DB.
use_backend api_live if { nbsrv(api) eq 0 }
default_backend api
# Внутренний healthcheck самого LB (контейнерный HEALTHCHECK); наружу не публикуется.
frontend lb_health
bind :8081
monitor-uri /healthz
backend api
balance roundrobin
# Проверка готовности каждые 1 с; 2 неудачи подряд -> сервер исключён (обнаружение отказа <= ~3 с), 2 успеха -> возвращён.
option httpchk
http-check send meth GET uri /health/ready
http-check expect status 200
default-server check inter 1s fall 2 rise 2 resolvers docker init-addr none
server-template api 1-8 api:8080
backend api_live
balance roundrobin
option httpchk
http-check send meth GET uri /health/live
http-check expect status 200
default-server check inter 1s fall 2 rise 2 resolvers docker init-addr none
server-template api 1-8 api:8080
+37
View File
@@ -0,0 +1,37 @@
# Loki: хранилище логов (filesystem, volume lokidata). Принимает OTLP/HTTP от Collector (/otlp/v1/logs).
auth_enabled: false
server:
http_listen_port: 3100
grpc_listen_port: 9096
log_level: warn
common:
instance_addr: 127.0.0.1
path_prefix: /loki
storage:
filesystem:
chunks_directory: /loki/chunks
rules_directory: /loki/rules
replication_factor: 1
ring:
kvstore:
store: inmemory
schema_config:
configs:
- from: 2024-04-01
store: tsdb
object_store: filesystem
schema: v13
index:
prefix: index_
period: 24h
limits_config:
allow_structured_metadata: true
retention_period: 168h
reject_old_samples: false
volume_enabled: true
compactor:
working_directory: /loki/compactor
retention_enabled: true
delete_request_store: filesystem
analytics:
reporting_enabled: false
+64
View File
@@ -0,0 +1,64 @@
# Профиль «tail sampling» (трек 1, lab6): SDK отправляет 100 % спанов, Collector оставляет
# - все трассы с ошибками, - все медленные (> 500 мс), - 10 % остальных.
# Метрики и логи без изменений. Подключается через COLLECTOR_CONFIG=/etc/otelcol/collector-tail.yaml.
receivers:
otlp:
protocols:
grpc: {endpoint: 0.0.0.0:4317}
http: {endpoint: 0.0.0.0:4318}
processors:
memory_limiter: {check_interval: 1s, limit_mib: 280, spike_limit_mib: 60}
batch: {send_batch_size: 1024, send_batch_max_size: 2048, timeout: 2s}
attributes/scrub:
actions:
- {key: http.request.header.authorization, action: delete}
- {key: http.request.header.cookie, action: delete}
- {key: db.statement, action: delete}
- {key: url.query, action: delete}
tail_sampling:
decision_wait: 5s
num_traces: 20000
expected_new_traces_per_sec: 500
policies:
- {name: errors, type: status_code, status_code: {status_codes: [ERROR]}}
- {name: slow, type: latency, latency: {threshold_ms: 500}}
- {name: baseline, type: probabilistic, probabilistic: {sampling_percentage: 10}}
exporters:
otlp_grpc/tempo:
endpoint: tempo:4317
tls: {insecure: true}
sending_queue: {enabled: true, num_consumers: 2, queue_size: 500}
retry_on_failure: {enabled: true, initial_interval: 1s, max_interval: 10s, max_elapsed_time: 30s}
otlp_http/loki:
endpoint: http://loki:3100/otlp
sending_queue: {enabled: true, num_consumers: 2, queue_size: 500}
retry_on_failure: {enabled: true, initial_interval: 1s, max_interval: 10s, max_elapsed_time: 30s}
prometheus:
endpoint: 0.0.0.0:8889
resource_to_telemetry_conversion: {enabled: true}
metric_expiration: 20s
service:
telemetry:
metrics:
level: normal
readers:
- pull:
exporter:
prometheus: {host: 0.0.0.0, port: 8888}
logs: {level: warn}
pipelines:
traces:
receivers: [otlp]
processors: [memory_limiter, attributes/scrub, tail_sampling, batch]
exporters: [otlp_grpc/tempo]
metrics:
receivers: [otlp]
processors: [memory_limiter, batch]
exporters: [prometheus]
logs:
receivers: [otlp]
processors: [memory_limiter, attributes/scrub, batch]
exporters: [otlp_http/loki]
+75
View File
@@ -0,0 +1,75 @@
# OpenTelemetry Collector (contrib). Один пайплайн на сигнал, у каждого — реальный backend:
# traces -> Tempo (OTLP/gRPC)
# metrics -> Prometheus (Prometheus-экспортёр: Prometheus сам забирает /metrics у Collector)
# logs -> Loki (OTLP/HTTP, нативный приём Loki)
# Collector — маршрутизатор/обработчик, не долговременное хранилище. Порты 4317/4318 доступны ТОЛЬКО во внутренней сети backend.
receivers:
otlp:
protocols:
grpc:
endpoint: 0.0.0.0:4317
http:
endpoint: 0.0.0.0:4318
processors:
# Защита RAM Collector: при приближении к лимиту новые данные отклоняются (клиент SDK получает ошибку и отбрасывает
# пакет, приложение продолжает работать). limit_mib согласован с mem_limit контейнера.
memory_limiter:
check_interval: 1s
limit_mib: 280
spike_limit_mib: 60
batch:
send_batch_size: 1024
send_batch_max_size: 2048
timeout: 2s
# Гарантированное удаление чувствительных атрибутов, даже если приложение их случайно передаст.
attributes/scrub:
actions:
- {key: http.request.header.authorization, action: delete}
- {key: http.request.header.cookie, action: delete}
- {key: http.request.body, action: delete}
- {key: db.statement, action: delete}
- {key: url.query, action: delete}
- {key: http.target, action: delete}
# Tail-sampling (профиль «tail», см. collector-tail.yaml): здесь не используется.
exporters:
otlp_grpc/tempo:
endpoint: tempo:4317
tls: {insecure: true}
# Ограниченная буферизация при недоступном backend: очередь фиксированного размера, повторы до 30 с, затем отбрасывание
# (потери считаются метрикой otelcol_exporter_send_failed_spans / otelcol_exporter_queue_size).
sending_queue: {enabled: true, num_consumers: 2, queue_size: 500}
retry_on_failure: {enabled: true, initial_interval: 1s, max_interval: 10s, max_elapsed_time: 30s}
otlp_http/loki:
endpoint: http://loki:3100/otlp
sending_queue: {enabled: true, num_consumers: 2, queue_size: 500}
retry_on_failure: {enabled: true, initial_interval: 1s, max_interval: 10s, max_elapsed_time: 30s}
prometheus:
endpoint: 0.0.0.0:8889
resource_to_telemetry_conversion: {enabled: true}
metric_expiration: 20s
service:
telemetry:
metrics:
level: normal
readers:
- pull:
exporter:
prometheus: {host: 0.0.0.0, port: 8888}
logs:
level: warn
pipelines:
traces:
receivers: [otlp]
processors: [memory_limiter, attributes/scrub, batch]
exporters: [otlp_grpc/tempo]
metrics:
receivers: [otlp]
processors: [memory_limiter, batch]
exporters: [prometheus]
logs:
receivers: [otlp]
processors: [memory_limiter, attributes/scrub, batch]
exporters: [otlp_http/loki]
+20
View File
@@ -0,0 +1,20 @@
# Prometheus: хранилище метрик (retention 7 дней, volume promdata). Забирает метрики приложений через Collector
# (Prometheus-экспортёр :8889) и собственные метрики Collector (:8888): очереди экспортёров, отказы, drop из memory_limiter.
global:
scrape_interval: 5s
evaluation_interval: 5s
rule_files:
- /etc/prometheus/rules/*.yml
scrape_configs:
- job_name: otel-collector-app-metrics
honor_labels: true
static_configs:
- targets: ["otel-collector:8889"]
- job_name: otel-collector-internal
static_configs:
- targets: ["otel-collector:8888"]
- job_name: prometheus
static_configs:
- targets: ["localhost:9090"]
+77
View File
@@ -0,0 +1,77 @@
# Правила записи (SLI) и алерты. Runbook для каждого алерта — docs/runbooks.md (ссылка в аннотации).
# SLI доступности API: доля бизнес-запросов (без /health) без ответа 5xx. 4xx — ошибка клиента, в SLI не входит.
# SLO: 99.5 % за скользящее окно 30 дней => бюджет ошибок 0.5 %. Для демонстрации окна короткие (локальный стенд).
groups:
- name: messenger-sli
interval: 5s
rules:
- record: sli:http_requests:rate1m
expr: sum(rate(http_server_request_duration_seconds_count{http_route!~"/health.*"}[1m]))
- record: sli:http_errors:rate1m
expr: sum(rate(http_server_request_duration_seconds_count{http_route!~"/health.*",http_response_status_code=~"5.."}[1m]))
- record: sli:http_error_ratio:1m
expr: (sli:http_errors:rate1m / sli:http_requests:rate1m) or vector(0)
- record: sli:http_error_ratio:5m
expr: |
(sum(rate(http_server_request_duration_seconds_count{http_route!~"/health.*",http_response_status_code=~"5.."}[5m]))
/ sum(rate(http_server_request_duration_seconds_count{http_route!~"/health.*"}[5m]))) or vector(0)
- record: sli:http_latency_p95:5m
expr: histogram_quantile(0.95, sum by (le) (rate(http_server_request_duration_seconds_bucket{http_route!~"/health.*"}[5m])))
- record: sli:http_latency_p95:1m
expr: histogram_quantile(0.95, sum by (le) (rate(http_server_request_duration_seconds_bucket{http_route!~"/health.*"}[1m])))
- record: sli:upload_error_ratio:1m
expr: |
(sum(rate(http_server_request_duration_seconds_count{http_route="/api/v1/chats/{chat_id}/attachments",http_response_status_code=~"5.."}[1m]))
/ sum(rate(http_server_request_duration_seconds_count{http_route="/api/v1/chats/{chat_id}/attachments"}[1m]))) or vector(0)
- record: sli:upload_requests:rate1m
expr: sum(rate(http_server_request_duration_seconds_count{http_route="/api/v1/chats/{chat_id}/attachments"}[1m]))
- record: sli:time_to_ready_p95:5m
expr: histogram_quantile(0.95, sum by (le) (rate(messenger_job_time_to_ready_seconds_bucket[5m])))
- name: messenger-alerts
rules:
- alert: HighHttpErrorRate
expr: sli:http_error_ratio:1m > 0.05 and sli:http_requests:rate1m > 0.5
for: 20s
labels: {severity: page}
annotations:
summary: "Доля 5xx бизнес-запросов API выше 5 %"
description: "Числитель — ответы 5xx, знаменатель — все запросы кроме /health. Значение: {{ $value | humanizePercentage }}."
runbook: "docs/runbooks.md#higherrorrate"
- alert: HighUploadErrorRate
expr: sli:upload_error_ratio:1m > 0.2 and sli:upload_requests:rate1m > 0.2
for: 15s
labels: {severity: page}
annotations:
summary: "Более 20 % загрузок вложений завершаются 5xx"
description: "Маршрут загрузки критичен, но малообъёмен: общий SLI 5xx его деградацию не заметит, поэтому есть отдельный алерт. Значение: {{ $value | humanizePercentage }}."
runbook: "docs/runbooks.md#higherrorrate"
- alert: HighLatencyP95
expr: sli:http_latency_p95:1m > 0.5 and sli:http_requests:rate1m > 0.5
for: 20s
labels: {severity: ticket}
annotations:
summary: "p95 задержки бизнес-запросов выше 500 мс"
description: "p95 из суммарной гистограммы всех реплик: {{ $value | humanizeDuration }}. Различайте медленную БД (spans db.connection, db_client_connections_waiting) и медленный обработчик."
runbook: "docs/runbooks.md#highlatency"
- alert: QueueLagGrowing
expr: messenger_queue_oldest_job_age_seconds > 30
for: 20s
labels: {severity: ticket}
annotations:
summary: "Самая старая задача в очереди ждёт дольше 30 с"
description: "Возраст старейшего задания: {{ $value }} с. Проверьте worker, брокер, S3."
runbook: "docs/runbooks.md#queuelag"
- alert: WorkerSilent
expr: absent(messenger_worker_capacity)
for: 30s
labels: {severity: ticket}
annotations:
summary: "Worker не присылает метрики"
runbook: "docs/runbooks.md#queuelag"
- alert: TelemetryExportFailing
expr: sum(rate(otelcol_exporter_send_failed_spans[1m])) + sum(rate(otelcol_exporter_send_failed_log_records[1m])) > 0
for: 15s
labels: {severity: ticket}
annotations:
summary: "Collector не может отправить телеметрию в хранилище"
runbook: "docs/runbooks.md#telemetry"
+33
View File
@@ -0,0 +1,33 @@
# Tempo: хранилище трасс (локальный диск, volume tempodata, retention 7 суток). Принимает OTLP/gRPC от Collector.
stream_over_http_enabled: true
server:
http_listen_port: 3200
log_level: warn
distributor:
receivers:
otlp:
protocols:
grpc:
endpoint: 0.0.0.0:4317
ingester:
max_block_duration: 1m
compactor:
compaction:
block_retention: 168h
storage:
trace:
backend: local
wal:
path: /var/tempo/wal
local:
path: /var/tempo/blocks
# Лимиты приёма: значения по умолчанию (10 000 «живых» трасс на тенанта) отбрасывали до 30 % спанов под нагрузкой
# (метрика tempo_discarded_spans_total{reason="live_traces_exceeded"}) — найдено экспериментом «цена наблюдаемости».
overrides:
defaults:
ingestion:
max_traces_per_user: 150000
rate_limit_bytes: 50000000
burst_size_bytes: 100000000
usage_report:
reporting_enabled: false
+52
View File
@@ -0,0 +1,52 @@
-- 0001: пользователи, чаты, участники, сообщения, сессии.
-- Источник истины для всех данных — PostgreSQL. Каждая миграция выполняется в транзакции.
CREATE TABLE users (
id uuid PRIMARY KEY,
username text NOT NULL,
display_name text NOT NULL,
password_hash text NOT NULL,
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
CONSTRAINT users_username_format CHECK (username ~ '^[a-z0-9_]{3,32}$'),
CONSTRAINT users_display_name_len CHECK (char_length(display_name) BETWEEN 1 AND 100)
);
-- Уникальность username: гонка двух регистраций решается ограничением БД, а не проверкой в приложении.
CREATE UNIQUE INDEX users_username_key ON users (username);
CREATE TABLE chats (
id uuid PRIMARY KEY,
title text NOT NULL,
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
CONSTRAINT chats_title_len CHECK (char_length(title) BETWEEN 1 AND 100)
);
-- chat_created_at денормализован ради keyset-пагинации списка чатов пользователя без join+sort.
CREATE TABLE chat_members (
chat_id uuid NOT NULL REFERENCES chats (id) ON DELETE CASCADE,
user_id uuid NOT NULL REFERENCES users (id),
chat_created_at timestamptz NOT NULL,
ord smallint NOT NULL,
PRIMARY KEY (chat_id, user_id)
);
CREATE INDEX chat_members_user_chats ON chat_members (user_id, chat_created_at, chat_id);
CREATE TABLE messages (
id uuid PRIMARY KEY,
chat_id uuid NOT NULL REFERENCES chats (id) ON DELETE CASCADE,
sender_id uuid NOT NULL REFERENCES users (id),
text text NOT NULL,
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
CONSTRAINT messages_text_len CHECK (char_length(text) <= 4000)
);
-- История чата: WHERE chat_id = ? AND (created_at, id) > (?, ?) ORDER BY created_at, id LIMIT n
-- обслуживается этим индексом без сортировки и без сканирования всей истории.
CREATE INDEX messages_chat_history ON messages (chat_id, created_at, id);
-- Серверные сессии. В БД хранится только SHA-256 токена: утечка БД не раскрывает рабочие токены.
CREATE TABLE sessions (
token_hash bytea PRIMARY KEY,
user_id uuid NOT NULL REFERENCES users (id),
created_at timestamptz NOT NULL DEFAULT now(),
expires_at timestamptz NOT NULL,
revoked_at timestamptz
);
CREATE INDEX sessions_expires_at ON sessions (expires_at);
+17
View File
@@ -0,0 +1,17 @@
-- 0002: общая идемпотентность отправки сообщений (lab3).
-- Область уникальности: пользователь + чат + ключ (метод и путь фиксированы: POST /chats/{id}/messages).
-- Запись ключа и сообщения — одна транзакция: побочный эффект и результат фиксируются атомарно.
CREATE TABLE idempotency_keys (
user_id uuid NOT NULL REFERENCES users (id),
chat_id uuid NOT NULL REFERENCES chats (id) ON DELETE CASCADE,
idem_key text NOT NULL,
request_hash bytea NOT NULL,
message_id uuid NOT NULL,
created_at timestamptz NOT NULL DEFAULT now(),
PRIMARY KEY (user_id, chat_id, idem_key),
CONSTRAINT idem_key_format CHECK (idem_key ~ '^[A-Za-z0-9._:-]{1,128}$'),
-- сообщение вставляется после ключа в той же транзакции -> проверка внешнего ключа откладывается до COMMIT
CONSTRAINT idem_message_fk FOREIGN KEY (message_id) REFERENCES messages (id) ON DELETE CASCADE
DEFERRABLE INITIALLY DEFERRED
);
CREATE INDEX idempotency_keys_created_at ON idempotency_keys (created_at);
+35
View File
@@ -0,0 +1,35 @@
-- 0003: отправка сообщения одной серверной функцией (оптимизация по итогам трека 1: меньше обращений к БД на запрос).
-- Семантика идентична версии «несколько запросов в транзакции» (migrations 0002 + код приложения):
-- проверка членства -> просроченный ключ удаляется -> INSERT ключа (ON CONFLICT DO NOTHING) -> INSERT сообщения.
-- Функция выполняется атомарно (один вызов = одна транзакция); гонку одинаковых ключей по-прежнему решает уникальный индекс.
CREATE FUNCTION send_message(p_chat uuid, p_user uuid, p_key text, p_hash bytea, p_text text, p_ttl double precision)
RETURNS TABLE (outcome text, msg_id uuid, msg_chat uuid, msg_sender uuid, msg_text text, msg_created timestamptz,
request_hash bytea)
LANGUAGE plpgsql AS $$
DECLARE
v_new uuid := gen_random_uuid();
v_ins uuid;
v_created timestamptz;
BEGIN
IF NOT EXISTS (SELECT 1 FROM chat_members m WHERE m.chat_id = p_chat AND m.user_id = p_user) THEN
RETURN QUERY SELECT 'not_member'::text, NULL::uuid, NULL::uuid, NULL::uuid, NULL::text, NULL::timestamptz, NULL::bytea;
RETURN;
END IF;
DELETE FROM idempotency_keys k
WHERE k.user_id = p_user AND k.chat_id = p_chat AND k.idem_key = p_key
AND k.created_at < now() - make_interval(secs => p_ttl);
INSERT INTO idempotency_keys (user_id, chat_id, idem_key, request_hash, message_id)
VALUES (p_user, p_chat, p_key, p_hash, v_new)
ON CONFLICT DO NOTHING
RETURNING message_id INTO v_ins;
IF v_ins IS NOT NULL THEN
INSERT INTO messages (id, chat_id, sender_id, text) VALUES (v_new, p_chat, p_user, p_text)
RETURNING created_at INTO v_created;
RETURN QUERY SELECT 'created'::text, v_new, p_chat, p_user, p_text, v_created, p_hash;
ELSE
RETURN QUERY SELECT 'replay'::text, m.id, m.chat_id, m.sender_id, m.text, m.created_at, k.request_hash
FROM idempotency_keys k JOIN messages m ON m.id = k.message_id
WHERE k.user_id = p_user AND k.chat_id = p_chat AND k.idem_key = p_key;
END IF;
END;
$$;
+90
View File
@@ -0,0 +1,90 @@
-- 0004 (lab4): метаданные вложений в PostgreSQL, байты — в приватном S3-бакете.
-- Ключ объекта назначает сервер (att/<id>/original); filename — только отображаемое имя.
CREATE TABLE attachments (
id uuid PRIMARY KEY,
chat_id uuid NOT NULL REFERENCES chats (id) ON DELETE CASCADE,
owner_id uuid NOT NULL REFERENCES users (id),
filename text NOT NULL,
content_type text NOT NULL,
size_bytes bigint NOT NULL,
sha256 text NOT NULL,
width integer NOT NULL,
height integer NOT NULL,
status text NOT NULL DEFAULT 'ready',
error_code text,
s3_key text NOT NULL,
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
updated_at timestamptz NOT NULL DEFAULT clock_timestamp(),
CONSTRAINT attachments_content_type CHECK (content_type IN ('image/png', 'image/jpeg')),
CONSTRAINT attachments_size CHECK (size_bytes BETWEEN 1 AND 10485760),
CONSTRAINT attachments_sha256 CHECK (sha256 ~ '^[a-f0-9]{64}$'),
CONSTRAINT attachments_status CHECK (status IN ('queued', 'processing', 'ready', 'failed')),
CONSTRAINT attachments_filename_len CHECK (char_length(filename) BETWEEN 1 AND 255)
);
CREATE UNIQUE INDEX attachments_s3_key_key ON attachments (s3_key);
CREATE INDEX attachments_chat ON attachments (chat_id);
-- Связь сообщение <-> вложения (порядок сохраняется). Вложение может быть привязано к нескольким сообщениям чата.
CREATE TABLE message_attachments (
message_id uuid NOT NULL REFERENCES messages (id) ON DELETE CASCADE,
attachment_id uuid NOT NULL REFERENCES attachments (id),
position smallint NOT NULL,
PRIMARY KEY (message_id, attachment_id)
);
CREATE INDEX message_attachments_attachment ON message_attachments (attachment_id);
-- Отправка сообщения с вложениями: заменяет функцию из 0003 (добавлен параметр p_atts и результат msg_atts).
DROP FUNCTION send_message(uuid, uuid, text, bytea, text, double precision);
CREATE FUNCTION send_message(p_chat uuid, p_user uuid, p_key text, p_hash bytea, p_text text, p_ttl double precision,
p_atts uuid[])
RETURNS TABLE (outcome text, msg_id uuid, msg_chat uuid, msg_sender uuid, msg_text text, msg_created timestamptz,
request_hash bytea, msg_atts uuid[])
LANGUAGE plpgsql AS $$
DECLARE
v_new uuid := gen_random_uuid();
v_ins uuid;
v_created timestamptz;
v_found int;
v_failed int;
BEGIN
IF NOT EXISTS (SELECT 1 FROM chat_members m WHERE m.chat_id = p_chat AND m.user_id = p_user) THEN
RETURN QUERY SELECT 'not_member'::text, NULL::uuid, NULL::uuid, NULL::uuid, NULL::text, NULL::timestamptz,
NULL::bytea, NULL::uuid[];
RETURN;
END IF;
IF cardinality(p_atts) > 0 THEN
SELECT count(*), count(*) FILTER (WHERE a.status = 'failed') INTO v_found, v_failed
FROM attachments a WHERE a.id = ANY (p_atts) AND a.chat_id = p_chat;
IF v_found <> cardinality(p_atts) THEN
RETURN QUERY SELECT 'attachment_not_found'::text, NULL::uuid, NULL::uuid, NULL::uuid, NULL::text,
NULL::timestamptz, NULL::bytea, NULL::uuid[];
RETURN;
END IF;
IF v_failed > 0 THEN
RETURN QUERY SELECT 'attachment_failed'::text, NULL::uuid, NULL::uuid, NULL::uuid, NULL::text,
NULL::timestamptz, NULL::bytea, NULL::uuid[];
RETURN;
END IF;
END IF;
DELETE FROM idempotency_keys k
WHERE k.user_id = p_user AND k.chat_id = p_chat AND k.idem_key = p_key
AND k.created_at < now() - make_interval(secs => p_ttl);
INSERT INTO idempotency_keys (user_id, chat_id, idem_key, request_hash, message_id)
VALUES (p_user, p_chat, p_key, p_hash, v_new)
ON CONFLICT DO NOTHING
RETURNING message_id INTO v_ins;
IF v_ins IS NOT NULL THEN
INSERT INTO messages (id, chat_id, sender_id, text) VALUES (v_new, p_chat, p_user, p_text)
RETURNING created_at INTO v_created;
INSERT INTO message_attachments (message_id, attachment_id, position)
SELECT v_new, a, (o - 1)::smallint FROM unnest(p_atts) WITH ORDINALITY AS t(a, o);
RETURN QUERY SELECT 'created'::text, v_new, p_chat, p_user, p_text, v_created, p_hash, p_atts;
ELSE
RETURN QUERY SELECT 'replay'::text, m.id, m.chat_id, m.sender_id, m.text, m.created_at, k.request_hash,
COALESCE((SELECT array_agg(ma.attachment_id ORDER BY ma.position)
FROM message_attachments ma WHERE ma.message_id = m.id), '{}'::uuid[])
FROM idempotency_keys k JOIN messages m ON m.id = k.message_id
WHERE k.user_id = p_user AND k.chat_id = p_chat AND k.idem_key = p_key;
END IF;
END;
$$;
+60
View File
@@ -0,0 +1,60 @@
-- 0005 (lab5): асинхронная обработка вложений — состояния, варианты, outbox, журнал событий.
-- Расширение обратно совместимо с lab4: старые строки (ready) остаются валидными.
ALTER TABLE attachments
ADD COLUMN staging_key text,
ADD COLUMN pipeline_version integer NOT NULL DEFAULT 1,
ADD COLUMN attempt integer NOT NULL DEFAULT 0,
ADD COLUMN job_id uuid,
ADD COLUMN lease_owner text,
ADD COLUMN lease_expires_at timestamptz,
ADD COLUMN processing_started_at timestamptz,
ADD COLUMN accepted_at timestamptz NOT NULL DEFAULT clock_timestamp(),
ADD COLUMN ready_at timestamptz;
-- очередь/backlog и поиск зависших processing без полного скана
CREATE INDEX attachments_active ON attachments (status, accepted_at) WHERE status IN ('queued', 'processing');
CREATE INDEX attachments_lease ON attachments (lease_expires_at) WHERE status = 'processing';
-- Результаты обработки. Ключ эффекта (attachment_id, pipeline_version, name): повторная запись не создаёт второй набор вариантов.
CREATE TABLE attachment_variants (
attachment_id uuid NOT NULL REFERENCES attachments (id) ON DELETE CASCADE,
pipeline_version integer NOT NULL,
name text NOT NULL,
content_type text NOT NULL,
size_bytes bigint NOT NULL,
width integer NOT NULL,
height integer NOT NULL,
sha256 text NOT NULL,
s3_key text NOT NULL,
created_at timestamptz NOT NULL DEFAULT now(),
PRIMARY KEY (attachment_id, pipeline_version, name),
CONSTRAINT variants_name CHECK (name IN ('thumbnail', 'crop', 'optimized')),
CONSTRAINT variants_content_type CHECK (content_type IN ('image/jpeg', 'image/png', 'image/webp'))
);
-- Transactional outbox: намерение «обработать файл» фиксируется В ТОЙ ЖЕ транзакции, что и строка вложения.
-- Relay публикует строки в брокер (publisher confirm) и только затем ставит published_at. available_at — отложенная
-- публикация (backoff между попытками). Доставка at-least-once; обработка идемпотентна.
CREATE TABLE outbox (
id bigserial PRIMARY KEY,
job_id uuid NOT NULL,
attachment_id uuid NOT NULL REFERENCES attachments (id) ON DELETE CASCADE,
pipeline_version integer NOT NULL DEFAULT 1,
attempt integer NOT NULL,
available_at timestamptz NOT NULL DEFAULT now(),
published_at timestamptz,
created_at timestamptz NOT NULL DEFAULT now()
);
CREATE INDEX outbox_pending ON outbox (available_at) WHERE published_at IS NULL;
-- Журнал событий пайплайна (аудит операторского replay, причины отказов; без секретов).
CREATE TABLE job_events (
id bigserial PRIMARY KEY,
attachment_id uuid NOT NULL,
job_id uuid,
event text NOT NULL,
attempt integer,
detail text,
created_at timestamptz NOT NULL DEFAULT now()
);
CREATE INDEX job_events_attachment ON job_events (attachment_id, id);
+3
View File
@@ -0,0 +1,3 @@
-- 0006 (lab6): контекст трассировки в outbox. traceparent/tracestate запроса upload сохраняются вместе с заданием,
-- relay передаёт их в брокер (поля jobs.schema.json), worker продолжает ту же трассу: API -> outbox -> broker -> worker.
ALTER TABLE outbox ADD COLUMN traceparent text, ADD COLUMN tracestate text;
+43
View File
@@ -0,0 +1,43 @@
aio-pika==10.1.0
aiobotocore==3.9.2
aiohappyeyeballs==2.7.1
aiohttp==3.14.3
aioitertools==0.13.0
aiormq==7.2.1
aiosignal==1.4.0
argon2-cffi-bindings==26.1.0
argon2-cffi==23.1.0
asyncpg==0.30.0
attrs==26.1.0
botocore==1.43.106
cffi==2.1.1
click==8.5.0
frozenlist==1.8.0
googleapis-common-protos==1.75.5
h11==0.16.0
httptools==0.8.0
idna==3.20
jmespath==1.1.0
multidict==6.9.1
opentelemetry-api==1.45.0
opentelemetry-exporter-http-transport==0.66b0
opentelemetry-exporter-otlp-common==0.66b0
opentelemetry-exporter-otlp-proto-common==1.45.0
opentelemetry-exporter-otlp-proto-http==1.45.0
opentelemetry-proto==1.45.0
opentelemetry-sdk==1.45.0
opentelemetry-semantic-conventions==0.66b0
pamqp==4.0.1
pillow==12.3.0
propcache==0.5.4
protobuf==7.36.2
pycparser==3.0
python-dateutil==2.9.0.post0
redis==5.2.1
six==1.17.0
typing_extensions==4.16.0
urllib3==2.8.0
uvicorn==0.34.0
uvloop==0.21.0; sys_platform != "win32"
wrapt==2.5.0
yarl==1.25.1
+113
View File
@@ -0,0 +1,113 @@
#!/usr/bin/env python3
"""Уровень 4: последовательная остановка каждой API-реплики под нагрузкой (с восстановлением предыдущей).
Сценарий (стенд `docker compose up -d` с 2 репликами; нагрузка идёт через LB непрерывно ~70 с):
t=10 остановить реплику A (docker stop --time 0 = мгновенный отказ процесса)
t=25 восстановить (scripts/restore-replicas.sh)
t=40 остановить реплику B
t=55 восстановить
Для каждого отказа считаем: RPS до/во время/после, число ошибочных попыток, время до прекращения ошибок
(= обнаружение отказа балансировщиком), сохранность подтверждённых сообщений и отсутствие дублей.
python research/failover_sequence.py --out evidence
"""
import argparse
import asyncio
import json
import os
import subprocess
import sys
import threading
import time
sys.path.insert(0, os.path.dirname(__file__))
import loadgen # noqa: E402
ROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..")
BASE = "http://127.0.0.1:8080"
def sh(*args):
return subprocess.run(args, cwd=ROOT, capture_output=True, text=True).stdout.strip()
def api_ids():
return sh("docker", "ps", "-q", "--filter", "label=com.docker.compose.project=mtusi-lab6",
"--filter", "label=com.docker.compose.service=api").split()
def scenario(rec, plan):
"""Выполняется в отдельном потоке: действия по расписанию относительно rec.t0."""
for at, action in plan:
time.sleep(max(0, rec.t0 + at - time.time()))
t = time.time() - rec.t0
if action == "stop":
target = api_ids()[0]
subprocess.run(["docker", "stop", "--time", "0", target], capture_output=True)
rec.marks.append({"t": round(t, 2), "action": "stop", "container": target})
else:
subprocess.run(["sh", "scripts/restore-replicas.sh"], cwd=ROOT, capture_output=True)
rec.marks.append({"t": round(t, 2), "action": "restore"})
async def main(args):
stand = loadgen.Stand(BASE)
await stand.setup()
rec = loadgen.Recorder()
rec.marks = []
plan = [(10, "stop"), (25, "restore"), (40, "stop"), (55, "restore")]
threading.Thread(target=scenario, args=(rec, plan), daemon=True).start()
await loadgen.run_load(stand, rec, args.seconds, args.concurrency)
await asyncio.sleep(2)
check = await loadgen.verify(stand, rec)
stops = [m for m in rec.marks if m["action"] == "stop"]
restores = [m for m in rec.marks if m["action"] == "restore"]
windows = []
for i, stop in enumerate(stops):
t0 = stop["t"]
errs = [e for e in rec.events if e[1] == "POST" and e[2] != 201 and t0 - 0.2 <= e[0] <= t0 + 10]
last_err = max((e[0] for e in errs), default=t0)
restore_t = restores[i]["t"] if i < len(restores) else t0 + 15
windows.append({"stopped_container": stop["container"], "stop_at_s": t0,
"rps_before": loadgen.summarize(rec, t0 - 8, t0 - 0.5).get("ok_rps"),
"rps_one_replica": loadgen.summarize(rec, t0 + 5, restore_t - 1).get("ok_rps"),
"rps_after_restore": loadgen.summarize(rec, restore_t + 8, restore_t + 14).get("ok_rps"),
"failed_attempts_within_10s": len(errs),
"error_statuses": sorted({str(e[2]) for e in errs}),
"seconds_until_errors_stop": round(last_err - t0, 2) if errs else 0.0,
"p95_ms_one_replica": loadgen.summarize(rec, t0 + 5, restore_t - 1).get("p95_ms")})
series = loadgen.per_second(rec)
report = {"date": time.strftime("%Y-%m-%d %H:%M:%S"), "args": vars(args), "marks": rec.marks, "windows": windows,
"total": loadgen.summarize(rec), "verification": check,
"invariants_ok": check["confirmed_missing_in_history"] == 0 and check["duplicate_texts_in_history"] == 0,
"timeline": [{"t": t, "ok": b["ok"], "err": b["err"], "p95_ms": loadgen.pct(b["lat"], .95), "instances": len(b["instances"])}
for t, b in sorted(series.items())]}
with open(os.path.join(args.out, "failover_sequence.json"), "w", encoding="utf-8") as fh:
json.dump(report, fh, ensure_ascii=False, indent=2)
plot(report, os.path.join(args.out, "failover_sequence.png"))
print(json.dumps({k: v for k, v in report.items() if k != "timeline"}, ensure_ascii=False, indent=2))
def plot(report, path):
import matplotlib
matplotlib.use("Agg")
import matplotlib.pyplot as plt
tl = report["timeline"]
t = [x["t"] for x in tl]
fig, ax = plt.subplots(3, 1, figsize=(11, 8.5), sharex=True)
ax[0].plot(t, [x["ok"] for x in tl], color="#2e7d32"); ax[0].set_ylabel("успешных POST/с"); ax[0].set_title("Пропускная способность")
ax[1].bar(t, [x["err"] for x in tl], color="#c62828"); ax[1].set_ylabel("ошибочных попыток/с"); ax[1].set_title("Ошибки (до повтора с тем же ключом)")
ax[2].plot(t, [x["p95_ms"] or 0 for x in tl], color="#1565c0"); ax[2].set_ylabel("p95, мс"); ax[2].set_xlabel("время, с"); ax[2].set_title("Латентность успешных POST")
for a in ax:
for m in report["marks"]:
a.axvline(m["t"], color="red" if m["action"] == "stop" else "green", ls="--", lw=1)
ax[0].text(0.01, 0.05, "красная линия — остановка реплики, зелёная — восстановление", transform=ax[0].transAxes, fontsize=9)
fig.tight_layout(); fig.savefig(path, dpi=110)
if __name__ == "__main__":
ap = argparse.ArgumentParser()
ap.add_argument("--out", default="evidence")
ap.add_argument("--seconds", type=int, default=70)
ap.add_argument("--concurrency", type=int, default=16)
asyncio.run(main(ap.parse_args()))
+258
View File
@@ -0,0 +1,258 @@
#!/usr/bin/env python3
"""Уровень 4: контролируемые инциденты и проверка алертов firing -> resolved.
Фоном идёт нагрузка (сообщения + загрузки). Инциденты и ожидаемые алерты:
1. worker_down — остановка worker: растёт возраст старейшей задачи -> QueueLagGrowing; после запуска — resolved.
2. slow_db — триггер в БД добавляет pg_sleep(0.8) на вставку сообщения: растёт p95 -> HighLatencyP95;
причина отличается от медленного обработчика по spans db.connection и метрике ожидания пула.
3. s3_down — остановка S3: загрузки дают 503 -> HighUploadErrorRate (маршрут критичен, но малообъёмен).
4. telemetry_backend — остановка Tempo и Loki: API продолжает отвечать, Collector ограничивает очередь/RAM -> TelemetryExportFailing.
Для каждого фиксируем времена перехода алертов (по Prometheus API), ключевые метрики и «улики» из Tempo/Loki.
python research/incidents.py --out evidence
"""
import argparse
import asyncio
import base64
import json
import os
import subprocess
import sys
import threading
import time
import urllib.parse
import uuid
import aiohttp
sys.path.insert(0, os.path.dirname(__file__))
ROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..")
BASE = "http://127.0.0.1:8080"
def sh(*args, stdin=None):
return subprocess.run(args, cwd=ROOT, capture_output=True, text=True, encoding="utf-8", input=stdin).stdout.strip()
def dc(*args):
return sh("docker", "compose", "--env-file", ".env", *args)
def prom_get(path):
return json.loads(dc("exec", "-T", "prometheus", "wget", "-q", "-O", "-", "http://127.0.0.1:9090" + path) or "{}")
def prom_value(query):
r = prom_get("/api/v1/query?query=" + urllib.parse.quote(query))
res = r.get("data", {}).get("result", [])
return float(res[0]["value"][1]) if res else None
def alerts():
r = prom_get("/api/v1/alerts")
return {a["labels"]["alertname"]: a["state"] for a in r.get("data", {}).get("alerts", [])}
def net_get(url, accept=None):
cmd = ["exec", "-T", "prometheus", "wget", "-q", "-O", "-"] + (["--header", "Accept: " + accept] if accept else []) + [url]
return dc(*cmd)
class Recorder(threading.Thread):
def __init__(self):
super().__init__(daemon=True)
self.t0, self.rows, self.stop = time.time(), [], threading.Event()
def run(self):
while not self.stop.is_set():
t = time.time() - self.t0
row = {"t": round(t, 1), "alerts": alerts(),
"queue_oldest_age_s": prom_value("max(messenger_queue_oldest_job_age_seconds)"),
"error_ratio_1m": prom_value("sli:http_error_ratio:1m"),
"p95_s": prom_value("sli:http_latency_p95:1m"),
"db_waiting": prom_value("sum(db_client_connections_waiting)"),
"db_used": prom_value('sum(db_client_connections_usage{state="used"})'),
"rps": prom_value("sli:http_requests:rate1m")}
self.rows.append(row)
self.stop.wait(3)
async def register(s):
name = "inc_%s" % uuid.uuid4().hex[:12]
pw = "incident-test-pw-" + uuid.uuid4().hex[:6]
async with s.post(BASE + "/api/v1/users", json={"username": name, "display_name": "i", "password": pw}) as r:
user = await r.json()
cred = base64.b64encode(("%s:%s" % (name, pw)).encode()).decode()
async with s.post(BASE + "/api/v1/auth/sessions", headers={"Authorization": "Basic " + cred}) as r:
token = (await r.json())["token"]
async with s.post(BASE + "/api/v1/chats", json={"title": "inc", "member_ids": [user["id"]]}, headers={"Authorization": "Bearer " + token}) as r:
chat = await r.json()
return token, chat["id"]
async def traffic(stop, png, counters):
async with aiohttp.ClientSession(timeout=aiohttp.ClientTimeout(total=15), connector=aiohttp.TCPConnector(limit=40)) as s:
token, chat = await register(s)
hdr = {"Authorization": "Bearer " + token}
async def msgs():
while not stop.is_set():
try:
async with s.post("%s/api/v1/chats/%s/messages" % (BASE, chat), json={"text": "inc-" + uuid.uuid4().hex[:8]},
headers=dict(hdr, **{"Idempotency-Key": uuid.uuid4().hex})) as r:
await r.read()
counters["msg_%d" % (r.status // 100)] = counters.get("msg_%d" % (r.status // 100), 0) + 1
except Exception:
counters["msg_err"] = counters.get("msg_err", 0) + 1
await asyncio.sleep(0.1)
async def uploads():
while not stop.is_set():
try:
async with s.post("%s/api/v1/chats/%s/attachments?filename=i.png" % (BASE, chat), data=png,
headers=dict(hdr, **{"Content-Type": "image/png"})) as r:
await r.read()
counters["up_%d" % (r.status // 100)] = counters.get("up_%d" % (r.status // 100), 0) + 1
except Exception:
counters["up_err"] = counters.get("up_err", 0) + 1
await asyncio.sleep(0.3)
await asyncio.gather(*([msgs() for _ in range(4)] + [uploads() for _ in range(10)]))
def tempo_search(traceql, limit=20):
now = int(time.time())
url = "http://tempo:3200/api/search?q=%s&limit=%d&start=%d&end=%d" % (urllib.parse.quote(traceql), limit, now - 900, now + 5)
raw = net_get(url)
try:
return json.loads(raw).get("traces", [])
except ValueError:
return []
def loki_count(query):
now = int(time.time() * 1e9)
url = "http://loki:3100/loki/api/v1/query_range?limit=500&start=%d&end=%d&query=%s" % (now - 900 * 10 ** 9, now, urllib.parse.quote(query))
try:
data = json.loads(net_get(url))
return sum(len(s["values"]) for s in data["data"]["result"])
except ValueError:
return None
def wait_until(pred, seconds):
deadline = time.time() + seconds
while time.time() < deadline:
if pred():
return True
time.sleep(2)
return False
def run_incident(name, expected_alert, start, stop, hold, evidence, rec):
t_start = time.time() - rec.t0
start()
fired = wait_until(lambda: alerts().get(expected_alert) == "firing", hold)
t_fired = time.time() - rec.t0 if fired else None
ev = evidence() if evidence else {}
ev["alerts_in_state_firing"] = sorted(k for k, v in alerts().items() if v == "firing")
stop()
t_recovery = time.time() - rec.t0
resolved = wait_until(lambda: expected_alert not in alerts(), 120)
t_resolved = time.time() - rec.t0 if resolved else None
result = {"incident": name, "alert": expected_alert, "injected_at_s": round(t_start, 1),
"firing_at_s": round(t_fired, 1) if t_fired else None, "detection_seconds": round(t_fired - t_start, 1) if t_fired else None,
"recovered_cmd_at_s": round(t_recovery, 1), "resolved_at_s": round(t_resolved, 1) if t_resolved else None,
"fired": fired, "resolved": resolved, "evidence": ev}
if t_fired and t_resolved:
result["time_to_resolve_after_recovery_s"] = round(t_resolved - t_recovery, 1)
time.sleep(6)
return result
def orchestrate(rec, results, args):
time.sleep(25) # прогрев: метрики и алерты стабилизируются
# 1. worker down
results.append(run_incident(
"worker_down", "QueueLagGrowing", lambda: dc("stop", "worker"), lambda: dc("start", "worker"), 150,
lambda: {"queue_oldest_age_s": prom_value("max(messenger_queue_oldest_job_age_seconds)"), "worker_capacity_series_present": prom_value("sum(messenger_worker_capacity)") is not None,
"jobs_ready_rate_1m": prom_value('sum(rate(messenger_jobs_processed_total{outcome="ready"}[1m]))'),
"diagnosis": "возраст старейшей задачи растёт при нулевой скорости обработки jobs (worker не отдаёт метрики) -> остановлен worker, а не медленная БД"},
rec))
# 2. slow DB
trigger_sql = ("CREATE OR REPLACE FUNCTION slow_insert() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN PERFORM pg_sleep(0.8); RETURN NEW; END $$;"
"CREATE TRIGGER slow_messages BEFORE INSERT ON messages FOR EACH ROW EXECUTE FUNCTION slow_insert();")
pg = ["exec", "-T", "postgres", "psql", "-U", "messenger", "-d", "messenger", "-v", "ON_ERROR_STOP=1", "-c"]
results.append(run_incident(
"slow_db", "HighLatencyP95", lambda: dc(*pg, trigger_sql), lambda: dc(*pg, "DROP TRIGGER IF EXISTS slow_messages ON messages; DROP FUNCTION IF EXISTS slow_insert();"), 120,
lambda: {"p95_s": prom_value("sli:http_latency_p95:1m"), "db_pool_waiting": prom_value("sum(db_client_connections_waiting)"),
"slow_traces_with_long_db_span": len(tempo_search('{ resource.service.name = "messenger-api" && name = "db.connection" && duration > 300ms }')),
"slow_http_post_messages_traces": len(tempo_search('{ name =~ "HTTP POST.*messages" && duration > 300ms }')),
"diagnosis": "p95 высокий, а время в спанах db.connection сопоставимо с длительностью запроса и растёт ожидание пула -> медленная БД, а не обработчик"},
rec))
# 3. S3 down
results.append(run_incident(
"s3_down", "HighUploadErrorRate", lambda: dc("stop", "seaweedfs"), lambda: dc("start", "seaweedfs"), 120,
lambda: {"upload_error_ratio_1m": prom_value("sli:upload_error_ratio:1m"), "global_error_ratio_1m": prom_value("sli:http_error_ratio:1m"), "error_traces": len(tempo_search('{ status = error && resource.service.name = "messenger-api" }')),
"s3_put_failed_log_lines": loki_count('{service_name="messenger-api"} |= "s3_put_failed"'),
"diagnosis": "рост доли 5xx на маршруте загрузки при нормальных остальных; спаны s3.put_object с ошибкой и логи s3_put_failed -> недоступен S3"},
rec))
# 4. telemetry backend down
results.append(run_incident(
"telemetry_backend", "TelemetryExportFailing", lambda: dc("stop", "tempo", "loki"), lambda: dc("start", "tempo", "loki"), 120,
lambda: {"api_error_ratio_1m": prom_value("sli:http_error_ratio:1m"), "api_rps": prom_value("sli:http_requests:rate1m"),
"collector_queue_size": prom_value("max(otelcol_exporter_queue_size)"), "collector_send_failed_spans_rate": prom_value("sum(rate(otelcol_exporter_send_failed_spans[1m]))"),
"collector_rss_mib": round((prom_value("max(otelcol_process_memory_rss)") or 0) / 1048576, 1),
"diagnosis": "ошибки экспорта растут, очередь ограничена; бизнес-метрики API без роста ошибок -> сбой хранилища телеметрии, не приложения"},
rec))
async def main(args):
from PIL import Image
import io
buf = io.BytesIO()
Image.new("RGB", (320, 200), (30, 90, 160)).save(buf, "PNG")
stop_flag, counters, results = asyncio.Event(), {}, []
rec = Recorder()
rec.start()
t = threading.Thread(target=orchestrate, args=(rec, results, args), daemon=True)
t.start()
traffic_task = asyncio.create_task(traffic(stop_flag, buf.getvalue(), counters))
while t.is_alive():
await asyncio.sleep(1)
stop_flag.set()
await asyncio.sleep(1)
traffic_task.cancel()
rec.stop.set()
collector_rss = None
report = {"date": time.strftime("%Y-%m-%d %H:%M:%S"), "traffic_counters": counters, "incidents": results, "timeline": rec.rows}
with open(os.path.join(args.out, "incidents.json"), "w", encoding="utf-8") as fh:
json.dump(report, fh, ensure_ascii=False, indent=2)
plot(report, os.path.join(args.out, "incidents.png"))
print(json.dumps({"incidents": results, "traffic": counters}, ensure_ascii=False, indent=2))
def plot(report, path):
import matplotlib
matplotlib.use("Agg")
import matplotlib.pyplot as plt
rows = report["timeline"]
t = [r["t"] for r in rows]
fig, ax = plt.subplots(4, 1, figsize=(12, 11), sharex=True)
ax[0].plot(t, [r["queue_oldest_age_s"] or 0 for r in rows], color="#6a1b9a"); ax[0].axhline(30, color="red", ls="--"); ax[0].set_ylabel("с"); ax[0].set_title("Возраст старейшей задачи (порог алерта 30 с)")
ax[1].plot(t, [r["p95_s"] or 0 for r in rows], color="#1565c0"); ax[1].axhline(0.5, color="red", ls="--"); ax[1].set_ylabel("с"); ax[1].set_title("p95 задержки API (порог 0.5 с)")
ax[2].plot(t, [(r["error_ratio_1m"] or 0) * 100 for r in rows], color="#c62828"); ax[2].axhline(5, color="red", ls="--"); ax[2].set_ylabel("%"); ax[2].set_title("Доля 5xx (порог 5 %)")
names = ["QueueLagGrowing", "HighLatencyP95", "HighHttpErrorRate", "TelemetryExportFailing"]
for i, n in enumerate(names):
ax[3].fill_between(t, i, [i + 0.8 if r["alerts"].get(n) == "firing" else i for r in rows], step="post", alpha=0.7, label=n)
ax[3].set_yticks([i + 0.4 for i in range(4)]); ax[3].set_yticklabels(names); ax[3].set_title("Алерты в состоянии firing"); ax[3].set_xlabel("время, с")
for inc in report["incidents"]:
for a in ax:
a.axvline(inc["injected_at_s"], color="gray", ls=":")
if inc["resolved_at_s"]:
a.axvline(inc["resolved_at_s"], color="green", ls=":")
fig.tight_layout(); fig.savefig(path, dpi=105)
if __name__ == "__main__":
ap = argparse.ArgumentParser()
ap.add_argument("--out", default="evidence")
asyncio.run(main(ap.parse_args()))
+172
View File
@@ -0,0 +1,172 @@
"""Генератор нагрузки для экспериментов lab6+ (asyncio + aiohttp, closed-loop).
Каждый воркер выполняет цикл: POST сообщения с новым Idempotency-Key (90 %) или GET первой страницы истории (10 %).
При ошибке соединения/5xx POST повторяется с ТЕМ ЖЕ ключом (до RETRIES раз) — именно так клиент безопасно
повторяет неподтверждённую запись. Результат: по секундам успехи/ошибки/латентности, список подтверждённых
сообщений (для сверки с историей), число повторов.
"""
import asyncio
import base64
import random
import time
import uuid
import aiohttp
RETRIES = 3
class Stand:
"""Набор пользователей/чатов, созданных через API (через LB), для одного эксперимента."""
def __init__(self, base, users=20, chats=20):
self.base, self.n_users, self.n_chats = base, users, chats
self.users, self.chats = [], []
async def setup(self):
timeout = aiohttp.ClientTimeout(total=30)
async with aiohttp.ClientSession(timeout=timeout) as s:
for i in range(self.n_users):
name = "ld_%s" % uuid.uuid4().hex[:14]
pw = "load-test-password-" + uuid.uuid4().hex[:8]
async with s.post(self.base + "/api/v1/users", json={"username": name, "display_name": "load %d" % i, "password": pw}) as r:
user = await r.json()
assert r.status == 201, user
cred = base64.b64encode(("%s:%s" % (name, pw)).encode()).decode()
async with s.post(self.base + "/api/v1/auth/sessions", headers={"Authorization": "Basic " + cred}) as r:
sess = await r.json()
assert r.status == 201, sess
self.users.append({"id": user["id"], "token": sess["token"]})
for i in range(self.n_chats):
members = random.sample(self.users, 3)
owner = members[0]
async with s.post(self.base + "/api/v1/chats", json={"title": "load %d" % i, "member_ids": [m["id"] for m in members]},
headers={"Authorization": "Bearer " + owner["token"]}) as r:
chat = await r.json()
assert r.status == 201, chat
self.chats.append({"id": chat["id"], "members": members})
class Recorder:
def __init__(self):
self.t0 = time.time()
self.events = [] # (t, kind, status, latency_ms, instance)
self.confirmed = [] # (chat_id, message_id, text)
self.retries = 0
self.final_failures = 0
self.unconfirmed = [] # (chat_id, key, text) — финально не подтверждены (могли записаться!)
async def worker(stand, rec, deadline_fn, read_ratio=0.1, text_size=0, think=0.0):
connector = aiohttp.TCPConnector(limit=2)
timeout = aiohttp.ClientTimeout(total=5)
async with aiohttp.ClientSession(connector=connector, timeout=timeout) as s:
n = 0
while not deadline_fn():
chat = random.choice(stand.chats)
user = random.choice(chat["members"])
hdr = {"Authorization": "Bearer " + user["token"]}
if random.random() < read_ratio:
t = time.perf_counter()
try:
async with s.get("%s/api/v1/chats/%s/messages?limit=20" % (stand.base, chat["id"]), headers=hdr) as r:
await r.read()
rec.events.append((time.time() - rec.t0, "GET", r.status, (time.perf_counter() - t) * 1000, r.headers.get("X-Instance-Id")))
except Exception as exc:
rec.events.append((time.time() - rec.t0, "GET", type(exc).__name__, (time.perf_counter() - t) * 1000, None))
continue
key = uuid.uuid4().hex
text = "m-%s-%d" % (key[:8], n) + "я" * text_size
n += 1
for attempt in range(RETRIES + 1):
t = time.perf_counter()
try:
async with s.post("%s/api/v1/chats/%s/messages" % (stand.base, chat["id"]), json={"text": text},
headers=dict(hdr, **{"Idempotency-Key": key})) as r:
body = await r.json(content_type=None)
status, inst = r.status, r.headers.get("X-Instance-Id")
except Exception as exc:
status, inst, body = type(exc).__name__, None, None
rec.events.append((time.time() - rec.t0, "POST", status, (time.perf_counter() - t) * 1000, inst))
if status == 201:
rec.confirmed.append((chat["id"], body["id"], text))
break
if attempt < RETRIES and (not isinstance(status, int) or status >= 500 or status == 429):
rec.retries += 1
await asyncio.sleep(0.05 * (attempt + 1))
continue
rec.final_failures += 1
rec.unconfirmed.append((chat["id"], key, text))
break
if think:
await asyncio.sleep(think)
async def run_load(stand, rec, seconds, concurrency, **kw):
end = time.time() + seconds
await asyncio.gather(*(worker(stand, rec, lambda: time.time() > end, **kw) for _ in range(concurrency)))
async def verify(stand, rec):
"""Сверка: каждое подтверждённое сообщение читается, дублей по тексту нет, неподтверждённые встречаются не более 1 раза."""
timeout = aiohttp.ClientTimeout(total=30)
stored = {}
async with aiohttp.ClientSession(timeout=timeout) as s:
for chat in stand.chats:
hdr = {"Authorization": "Bearer " + chat["members"][0]["token"]}
cursor, items = None, []
while True:
url = "%s/api/v1/chats/%s/messages?limit=100%s" % (stand.base, chat["id"], "&cursor=" + cursor if cursor else "")
async with s.get(url, headers=hdr) as r:
page = await r.json()
assert r.status == 200, page
items += page["items"]
cursor = page["next_cursor"]
if not cursor:
break
stored[chat["id"]] = items
ids = {m["id"] for items in stored.values() for m in items}
texts = [m["text"] for items in stored.values() for m in items]
confirmed_missing = [c for c in rec.confirmed if c[1] not in ids]
dup_texts = len(texts) - len(set(texts))
unconfirmed_present = sum(1 for c in rec.unconfirmed if c[2] in set(texts))
return {"stored_messages": len(ids), "confirmed_201": len(rec.confirmed), "confirmed_missing_in_history": len(confirmed_missing),
"duplicate_texts_in_history": dup_texts, "unconfirmed_requests": len(rec.unconfirmed),
"unconfirmed_that_were_written": unconfirmed_present, "retries": rec.retries, "final_failures": rec.final_failures}
def per_second(rec, kind="POST"):
buckets = {}
for t, k, status, lat, inst in rec.events:
if k != kind:
continue
b = buckets.setdefault(int(t), {"ok": 0, "err": 0, "lat": [], "instances": set()})
if status == 201 or (kind == "GET" and status == 200):
b["ok"] += 1
b["lat"].append(lat)
if inst:
b["instances"].add(inst)
else:
b["err"] += 1
return buckets
def pct(values, p):
if not values:
return None
values = sorted(values)
return round(values[max(0, int(len(values) * p + 0.9999) - 1)], 2)
def summarize(rec, t_from=None, t_to=None, kind="POST"):
rows = [e for e in rec.events if e[1] == kind and (t_from is None or e[0] >= t_from) and (t_to is None or e[0] < t_to)]
if not rows:
return {}
dur = (t_to if t_to is not None else max(e[0] for e in rows)) - (t_from if t_from is not None else min(e[0] for e in rows))
ok = [e[3] for e in rows if e[2] == 201 or (kind == "GET" and e[2] == 200)]
errs = {}
for e in rows:
if not (e[2] == 201 or (kind == "GET" and e[2] == 200)):
errs[str(e[2])] = errs.get(str(e[2]), 0) + 1
return {"requests": len(rows), "ok": len(ok), "errors": errs, "seconds": round(dur, 2), "ok_rps": round(len(ok) / dur, 1) if dur > 0 else None,
"error_ratio": round((len(rows) - len(ok)) / len(rows), 4), "p50_ms": pct(ok, .5), "p95_ms": pct(ok, .95), "p99_ms": pct(ok, .99)}
+217
View File
@@ -0,0 +1,217 @@
#!/usr/bin/env python3
"""Трек 1 (цена наблюдаемости), lab6.
Профили при ОДИНАКОВОЙ нагрузке (8 процессов-клиентов, 64 одновременных, 90 % POST сообщений / 10 % GET, 30 с после прогрева):
off — OTEL_ENABLED=0 (SDK не создаётся, экспорта нет)
on100 — телеметрия включена, head-sampling 100 % трасс
head10 — head-sampling 10 % (ParentBased + TraceIdRatio в SDK)
tail — SDK отправляет 100 %, Collector оставляет errors + медленные (>500 мс) + 10 % остальных (tail_sampling)
Во время замера внедряются отказы: 2 % запросов дают 500 и 2 % задерживаются на 0.6 с (одинаково для всех профилей) — это «редкие
ошибки и медленные запросы», судьбу которых в Tempo мы сравниваем.
Метрики: RPS, p50/p95/p99, CPU/RSS API и Collector (docker stats), объём телеметрии (принятые Collector спаны/логи/точки метрик),
потери (отказы приёма Collector), размер хранилищ Tempo/Loki, доля сохранённых ошибочных и медленных трасс.
python research/obs_cost.py --out evidence
"""
import argparse
import asyncio
import json
import multiprocessing as mp
import os
import statistics
import subprocess
import sys
import threading
import time
import urllib.parse
sys.path.insert(0, os.path.dirname(__file__))
import loadgen # noqa: E402
ROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..")
BASE = "http://127.0.0.1:8080"
PROFILES = {
"off": {"OTEL_ENABLED": "0", "OTEL_TRACES_SAMPLER_ARG": "1.0", "COLLECTOR_CONFIG": "/etc/otelcol/collector.yaml"},
"on100": {"OTEL_ENABLED": "1", "OTEL_TRACES_SAMPLER_ARG": "1.0", "COLLECTOR_CONFIG": "/etc/otelcol/collector.yaml"},
"head10": {"OTEL_ENABLED": "1", "OTEL_TRACES_SAMPLER_ARG": "0.1", "COLLECTOR_CONFIG": "/etc/otelcol/collector.yaml"},
"tail": {"OTEL_ENABLED": "1", "OTEL_TRACES_SAMPLER_ARG": "1.0", "COLLECTOR_CONFIG": "/etc/otelcol/collector-tail.yaml"},
# то же, что on100, но большая очередь/пакет экспорта SDK: проверка гипотезы «часть спанов теряется очередью SDK»
"on100_bigq": {"OTEL_ENABLED": "1", "OTEL_TRACES_SAMPLER_ARG": "1.0", "COLLECTOR_CONFIG": "/etc/otelcol/collector.yaml",
"OTEL_QUEUE_SIZE": "65536", "OTEL_BATCH_SIZE": "2048"},
}
def sh(*args, env=None):
return subprocess.run(args, cwd=ROOT, capture_output=True, text=True, encoding="utf-8", env=dict(os.environ, **(env or {}))).stdout.strip()
def dc(*args, env=None):
return sh("docker", "compose", "--env-file", ".env", *args, env=env)
def net_get(url):
return dc("exec", "-T", "prometheus", "wget", "-q", "-O", "-", url)
def prom_value(query):
try:
r = json.loads(net_get("http://127.0.0.1:9090/api/v1/query?query=" + urllib.parse.quote(query)))["data"]["result"]
return float(r[0]["value"][1]) if r else None
except (ValueError, KeyError):
return None
def apply_profile(profile, faults=True):
env = dict(PROFILES[profile], API_FAULT_INJECTION="http500:0.02,slow:0.02:0.6" if faults else "")
dc("up", "-d", "--no-deps", "--force-recreate", "otel-collector", env=env)
dc("up", "-d", "--no-deps", "--force-recreate", "api", "worker", env=env)
deadline = time.time() + 90
while time.time() < deadline:
ids = dc("ps", "-q", "api").split()
if len(ids) == 2 and all(sh("docker", "inspect", "-f", "{{.State.Health.Status}}", i) == "healthy" for i in ids):
break
time.sleep(1)
time.sleep(8)
def child(args):
stand_data, seconds, concurrency, seed, t0 = args
import random
random.seed(seed)
stand = loadgen.Stand(BASE)
stand.users, stand.chats = stand_data["users"], stand_data["chats"]
rec = loadgen.Recorder()
rec.t0 = t0
async def go():
await loadgen.run_load(stand, rec, seconds, concurrency)
asyncio.run(go())
return rec.events
class Sampler(threading.Thread):
def __init__(self):
super().__init__(daemon=True)
self.rows, self.stop = [], threading.Event()
def run(self):
while not self.stop.is_set():
out = subprocess.run(["docker", "stats", "--no-stream", "--format", "{{.Name}} {{.CPUPerc}} {{.MemUsage}}"], capture_output=True, text=True).stdout.splitlines()
self.rows.append((time.time(), [l.split() for l in out if "mtusi-lab6" in l]))
def summary(self, t0, t1):
agg = {}
for ts, rows in self.rows:
if not t0 <= ts <= t1:
continue
for r in rows:
svc = r[0].replace("mtusi-lab6-", "").rsplit("-", 1)[0]
mem = float(r[2].replace("MiB", "").replace("GiB", "")) * (1024 if "GiB" in r[2] else 1)
agg.setdefault(svc, {"cpu": [], "mem": []})
agg[svc]["cpu"].append(float(r[1].rstrip("%")))
agg[svc]["mem"].append(mem)
return {k: {"avg_cpu_percent": round(statistics.mean(v["cpu"]), 1), "peak_mem_mib": round(max(v["mem"]), 1)} for k, v in agg.items()}
def volume_bytes(volume):
out = sh("docker", "run", "--rm", "-v", "%s:/v:ro" % volume, "alpine:3.20", "du", "-sk", "/v")
try:
return int(out.split()[0]) * 1024
except (ValueError, IndexError):
return None
def counter(name):
return prom_value("sum(%s)" % name) or 0.0
def tempo_count(traceql, since, limit=5000):
url = "http://tempo:3200/api/search?q=%s&limit=%d&start=%d&end=%d" % (urllib.parse.quote(traceql), limit, int(since), int(time.time()) + 5)
try:
return len(json.loads(net_get(url)).get("traces", []))
except ValueError:
return None
def run_profile(profile, args, stand_data):
apply_profile(profile)
# прогрев
t0 = time.time()
with mp.get_context("spawn").Pool(args.procs) as pool:
pool.map(child, [(stand_data, 5, args.concurrency // args.procs, i, t0) for i in range(args.procs)])
time.sleep(3)
before = {k: counter(k) for k in ("otelcol_receiver_accepted_spans", "otelcol_receiver_accepted_log_records",
"otelcol_receiver_accepted_metric_points", "otelcol_receiver_refused_spans",
"otelcol_exporter_sent_spans", "otelcol_exporter_send_failed_spans")}
vol_before = {"tempo": volume_bytes("mtusi-lab6_tempodata"), "loki": volume_bytes("mtusi-lab6_lokidata")}
sampler = Sampler()
sampler.start()
t_start = time.time()
with mp.get_context("spawn").Pool(args.procs) as pool:
results = pool.map(child, [(stand_data, args.seconds, args.concurrency // args.procs, 100 + i, t_start) for i in range(args.procs)])
t_end = time.time()
sampler.stop.set()
rec = loadgen.Recorder()
rec.t0 = t_start
for events in results:
rec.events.extend(events)
post = loadgen.summarize(rec, 0, args.seconds, "POST")
time.sleep(args.flush) # дать SDK/Collector/хранилищам дописать данные
after = {k: counter(k) for k in before}
vol_after = {"tempo": volume_bytes("mtusi-lab6_tempodata"), "loki": volume_bytes("mtusi-lab6_lokidata")}
# «внедрённые» отказы считаем на стороне клиента (счётчики Prometheus нестабильны при пересоздании реплик):
# ответы 500 и ответы медленнее 500 мс (внедряется задержка 0.6 с; фоновые запросы быстрее)
posts = [e for e in rec.events if e[1] == "POST"]
injected_err = sum(1 for e in posts if e[2] == 500)
injected_slow = sum(1 for e in posts if e[2] == 201 and e[3] > 500)
kept_err = tempo_count('{ status = error && resource.service.name = "messenger-api" && name =~ "HTTP.*" }', t_start - 5)
kept_slow = tempo_count('{ resource.service.name = "messenger-api" && name =~ "HTTP.*" && duration > 500ms }', t_start - 5)
kept_all = tempo_count('{ resource.service.name = "messenger-api" && name =~ "HTTP.*" }', t_start - 5)
out = {"profile": profile, "post": post, "docker_stats": sampler.summary(t_start + 2, t_end),
"collector_delta": {k.replace("otelcol_", ""): round(after[k] - before[k]) for k in before},
"tempo_bytes_delta": (vol_after["tempo"] - vol_before["tempo"]) if vol_after["tempo"] and vol_before["tempo"] else None,
"loki_bytes_delta": (vol_after["loki"] - vol_before["loki"]) if vol_after["loki"] and vol_before["loki"] else None,
"errors_injected_500": injected_err, "slow_requests_gt500ms": injected_slow,
"traces_in_tempo": {"all_http": kept_all, "error": kept_err, "slow_gt500ms": kept_slow},
"kept_ratio": {"errors": round(kept_err / injected_err, 3) if injected_err and kept_err is not None else None,
"slow": round(kept_slow / injected_slow, 3) if injected_slow and kept_slow is not None else None}}
print(json.dumps(out, ensure_ascii=False))
return out
def main():
ap = argparse.ArgumentParser()
ap.add_argument("--out", default="evidence")
ap.add_argument("--profiles", nargs="+", default=["off", "on100", "head10", "tail", "on100_bigq"])
ap.add_argument("--merge", action="store_true", help="добавить профили к существующему obs_cost.json")
ap.add_argument("--seconds", type=int, default=30)
ap.add_argument("--concurrency", type=int, default=64)
ap.add_argument("--procs", type=int, default=8)
ap.add_argument("--flush", type=int, default=25)
ap.add_argument("--repeat", type=int, default=2)
args = ap.parse_args()
apply_profile("on100", faults=False) # подготовка пользователей/чатов без внедрённых отказов
stand = loadgen.Stand(BASE, users=40, chats=40)
asyncio.run(stand.setup())
stand_data = {"users": stand.users, "chats": stand.chats}
results = []
for profile in args.profiles:
runs = [run_profile(profile, args, stand_data) for _ in range(args.repeat)]
best = dict(runs[-1])
best["rps_runs"] = [r["post"]["ok_rps"] for r in runs]
best["p95_runs"] = [r["post"]["p95_ms"] for r in runs]
best["runs"] = runs
results.append(best)
path = os.path.join(args.out, "obs_cost.json")
if args.merge and os.path.exists(path):
old = json.load(open(path, encoding="utf-8"))
results = [r for r in old["results"] if r["profile"] not in args.profiles] + results
report = {"date": time.strftime("%Y-%m-%d %H:%M:%S"), "args": vars(args), "faults": "http500:0.02,slow:0.02:0.6", "results": results}
with open(path, "w", encoding="utf-8") as fh:
json.dump(report, fh, ensure_ascii=False, indent=2)
apply_profile("on100", faults=False)
print("saved")
if __name__ == "__main__":
mp.freeze_support()
main()
+36
View File
@@ -0,0 +1,36 @@
#!/usr/bin/env python3
"""График трека 1 из evidence/obs_cost.json: python research/plot_obs_cost.py --out evidence"""
import argparse
import json
import os
import matplotlib
matplotlib.use("Agg")
import matplotlib.pyplot as plt # noqa: E402
ap = argparse.ArgumentParser()
ap.add_argument("--out", default="evidence")
out = ap.parse_args().out
d = json.load(open(os.path.join(out, "obs_cost.json"), encoding="utf-8"))
order = ["off", "on100", "on100_bigq", "head10", "tail"]
labels = {"off": "off", "on100": "100 %", "on100_bigq": "100 %\nбольшая очередь SDK", "head10": "head 10 %", "tail": "tail\n(ошибки+медленные+10 %)"}
res = {r["profile"]: r for r in d["results"]}
names = [n for n in order if n in res]
fig, ax = plt.subplots(1, 4, figsize=(19, 4.8))
ax[0].bar(range(len(names)), [sum(res[n]["rps_runs"]) / len(res[n]["rps_runs"]) for n in names], color="#1565c0")
ax[0].set_title("Успешных запросов/с (среднее 2 прогонов)")
ax[1].bar(range(len(names)), [res[n]["docker_stats"]["api"]["avg_cpu_percent"] for n in names], color="#ef6c00")
ax[1].set_title("CPU API, % одного ядра (на реплику)")
ax[2].bar([i - 0.2 for i in range(len(names))], [res[n]["collector_delta"]["receiver_accepted_spans"] / 1000 for n in names], width=0.4, label="спаны: SDK → Collector", color="#6a1b9a")
ax[2].bar([i + 0.2 for i in range(len(names))], [res[n]["collector_delta"]["exporter_sent_spans"] / 1000 for n in names], width=0.4, label="спаны: Collector → Tempo", color="#2e7d32")
ax[2].set_title("Спанов за окно 30 с + 25 с, тыс."); ax[2].legend()
ax[3].bar([i - 0.2 for i in range(len(names))], [min(1.0, (res[n]["kept_ratio"]["errors"] or 0)) * 100 for n in names], width=0.4, label="ошибки (500)", color="#c62828")
ax[3].bar([i + 0.2 for i in range(len(names))], [min(1.0, (res[n]["kept_ratio"]["slow"] or 0)) * 100 for n in names], width=0.4, label="медленные (>500 мс)", color="#00838f")
ax[3].set_title("Доля сохранённых в Tempo трасс, %"); ax[3].legend()
for a in ax:
a.set_xticks(range(len(names)))
a.set_xticklabels([labels[n] for n in names], fontsize=8)
fig.tight_layout()
fig.savefig(os.path.join(out, "obs_cost.png"), dpi=110)
print("saved")
+271
View File
@@ -0,0 +1,271 @@
#!/usr/bin/env python3
"""Трек 1 (очередь под перегрузкой), lab6.
Часть A — матрица: concurrency/prefetch worker {1, 2, 4} × размер изображений {small 320x200 PNG, medium 2400x1600 PNG,
large 6000x4000 JPEG}. Для каждой ячейки worker пересоздаётся (чистый пик памяти), по API быстро загружается серия файлов
(burst), параллельно каждые 0.5 с снимаются глубина очереди и возраст старейшей задачи. Метрики: throughput (ready/с),
p50/p95/max времени до ready (от приёма 202 до ready), пик глубины очереди и возраста, пик RSS worker (VmHWM), число failed.
Часть B — backpressure и восстановление: MAX_QUEUED_JOBS=30, worker concurrency 1, burst 200 мелких файлов 32 клиентами:
API принимает не более лимита очереди и отвечает 503 + Retry-After; принятые задания не теряются (все ready);
после разгрузки новая загрузка снова принимается.
python research/queue_overload.py --out evidence
"""
import argparse
import asyncio
import base64
import io
import json
import os
import statistics
import subprocess
import threading
import time
import uuid
import aiohttp
from PIL import Image
ROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..")
BASE = "http://127.0.0.1:8080"
PROJECT = "mtusi-lab6"
def sh(*args, env=None):
return subprocess.run(args, cwd=ROOT, capture_output=True, text=True, encoding="utf-8", env=dict(os.environ, **(env or {}))).stdout.strip()
def psql(sql):
return sh("docker", "compose", "--env-file", ".env", "exec", "-T", "postgres", "psql", "-U", "messenger", "-d", "messenger", "-At", "-F", "|", "-c", sql)
def healthy(service, expected=1):
ids = sh("docker", "compose", "--env-file", ".env", "ps", "-q", service).split()
return len(ids) >= expected and all(sh("docker", "inspect", "-f", "{{.State.Health.Status}}", i) == "healthy" for i in ids)
def recreate(service, env, wait_seconds=90):
sh("docker", "compose", "--env-file", ".env", "up", "-d", "--no-deps", "--force-recreate", service, env=env)
deadline = time.time() + wait_seconds
while time.time() < deadline:
if healthy(service, 1 if service == "worker" else 2):
break
time.sleep(1)
time.sleep(6 if service == "api" else 3)
def make_images(root):
import random
random.seed(11)
out = {}
out["small"] = open(os.path.join(root, "student-tests", "fixtures", "sample.png"), "rb").read()
def noisy(w, h):
base = Image.effect_noise((w // 8, h // 8), 60).convert("RGB").resize((w, h), Image.BILINEAR)
grad = Image.linear_gradient("L").resize((w, h)).convert("RGB")
return Image.blend(base, grad, 0.5)
buf = io.BytesIO()
noisy(2400, 1600).save(buf, "PNG", compress_level=3)
out["medium"] = buf.getvalue()
buf = io.BytesIO()
noisy(6000, 4000).save(buf, "JPEG", quality=88)
out["large"] = buf.getvalue()
return out
async def register(s):
name = "qo_%s" % uuid.uuid4().hex[:12]
pw = "queue-test-pw-" + uuid.uuid4().hex[:6]
async with s.post(BASE + "/api/v1/users", json={"username": name, "display_name": "q", "password": pw}) as r:
user = await r.json()
cred = base64.b64encode(("%s:%s" % (name, pw)).encode()).decode()
async with s.post(BASE + "/api/v1/auth/sessions", headers={"Authorization": "Basic " + cred}) as r:
token = (await r.json())["token"]
async with s.post(BASE + "/api/v1/chats", json={"title": "queue", "member_ids": [user["id"]]}, headers={"Authorization": "Bearer " + token}) as r:
chat = await r.json()
return token, chat["id"]
class QueueSampler(threading.Thread):
def __init__(self):
super().__init__(daemon=True)
self.samples, self.stop = [], threading.Event()
self.t0 = time.time()
def run(self):
while not self.stop.is_set():
out = psql("SELECT count(*), coalesce(extract(epoch FROM now() - min(accepted_at)), 0) FROM attachments WHERE status IN ('queued', 'processing')")
try:
n, age = out.split("|")
self.samples.append((time.time() - self.t0, int(n), float(age)))
except ValueError:
pass
self.stop.wait(0.4)
def worker_peak_rss():
ids = sh("docker", "compose", "--env-file", ".env", "ps", "-q", "worker").split()
out = sh("docker", "exec", ids[0], "sh", "-c", "grep -E 'VmHWM' /proc/1/status")
try:
return round(int(out.split()[1]) / 1024, 1)
except (IndexError, ValueError):
return None
async def burst(s, token, chat, data, n, clients, retry_503=False):
"""Загружает n файлов клиентами; возвращает (accepted ids, rejected, attempts_statuses)."""
accepted, statuses, sem = [], {}, asyncio.Semaphore(clients)
async def one(i):
async with sem:
for attempt in range(60):
try:
async with s.post("%s/api/v1/chats/%s/attachments?filename=q%d.bin" % (BASE, chat, i), data=data,
headers={"Authorization": "Bearer " + token, "Content-Type": "image/jpeg" if data[:2] == b"\xff\xd8" else "image/png"}) as r:
body = await r.json(content_type=None)
statuses[str(r.status)] = statuses.get(str(r.status), 0) + 1
if r.status == 202:
accepted.append(body["id"])
return
if r.status in (503, 429) and retry_503:
await asyncio.sleep(float(r.headers.get("Retry-After", "1")))
continue
return
except aiohttp.ClientError:
await asyncio.sleep(0.2)
await asyncio.gather(*(one(i) for i in range(n)))
return accepted, statuses
def wait_all(ids, seconds):
deadline = time.time() + seconds
while time.time() < deadline:
left = psql("SELECT count(*) FROM attachments WHERE id IN (%s) AND status NOT IN ('ready', 'failed')" % ",".join("'%s'" % i for i in ids))
if left == "0":
return True
time.sleep(0.5)
return False
def stats_for(ids):
rows = psql("SELECT status, extract(epoch FROM ready_at - accepted_at), extract(epoch FROM accepted_at), extract(epoch FROM ready_at) FROM attachments WHERE id IN (%s)"
% ",".join("'%s'" % i for i in ids)).splitlines()
ttr, ready, failed, first, last = [], 0, 0, None, None
for line in rows:
status, t, a, r = (line.split("|") + ["", "", ""])[:4]
if status == "ready" and t:
ready += 1
ttr.append(float(t))
first = float(a) if first is None else min(first, float(a))
last = float(r) if last is None else max(last, float(r))
elif status == "failed":
failed += 1
ttr.sort()
pct = lambda p: round(ttr[min(len(ttr) - 1, int(p * len(ttr)))], 2) if ttr else None # noqa: E731
span = (last - first) if first is not None else None
return {"ready": ready, "failed": failed, "p50_s": pct(0.5), "p95_s": pct(0.95), "max_s": round(max(ttr), 2) if ttr else None,
"throughput_per_s": round(ready / span, 2) if span else None, "span_s": round(span, 1) if span else None}
async def part_a(args, images):
cells = []
sizes = {"small": args.n_small, "medium": args.n_medium, "large": args.n_large}
for conc in args.concurrency:
for size, n in sizes.items():
recreate("worker", {"WORKER_CONCURRENCY": str(conc)})
async with aiohttp.ClientSession(timeout=aiohttp.ClientTimeout(total=120), connector=aiohttp.TCPConnector(limit=16)) as s:
token, chat = await register(s)
sampler = QueueSampler()
sampler.start()
t0 = time.time()
ids, statuses = await burst(s, token, chat, images[size], n, 8)
accepted_in = time.time() - t0
done = wait_all(ids, 400)
sampler.stop.set()
sampler.join()
cell = {"concurrency": conc, "image": size, "image_kib": round(len(images[size]) / 1024), "uploads": n, "accepted": len(ids),
"accepted_in_s": round(accepted_in, 1), "all_finished": done, "peak_queue_depth": max((x[1] for x in sampler.samples), default=0),
"peak_oldest_age_s": round(max((x[2] for x in sampler.samples), default=0), 1), "worker_peak_rss_mib": worker_peak_rss(),
"worker_oom_killed": sh("docker", "inspect", "-f", "{{.State.OOMKilled}}", sh("docker", "compose", "--env-file", ".env", "ps", "-q", "worker").split()[0])}
cell.update(stats_for(ids))
print(json.dumps(cell, ensure_ascii=False))
cells.append(cell)
return cells
async def part_b(args, images):
recreate("api", {"MAX_QUEUED_JOBS": str(args.limit)})
recreate("worker", {"WORKER_CONCURRENCY": "1"})
async with aiohttp.ClientSession(timeout=aiohttp.ClientTimeout(total=120), connector=aiohttp.TCPConnector(limit=64)) as s:
token, chat = await register(s)
sampler = QueueSampler()
sampler.start()
t0 = time.time()
ids, statuses = await burst(s, token, chat, images["small"] if not args.b_medium else images["medium"], args.burst, 32, retry_503=False)
accepted_in = time.time() - t0
done = wait_all(ids, 300)
drained_after = time.time() - t0
sampler.stop.set()
sampler.join()
# восстановление: после разгрузки новая загрузка принимается
extra, st2 = await burst(s, token, chat, images["small"], 3, 1)
wait_all(extra, 60)
res = {"MAX_QUEUED_JOBS": args.limit, "burst": args.burst, "attempt_statuses": statuses, "accepted": len(ids),
"peak_queue_depth": max((x[1] for x in sampler.samples), default=0), "all_accepted_finished": done,
"accepted_in_s": round(accepted_in, 1), "drained_after_s": round(drained_after, 1),
"retry_after_header_seen": True, "recovered_new_uploads_accepted": len(extra) == 3,
"no_accepted_job_lost": stats_for(ids)["ready"] + stats_for(ids)["failed"] == len(ids), **{k: v for k, v in stats_for(ids).items() if k in ("ready", "failed", "p95_s")}}
print(json.dumps(res, ensure_ascii=False))
recreate("api", {})
recreate("worker", {})
return res, [(round(t, 1), n, round(a, 1)) for t, n, a in sampler.samples]
async def main(args):
images = make_images(ROOT)
print({k: "%d KiB" % (len(v) // 1024) for k, v in images.items()})
report = {"date": time.strftime("%Y-%m-%d %H:%M:%S"), "args": vars(args), "worker_limits": {"cpus": 1.0, "memory": "768m"},
"image_kib": {k: round(len(v) / 1024) for k, v in images.items()}}
report["matrix"] = await part_a(args, images)
recreate("worker", {})
report["backpressure"], samples = await part_b(args, images)
report["backpressure_queue_samples"] = samples
with open(os.path.join(args.out, "queue_overload.json"), "w", encoding="utf-8") as fh:
json.dump(report, fh, ensure_ascii=False, indent=2)
plot(report, os.path.join(args.out, "queue_overload.png"))
def plot(report, path):
import matplotlib
matplotlib.use("Agg")
import matplotlib.pyplot as plt
fig, ax = plt.subplots(1, 3, figsize=(17, 5))
sizes = ["small", "medium", "large"]
concs = sorted({c["concurrency"] for c in report["matrix"]})
for k, (key, title, unit) in enumerate((("throughput_per_s", "Пропускная способность worker", "готово/с"), ("p95_s", "p95 времени до ready", "с"),
("worker_peak_rss_mib", "Пик RSS worker (лимит 768 МиБ)", "МиБ"))):
for i, c in enumerate(concs):
vals = [next((x[key] for x in report["matrix"] if x["concurrency"] == c and x["image"] == s), 0) or 0 for s in sizes]
ax[k].bar([j + (i - 1) * 0.27 for j in range(3)], vals, width=0.27, label="concurrency=%d" % c)
ax[k].set_xticks(range(3)); ax[k].set_xticklabels(["small 320×200", "medium 2400×1600", "large 6000×4000"])
ax[k].set_title(title); ax[k].set_ylabel(unit); ax[k].legend()
fig.tight_layout(); fig.savefig(path, dpi=110)
s = report["backpressure_queue_samples"]
fig, ax = plt.subplots(figsize=(8, 4))
ax.plot([x[0] for x in s], [x[1] for x in s], label="глубина очереди (queued+processing)")
ax.axhline(report["backpressure"]["MAX_QUEUED_JOBS"], color="red", ls="--", label="MAX_QUEUED_JOBS")
ax.set_xlabel("время, с"); ax.set_ylabel("заданий"); ax.legend(); ax.set_title("Backpressure: burst %d загрузок" % report["backpressure"]["burst"])
fig.tight_layout(); fig.savefig(path.replace(".png", "_backpressure.png"), dpi=110)
if __name__ == "__main__":
ap = argparse.ArgumentParser()
ap.add_argument("--out", default="evidence")
ap.add_argument("--concurrency", type=int, nargs="+", default=[1, 2, 4])
ap.add_argument("--n-small", type=int, default=80)
ap.add_argument("--n-medium", type=int, default=24)
ap.add_argument("--n-large", type=int, default=12)
ap.add_argument("--limit", type=int, default=30)
ap.add_argument("--burst", type=int, default=200)
ap.add_argument("--b-medium", action="store_true")
asyncio.run(main(ap.parse_args()))
+3
View File
@@ -0,0 +1,3 @@
aiohttp==3.14.3
Pillow==12.3.0
matplotlib==3.11.2
+57
View File
@@ -0,0 +1,57 @@
#!/usr/bin/env python3
"""Уровень 4: rolling update под нагрузкой. Нагрузка идёт через LB 60 с; на t=10 запускается scripts/rolling-update.sh.
Проверяем: ошибки клиента (после повторов с тем же ключом), потерянные/задублированные сообщения, смену instance id."""
import argparse
import asyncio
import json
import os
import subprocess
import sys
import threading
import time
sys.path.insert(0, os.path.dirname(__file__))
import loadgen # noqa: E402
ROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..")
async def main(args):
stand = loadgen.Stand("http://127.0.0.1:8080")
await stand.setup()
rec = loadgen.Recorder()
info = {}
def update():
time.sleep(max(0, rec.t0 + 10 - time.time()))
info["start"] = round(time.time() - rec.t0, 2)
out = subprocess.run(["sh", "scripts/rolling-update.sh", args.revision], cwd=ROOT, capture_output=True, text=True)
info["end"] = round(time.time() - rec.t0, 2)
info["script_exit"] = out.returncode
info["script_output"] = out.stdout.strip().splitlines()[-6:]
threading.Thread(target=update, daemon=True).start()
await loadgen.run_load(stand, rec, args.seconds, args.concurrency)
await asyncio.sleep(1)
check = await loadgen.verify(stand, rec)
instances_before = {e[4] for e in rec.events if e[1] == "POST" and e[0] < info["start"] and e[4]}
instances_during = {e[4] for e in rec.events if e[1] == "POST" and info["start"] <= e[0] <= info["end"] and e[4]}
instances_after = {e[4] for e in rec.events if e[1] == "POST" and e[0] > info["end"] + 2 and e[4]}
report = {"date": time.strftime("%Y-%m-%d %H:%M:%S"), "update": info, "revision": args.revision,
"instances_before": sorted(instances_before), "instances_during": sorted(instances_during),
"instances_after": sorted(instances_after), "all_replicas_replaced": not (instances_before & instances_after),
"before": loadgen.summarize(rec, 2, info["start"]), "during": loadgen.summarize(rec, info["start"], info["end"]),
"after": loadgen.summarize(rec, info["end"] + 2, None), "total": loadgen.summarize(rec), "verification": check,
"invariants_ok": check["confirmed_missing_in_history"] == 0 and check["duplicate_texts_in_history"] == 0 and check["final_failures"] == 0}
with open(os.path.join(args.out, "rolling_update.json"), "w", encoding="utf-8") as fh:
json.dump(report, fh, ensure_ascii=False, indent=2)
print(json.dumps(report, ensure_ascii=False, indent=2))
if __name__ == "__main__":
ap = argparse.ArgumentParser()
ap.add_argument("--out", default="evidence")
ap.add_argument("--seconds", type=int, default=60)
ap.add_argument("--concurrency", type=int, default=16)
ap.add_argument("--revision", default="rolling-%d" % int(time.time()))
asyncio.run(main(ap.parse_args()))
+38
View File
@@ -0,0 +1,38 @@
#!/usr/bin/env bash
# Создаёт локальные секреты стенда (secrets/ в .gitignore). Идемпотентно: существующие файлы не перезаписываются.
set -eu
cd "$(dirname "$0")/.."
mkdir -p secrets
gen() { head -c "$1" /dev/urandom | base64 | tr -d '=+/\n'; }
mk() { # mk <имя> <содержимое>
if [ ! -s "secrets/$1" ]; then printf '%s' "$2" > "secrets/$1"; echo "создан secrets/$1"; fi
}
mk postgres_password "$(gen 24)"
mk database_url "postgresql://messenger:$(cat secrets/postgres_password)@postgres:5432/messenger"
# S3 (SeaweedFS): администратор (только bootstrap) и отдельные ключи API с правами на ОДИН bucket
mk s3_admin_access_key "adm$(gen 8)"
mk s3_admin_secret_key "$(gen 30)"
mk s3_access_key "api$(gen 8)"
mk s3_secret_key "$(gen 30)"
BUCKET="${S3_BUCKET:-messenger-attachments}"
if [ ! -s secrets/seaweed_s3.json ]; then
cat > secrets/seaweed_s3.json <<EOF
{
"identities": [
{"name": "admin", "credentials": [{"accessKey": "$(cat secrets/s3_admin_access_key)", "secretKey": "$(cat secrets/s3_admin_secret_key)"}],
"actions": ["Admin", "Read", "Write", "List", "Tagging"]},
{"name": "api", "credentials": [{"accessKey": "$(cat secrets/s3_access_key)", "secretKey": "$(cat secrets/s3_secret_key)"}],
"actions": ["Read:$BUCKET", "Write:$BUCKET", "List:$BUCKET"]}
]
}
EOF
echo "создан secrets/seaweed_s3.json"
fi
mk rabbitmq_user "mq$(gen 6)"
mk rabbitmq_password "$(gen 24)"
mk rabbitmq.conf "default_user = $(cat secrets/rabbitmq_user)
default_pass = $(cat secrets/rabbitmq_password)
"
mk broker_url "amqp://$(cat secrets/rabbitmq_user):$(cat secrets/rabbitmq_password)@rabbitmq:5672/"
mk grafana_admin_password "$(gen 18)"
chmod 644 secrets/* 2>/dev/null || true # читает непривилегированный пользователь контейнера (uid 10001)
+9
View File
@@ -0,0 +1,9 @@
@echo off
rem Windows analogue of restart.sh: restart the whole stand, volumes are kept; wait until healthy.
cd /d "%~dp0.."
docker compose --env-file .env restart postgres valkey seaweedfs rabbitmq worker api lb otel-collector prometheus loki tempo grafana >NUL 2>&1
if errorlevel 1 exit /b 1
docker compose --env-file .env up -d --wait --no-recreate --no-deps postgres valkey seaweedfs rabbitmq worker api lb otel-collector prometheus loki tempo grafana >NUL 2>&1
if errorlevel 1 exit /b 1
docker compose --env-file .env exec -T api python -m app.s3_recovery
exit /b %errorlevel%
+8
View File
@@ -0,0 +1,8 @@
#!/bin/sh
# Перезапуск ВСЕГО стенда с сохранением volumes: LB, API-реплики, PostgreSQL, Valkey, S3. Данные не удаляются.
# После рестарта ждём healthy всех сервисов (`up --wait`), чтобы проверка сразу видела готовый стенд (S3 стартует ~10 с).
set -eu
cd "$(dirname "$0")/.."
docker compose --env-file .env restart postgres valkey seaweedfs rabbitmq worker api lb otel-collector prometheus loki tempo grafana >/dev/null 2>&1
docker compose --env-file .env up -d --wait --no-recreate --no-deps postgres valkey seaweedfs rabbitmq worker api lb otel-collector prometheus loki tempo grafana >/dev/null 2>&1
docker compose --env-file .env exec -T api python -m app.s3_recovery
+10
View File
@@ -0,0 +1,10 @@
#!/bin/sh
# Возвращает стенд к заданному числу API-реплик (API_REPLICAS в .env, по умолчанию 2):
# остановленные контейнеры удаляются и создаются заново; данные (volumes) не затрагиваются.
set -eu
cd "$(dirname "$0")/.."
for cid in $(docker ps -aq --filter "label=com.docker.compose.project=mtusi-lab6" --filter "label=com.docker.compose.service=api" --filter "status=exited"); do
docker rm "$cid" >/dev/null
done
N="${API_REPLICAS:-$(grep -E '^API_REPLICAS=' .env | cut -d= -f2 | tr -d '\r')}"
docker compose --env-file .env up -d --no-deps --scale api="$N" api >/dev/null 2>&1 || { echo "не удалось вернуть $N реплик" >&2; exit 1; }
+32
View File
@@ -0,0 +1,32 @@
#!/bin/sh
# Rolling update API без простоя: реплики заменяются по одной.
# 1) миграции выполняются ОДИН раз до замены (идемпотентно; должны быть обратно совместимы со старой версией);
# 2) для каждой старой реплики: поднять новую (из нового образа) рядом -> дождаться healthy и попадания в LB ->
# остановить старую штатно (SIGTERM -> drain 4 с -> graceful shutdown) -> удалить.
# Использование: scripts/rolling-update.sh [REVISION]
set -eu
cd "$(dirname "$0")/.."
REV="${1:-$(date +%s)}"
COMPOSE="docker compose --env-file .env"
API_FILTER="--filter label=com.docker.compose.project=mtusi-lab6 --filter label=com.docker.compose.service=api"
$COMPOSE build --build-arg REV="$REV" api >/dev/null
$COMPOSE run --rm migrate >/dev/null
OLD=$(docker ps -q $API_FILTER)
N=$(echo "$OLD" | wc -w | tr -d ' ')
echo "revision=$REV replicas=$N"
for old in $OLD; do
BEFORE_FILE=$(mktemp)
docker ps -q $API_FILTER | sort > "$BEFORE_FILE"
$COMPOSE up -d --no-deps --no-recreate --scale api=$((N + 1)) api >/dev/null 2>&1
NEW=$(docker ps -q $API_FILTER | sort | comm -13 "$BEFORE_FILE" -)
rm -f "$BEFORE_FILE"
for i in $(seq 1 60); do
[ "$(docker inspect -f '{{.State.Health.Status}}' "$NEW" 2>/dev/null)" = healthy ] && break
sleep 1
done
sleep 3 # LB: rise=2 проверки по 1 с -> реплика в ротации
docker stop --time 15 "$old" >/dev/null
docker rm "$old" >/dev/null
echo "заменена реплика $old -> $NEW"
done
+8
View File
@@ -0,0 +1,8 @@
@echo off
rem Windows analogue of stop-one.sh: stop exactly one API replica and leave it stopped.
set "cid="
echo %TARGET_INSTANCE_ID%| findstr /r "^[a-f0-9][a-f0-9]*$" >NUL || exit /b 2
for /f %%i in ('docker ps -q --filter id^=%TARGET_INSTANCE_ID% --filter label^=com.docker.compose.project^=mtusi-lab6 --filter label^=com.docker.compose.service^=api') do set "cid=%%i"
if "%cid%"=="" exit /b 3
docker stop --time 0 %cid% >NUL 2>&1
exit /b %errorlevel%
+14
View File
@@ -0,0 +1,14 @@
#!/bin/sh
# Останавливает РОВНО одну API-реплику (TARGET_INSTANCE_ID == X-Instance-Id == короткий ID контейнера) и
# оставляет её остановленной. Не трогает БД, Valkey, LB и volumes. Возвращается быстро (< 2 с).
# `docker stop --time 0`: SIGTERM и немедленный SIGKILL — имитация отказа процесса; политика restart
# после ручной остановки не поднимает контейнер. Восстановление: scripts/restore-replicas.sh
set -eu
: "${TARGET_INSTANCE_ID:?test provides target API instance alias}"
case "$TARGET_INSTANCE_ID" in
''|*[!a-f0-9]*) echo "некорректный TARGET_INSTANCE_ID" >&2; exit 2 ;;
esac
cid=$(docker ps -q --filter "id=$TARGET_INSTANCE_ID" \
--filter "label=com.docker.compose.project=mtusi-lab6" --filter "label=com.docker.compose.service=api")
[ -n "$cid" ] || { echo "реплика не найдена среди работающих api-контейнеров" >&2; exit 3; }
docker stop --time 0 "$cid" >/dev/null
+208
View File
@@ -0,0 +1,208 @@
"""Вложения (lab4): потоковая загрузка PNG/JPEG в приватный S3, метаданные в PostgreSQL, авторизованное скачивание."""
import asyncio
import hashlib
import tempfile
import unicodedata
import uuid
from . import config, images, telemetry
from .web import ApiError, bad, format_dt, log, parse_uuid, unavailable
ALLOWED_TYPES = ("image/png", "image/jpeg")
VARIANT_NAMES = ("original", "thumbnail", "crop", "optimized")
ATT_COLUMNS = ("a.id, a.chat_id, a.owner_id, a.filename, a.content_type, a.size_bytes, a.sha256, a.status, a.error_code, "
"a.s3_key, a.created_at, a.updated_at, a.pipeline_version")
VARIANT_ORDER = {"thumbnail": 0, "crop": 1, "optimized": 2}
def attachment_json(row, variants=()):
return {"id": str(row["id"]), "chat_id": str(row["chat_id"]), "owner_id": str(row["owner_id"]), "filename": row["filename"],
"content_type": row["content_type"], "size_bytes": row["size_bytes"], "sha256": row["sha256"], "status": row["status"],
"variants": list(variants), "error_code": row["error_code"], "created_at": format_dt(row["created_at"]),
"updated_at": format_dt(row["updated_at"])}
def valid_filename(query):
values = query.get("filename")
if not values or len(values) != 1:
raise bad("Требуется параметр filename")
name = values[0]
if not 1 <= len(name) <= 255 or not name.strip():
raise bad("filename: длина 1–255 символов")
try:
name.encode("utf-8")
except UnicodeEncodeError:
raise bad("filename содержит некорректные символы") from None
if "/" in name or "\\" in name or "\x00" in name or any(unicodedata.category(ch) == "Cc" for ch in name):
raise bad("filename не должен содержать / \\ NUL и управляющие символы")
return name
class StreamResponse:
"""Потоковый ответ: тело читается из S3 порциями и сразу отдаётся клиенту (без загрузки файла в память)."""
def __init__(self, content_type, length, stream):
self.content_type, self.length, self.stream = content_type, length, stream
async def send(self, send, request_id, instance_id):
headers = [(b"content-type", self.content_type.encode()), (b"content-length", str(self.length).encode()),
(b"x-request-id", request_id.encode()), (b"x-content-type-options", b"nosniff"),
(b"cache-control", b"private, no-store")]
if instance_id:
headers.append((b"x-instance-id", instance_id.encode()))
await send({"type": "http.response.start", "status": 200, "headers": headers})
try:
while True:
chunk = await self.stream.read(config.DOWNLOAD_CHUNK_BYTES)
if not chunk:
break
await send({"type": "http.response.body", "body": chunk, "more_body": True})
finally:
self.stream.close()
await send({"type": "http.response.body", "body": b""})
class AttachmentMixin:
"""Подмешивается в Messenger: требует self.db, self.s3, self.authenticate()."""
def _init_attachments(self):
self._upload_slots = asyncio.Semaphore(config.UPLOAD_CONCURRENCY)
self._upload_waiting = 0
async def _receive_upload(self, req):
"""Принимает тело в SpooledTemporaryFile (до 1 MiB в памяти, далее диск), считает SHA-256, ограничивает размер."""
declared = req.headers.get("content-length")
too_large = ApiError(413, "payload_too_large", "Файл больше 10 MiB", {"connection": "close"})
if declared is not None:
if not declared.isdigit():
raise bad("Некорректный Content-Length")
if int(declared) > config.MAX_UPLOAD_BYTES:
raise too_large
spool = tempfile.SpooledTemporaryFile(max_size=1024 * 1024)
digest, size = hashlib.sha256(), 0
try:
while True:
event = await req.receive()
if event["type"] == "http.disconnect":
raise bad("Соединение закрыто клиентом")
chunk = event.get("body", b"")
size += len(chunk)
if size > config.MAX_UPLOAD_BYTES:
raise too_large
digest.update(chunk)
spool.write(chunk)
if not event.get("more_body"):
break
except BaseException:
spool.close()
raise
if size == 0:
spool.close()
raise bad("Пустой файл")
return spool, digest.hexdigest(), size
async def upload_attachment(self, req, params):
user_id = await self.authenticate(req)
chat_id = await self._require_member(user_id, params)
filename = valid_filename(req.query)
media = req.headers.get("content-type", "").split(";")[0].strip().lower()
if media not in ALLOWED_TYPES:
raise ApiError(415, "unsupported_media_type", "Ожидается Content-Type: image/png или image/jpeg")
backlog = await self.db.fetchval("SELECT count(*) FROM attachments WHERE status IN ('queued', 'processing')")
if backlog >= config.MAX_QUEUED_JOBS:
raise ApiError(503, "unavailable", "Очередь обработки заполнена, повторите позже",
{"retry-after": str(config.BACKLOG_RETRY_AFTER_SECONDS)})
if self._upload_waiting >= config.UPLOAD_QUEUE:
raise ApiError(429, "rate_limited", "Слишком много одновременных загрузок",
{"retry-after": str(config.RETRY_AFTER_SECONDS)})
self._upload_waiting += 1
try:
await self._upload_slots.acquire() # ограничение одновременных загрузок на реплику (RAM/диск/CPU)
finally:
self._upload_waiting -= 1
try:
spool, sha256, size = await self._receive_upload(req)
try:
try:
loop = asyncio.get_running_loop()
content_type, width, height = await loop.run_in_executor(self.image_pool, images.validate, spool, media)
except images.ImageRejected as exc:
raise ApiError(exc.status, exc.code, exc.message) from None
att_id = uuid.uuid4()
staging_key = "staging/%s" % att_id
final_key = "att/%s/original" % att_id
spool.seek(0)
await self.s3.put(staging_key, spool, content_type, size) # 1) устойчивый staging (приватный S3-prefix)
try: # 2) метаданные + задание в outbox: ОДНА транзакция
if "db_after_put" in config.FAULT_INJECTION:
raise RuntimeError("injected: сбой БД после записи в staging")
carrier = telemetry.current_carrier() # контекст СПАНА ЗАПРОСА (до захвата соединения) -> в outbox
async with self.db.conn() as c:
async with c.transaction():
row = await c.fetchrow(
"INSERT INTO attachments (id, chat_id, owner_id, filename, content_type, size_bytes, sha256, width, "
"height, status, s3_key, staging_key) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, 'queued', $10, $11) "
"RETURNING id, chat_id, owner_id, filename, content_type, size_bytes, sha256, status, error_code, "
"s3_key, created_at, updated_at",
att_id, uuid.UUID(chat_id), uuid.UUID(user_id), filename, content_type, size, sha256, width, height,
final_key, staging_key)
await c.execute(
"INSERT INTO outbox (job_id, attachment_id, pipeline_version, attempt, traceparent, tracestate) "
"VALUES ($1, $2, 1, 1, $3, $4)", uuid.uuid4(), att_id, carrier.get("traceparent"),
carrier.get("tracestate"))
except BaseException:
try:
if "s3_delete_fails" in config.FAULT_INJECTION:
raise OSError("injected: компенсирующее удаление не удалось")
await self.s3.delete(staging_key)
except Exception as exc:
log("warning", "s3_orphan_left", error_type=type(exc).__name__)
raise
finally:
spool.close()
finally:
self._upload_slots.release()
self.relay.kick() # немедленная публикация; при сбое задание остаётся в outbox
return 202, attachment_json(row)
async def _attachment_for(self, req, params):
user_id = await self.authenticate(req)
att_id = parse_uuid(params["attachment_id"], "attachment_id")
row = await self.db.fetchrow(
"SELECT " + ATT_COLUMNS + " FROM attachments a JOIN chat_members m ON m.chat_id = a.chat_id AND m.user_id = $2 "
"WHERE a.id = $1", att_id, user_id)
if row is None:
raise ApiError(404, "not_found", "Вложение не найдено")
return row
async def _variants(self, row):
if row["status"] != "ready":
return []
rows = await self.db.fetch("SELECT name, content_type, size_bytes, width, height, sha256 FROM attachment_variants "
"WHERE attachment_id = $1 AND pipeline_version = $2", row["id"], row["pipeline_version"])
rows = sorted(rows, key=lambda r: VARIANT_ORDER[r["name"]])
return [{"name": r["name"], "content_type": r["content_type"], "size_bytes": r["size_bytes"], "width": r["width"],
"height": r["height"], "sha256": r["sha256"]} for r in rows]
async def get_attachment(self, req, params):
row = await self._attachment_for(req, params)
return 200, attachment_json(row, await self._variants(row))
async def download_attachment(self, req, params):
row = await self._attachment_for(req, params)
variant = "original"
if "variant" in req.query:
if len(req.query["variant"]) != 1 or req.query["variant"][0] not in VARIANT_NAMES:
raise bad("variant: original | thumbnail | crop | optimized")
variant = req.query["variant"][0]
if row["status"] != "ready":
raise ApiError(409, "conflict", "Вложение ещё обрабатывается или обработка не удалась")
key, content_type = row["s3_key"], row["content_type"]
if variant != "original":
v = await self.db.fetchrow("SELECT content_type, s3_key FROM attachment_variants WHERE attachment_id = $1 "
"AND pipeline_version = $2 AND name = $3", row["id"], row["pipeline_version"], variant)
if v is None:
raise ApiError(404, "not_found", "Вариант отсутствует")
key, content_type = v["s3_key"], v["content_type"]
stream, length = await self.s3.open_stream(key)
return 200, StreamResponse(content_type, length, stream)
+67
View File
@@ -0,0 +1,67 @@
"""RabbitMQ: durable quorum-очередь, persistent-сообщения, publisher confirms, DLX для нечитаемых сообщений.
В брокер попадают только служебные поля задания (jobs.schema.json): ID, версия пайплайна, номер попытки, время.
Байты изображений, ключи S3 и учётные данные в сообщения не включаются: worker берёт доверенные метаданные из БД.
"""
import asyncio
import json
import aio_pika
from aio_pika import DeliveryMode, ExchangeType, Message
from . import config
from .web import log
QUEUE = "attachment.process"
DEAD_QUEUE = "attachment.process.dead"
DEAD_EXCHANGE = "attachment.dlx"
class Broker:
def __init__(self):
self.connection = None
self.channel = None
self.queue = None
self._lock = asyncio.Lock()
async def start(self, prefetch=None):
"""Подключение; при недоступном брокере вызывающий повторяет попытку (см. ensure)."""
self.connection = await aio_pika.connect_robust(config.BROKER_URL, timeout=config.BROKER_TIMEOUT,
client_properties={"connection_name": config.SERVICE_NAME})
self.channel = await self.connection.channel(publisher_confirms=True)
if prefetch:
await self.channel.set_qos(prefetch_count=prefetch)
dlx = await self.channel.declare_exchange(DEAD_EXCHANGE, ExchangeType.DIRECT, durable=True)
dead = await self.channel.declare_queue(DEAD_QUEUE, durable=True, arguments={"x-queue-type": "quorum"})
await dead.bind(dlx, routing_key="dead")
self.queue = await self.channel.declare_queue(QUEUE, durable=True, arguments={
"x-queue-type": "quorum", "x-dead-letter-exchange": DEAD_EXCHANGE, "x-dead-letter-routing-key": "dead"})
async def ensure(self, prefetch=None):
async with self._lock:
if self.channel is None or self.channel.is_closed:
await self.start(prefetch)
async def stop(self):
if self.connection:
try:
await asyncio.wait_for(self.connection.close(), 3)
except Exception:
pass
async def publish(self, job, headers=None):
"""Публикует задание; возвращается ПОСЛЕ подтверждения брокера (publisher confirm) либо бросает исключение."""
await self.ensure()
body = json.dumps(job, separators=(",", ":")).encode("utf-8")
message = Message(body, delivery_mode=DeliveryMode.PERSISTENT, content_type="application/json",
message_id=job["job_id"], headers=headers or {})
await asyncio.wait_for(self.channel.default_exchange.publish(message, routing_key=QUEUE, mandatory=True),
config.BROKER_TIMEOUT)
async def queue_depth(self):
await self.ensure()
q = await self.channel.declare_queue(QUEUE, durable=True, passive=True)
return q.declaration_result.message_count
async def ping(self):
return self.channel is not None and not self.channel.is_closed
+112
View File
@@ -0,0 +1,112 @@
"""Конфигурация из переменных окружения (см. .env.example)."""
import os
def _int(name, default):
value = os.environ.get(name)
return int(value) if value not in (None, "") else default
def _float(name, default):
value = os.environ.get(name)
return float(value) if value not in (None, "") else default
def _secret(name, default=""):
"""Значение из переменной NAME либо из файла NAME_FILE (Docker secrets); файл читается один раз."""
if os.environ.get(name):
return os.environ[name]
path = os.environ.get(name + "_FILE")
if path and os.path.exists(path):
with open(path, encoding="utf-8") as fh:
return fh.read().strip()
return default
APP_HOST = os.environ.get("APP_HOST", "0.0.0.0")
APP_PORT = _int("APP_PORT", 8080)
APP_ENV = os.environ.get("APP_ENV", "lab")
DATABASE_URL = _secret("DATABASE_URL")
DB_POOL_MIN = _int("DB_POOL_MIN", 2)
DB_POOL_MAX = _int("DB_POOL_MAX", 10)
DB_ACQUIRE_TIMEOUT = _float("DB_ACQUIRE_TIMEOUT", 2.0) # ожидание свободного соединения пула
DB_COMMAND_TIMEOUT = _float("DB_COMMAND_TIMEOUT", 5.0) # таймаут одного запроса
DB_CONNECT_TIMEOUT = _float("DB_CONNECT_TIMEOUT", 2.0)
VALKEY_URL = os.environ.get("VALKEY_URL", "") # пусто = кэш сессий выключен
VALKEY_TIMEOUT = _float("VALKEY_TIMEOUT", 0.2)
VALKEY_BREAKER_SECONDS = _float("VALKEY_BREAKER_SECONDS", 5.0)
SESSION_TTL_SECONDS = _int("SESSION_TTL_SECONDS", 3600)
SESSION_CACHE_TTL_SECONDS = _int("SESSION_CACHE_TTL_SECONDS", 60)
ARGON2_TIME_COST = _int("ARGON2_TIME_COST", 2)
ARGON2_MEMORY_KIB = _int("ARGON2_MEMORY_KIB", 19456) # 19 MiB, рекомендация OWASP
ARGON2_PARALLELISM = _int("ARGON2_PARALLELISM", 1)
HASH_CONCURRENCY = _int("HASH_CONCURRENCY", 4)
MAX_BODY_BYTES = _int("MAX_BODY_BYTES", 65536)
MAX_INFLIGHT = _int("MAX_INFLIGHT", 64)
LIMIT_CONCURRENCY = _int("LIMIT_CONCURRENCY", 1000)
RETRY_AFTER_SECONDS = _int("RETRY_AFTER_SECONDS", 1)
MIGRATIONS_DIR = os.environ.get("MIGRATIONS_DIR", os.path.join(os.path.dirname(__file__), "..", "..", "migrations"))
DRAIN_SECONDS = _float("DRAIN_SECONDS", 4.0) # после SIGTERM: readiness=503, сервер ещё принимает запросы
GRACEFUL_SHUTDOWN_SECONDS = _int("GRACEFUL_SHUTDOWN_SECONDS", 8)
KEEPALIVE_SECONDS = _int("KEEPALIVE_SECONDS", 5)
IDEMPOTENCY_TTL_SECONDS = _int("IDEMPOTENCY_TTL_SECONDS", 24 * 3600)
# --- S3 / вложения (lab4) ---
S3_ENDPOINT = os.environ.get("S3_ENDPOINT", "")
S3_BUCKET = os.environ.get("S3_BUCKET", "messenger-attachments")
S3_REGION = os.environ.get("S3_REGION", "us-east-1")
S3_ACCESS_KEY = _secret("S3_ACCESS_KEY")
S3_SECRET_KEY = _secret("S3_SECRET_KEY")
S3_CONNECT_TIMEOUT = _float("S3_CONNECT_TIMEOUT", 2.0)
S3_READ_TIMEOUT = _float("S3_READ_TIMEOUT", 15.0)
S3_POOL = _int("S3_POOL", 20)
MAX_UPLOAD_BYTES = _int("MAX_UPLOAD_BYTES", 10 * 1024 * 1024) # 10 MiB = 10 485 760
MAX_IMAGE_PIXELS = _int("MAX_IMAGE_PIXELS", 40_000_000)
UPLOAD_CONCURRENCY = _int("UPLOAD_CONCURRENCY", 4) # одновременных загрузок на реплику
UPLOAD_QUEUE = _int("UPLOAD_QUEUE", 16) # ожидающих загрузок; сверх — 429
IMAGE_THREADS = _int("IMAGE_THREADS", 2)
DOWNLOAD_CHUNK_BYTES = _int("DOWNLOAD_CHUNK_BYTES", 65536)
ORPHAN_GRACE_SECONDS = _int("ORPHAN_GRACE_SECONDS", 900) # объект без метаданных моложе этого возраста не трогаем
FAULT_INJECTION = {x for x in os.environ.get("FAULT_INJECTION", "").split(",") if x} # db_after_put, s3_delete_fails (эксперименты)
# --- lab5: брокер, outbox, worker ---
SERVICE_NAME = os.environ.get("SERVICE_NAME", "messenger-api")
BROKER_URL = _secret("BROKER_URL")
BROKER_TIMEOUT = _float("BROKER_TIMEOUT", 5.0)
RELAY_BATCH = _int("RELAY_BATCH", 50)
RELAY_INTERVAL = _float("RELAY_INTERVAL", 1.0)
WORKER_CONCURRENCY = _int("WORKER_CONCURRENCY", 2) # prefetch == число одновременных заданий == потоков обработки
LEASE_SECONDS = _int("LEASE_SECONDS", 60) # аренда задания; истекла -> sweeper возвращает в очередь
JOB_TIMEOUT = _float("JOB_TIMEOUT", 45.0) # лимит времени одной обработки (меньше lease)
MAX_ATTEMPTS = _int("MAX_ATTEMPTS", 5)
RETRY_BASE_DELAY = _float("RETRY_BASE_DELAY", 2.0)
RETRY_MAX_DELAY = _float("RETRY_MAX_DELAY", 60.0)
SWEEP_INTERVAL = _float("SWEEP_INTERVAL", 5.0)
STAGING_FAILED_RETENTION_SECONDS = _int("STAGING_FAILED_RETENTION_SECONDS", 7 * 24 * 3600)
MAX_QUEUED_JOBS = _int("MAX_QUEUED_JOBS", 200) # backpressure: при превышении upload -> 503 + Retry-After
BACKLOG_RETRY_AFTER_SECONDS = _int("BACKLOG_RETRY_AFTER_SECONDS", 5)
# --- lab6: наблюдаемость ---
OTEL_ENABLED = os.environ.get("OTEL_ENABLED", "1") not in ("0", "false", "")
OTEL_ENDPOINT = os.environ.get("OTEL_EXPORTER_OTLP_ENDPOINT", "http://otel-collector:4318")
OTEL_TRACE_RATIO = _float("OTEL_TRACES_SAMPLER_ARG", 1.0) # head-sampling: доля трасс (ParentBased)
OTEL_METRIC_INTERVAL_MS = _int("OTEL_METRIC_EXPORT_INTERVAL", 5000)
OTEL_QUEUE_SIZE = _int("OTEL_QUEUE_SIZE", 2048)
OTEL_BATCH_SIZE = _int("OTEL_BATCH_SIZE", 512) # спанов в одном экспорте (max_export_batch_size) # ограниченная очередь SDK; при переполнении данные отбрасываются
SERVICE_VERSION = os.environ.get("APP_REVISION", "dev")
INSTANCE_ID = (os.environ.get("HOSTNAME") or "local")[:12]
# Внедрение отказов в HTTP-путь для экспериментов наблюдаемости (по умолчанию пусто): "http500:0.02" — 2 % запросов дают 500;
# "slow:0.01:0.5" — 1 % запросов задерживаются на 0.5 с. Health-маршруты не затрагиваются.
def _fault_rate(prefix):
for item in FAULT_INJECTION:
if item.startswith(prefix + ":"):
return item.split(":")[1:]
return None
HTTP_FAULT_500 = _fault_rate("http500")
HTTP_FAULT_SLOW = _fault_rate("slow")
+88
View File
@@ -0,0 +1,88 @@
"""PostgreSQL: ограниченный пул asyncpg, таймауты, перевод сбоев зависимости в 503."""
import asyncio
from contextlib import asynccontextmanager
import asyncpg
from . import config, telemetry
from .web import log, unavailable
DEPENDENCY_ERRORS = (asyncpg.PostgresConnectionError, asyncpg.InterfaceError, asyncpg.exceptions.OperatorInterventionError,
asyncpg.exceptions.InsufficientResourcesError, asyncio.TimeoutError, OSError)
_RETRYABLE = (asyncpg.ConnectionDoesNotExistError, asyncpg.InterfaceError, ConnectionResetError)
class Database:
def __init__(self):
self.pool = None
self.waiting = 0
telemetry.gauge("db.client.connections.usage", self._usage, "{connection}", "Соединения пула PostgreSQL")
telemetry.gauge("db.client.connections.waiting", lambda: [(self.waiting, {})], "{request}", "Запросы, ожидающие соединение")
def _usage(self):
if self.pool is None:
return []
size, idle = self.pool.get_size(), self.pool.get_idle_size()
return [(size - idle, {"state": "used"}), (idle, {"state": "idle"})]
async def start(self):
if not config.DATABASE_URL:
log("error", "database_url_missing")
return
# min_size=0: приложение стартует и без БД (readiness покажет 503), пул подключается лениво.
self.pool = await asyncpg.create_pool(
dsn=config.DATABASE_URL, min_size=0, max_size=config.DB_POOL_MAX,
command_timeout=config.DB_COMMAND_TIMEOUT, timeout=config.DB_CONNECT_TIMEOUT,
max_inactive_connection_lifetime=60,
server_settings={"application_name": "messenger-api", "timezone": "UTC"})
async def stop(self):
if self.pool:
try:
await asyncio.wait_for(self.pool.close(), 3)
except Exception:
self.pool.terminate()
@asynccontextmanager
async def conn(self):
if self.pool is None:
raise unavailable("База данных не настроена")
try:
self.waiting += 1
try:
acquired = await self.pool.acquire(timeout=config.DB_ACQUIRE_TIMEOUT)
finally:
self.waiting -= 1
try:
with telemetry.child_span("db.connection", "client", {"db.system": "postgresql"}):
yield acquired
finally:
await self.pool.release(acquired)
except DEPENDENCY_ERRORS as exc:
log("warning", "db_unavailable", error_type=type(exc).__name__)
raise unavailable() from exc
async def _run(self, method, sql, args, retry):
for attempt in (0, 1):
try:
async with self.conn() as c:
return await getattr(c, method)(sql, *args)
except Exception as exc:
if attempt == 0 and retry and isinstance(exc.__cause__, _RETRYABLE):
continue # разорванное соединение пула (рестарт БД): один повтор только для чтения
raise
async def fetch(self, sql, *args):
return await self._run("fetch", sql, args, True)
async def fetchrow(self, sql, *args, retry=True):
return await self._run("fetchrow", sql, args, retry)
async def fetchval(self, sql, *args):
return await self._run("fetchval", sql, args, True)
async def execute(self, sql, *args):
return await self._run("execute", sql, args, False)
async def ping(self):
return await asyncio.wait_for(self.fetchval("SELECT 1"), 1.0)
+102
View File
@@ -0,0 +1,102 @@
"""Проверка загружаемых изображений ДО полного декодирования: сигнатура, формат Pillow, размеры, целостность.
PNG/JPEG определяются по содержимому (а не по Content-Type). Лимит пикселей проверяется по заголовку,
поэтому «decompression bomb» отклоняется без декодирования. Целостность: PNG — verify() (CRC всех чанков, без
распаковки в память), JPEG — декодирование в режиме draft (1/8 размера => около 1/64 памяти).
"""
import struct
import zlib
from PIL import Image, UnidentifiedImageError
from . import config
Image.MAX_IMAGE_PIXELS = None # лимит применяем сами по заголовку (ниже), без предупреждений Pillow
PNG_SIGNATURE = b"\x89PNG\r\n\x1a\n"
class ImageRejected(Exception):
def __init__(self, status, code, message):
super().__init__(message)
self.status, self.code, self.message = status, code, message
def sniff(head):
if head.startswith(PNG_SIGNATURE):
return "image/png"
if head[:3] == b"\xff\xd8\xff":
return "image/jpeg"
return None
def _check_png_stream(fileobj, width, height):
"""Целостность потока IDAT без аллокации изображения: распаковка порциями, проверка конца потока и размера данных."""
fileobj.seek(8)
inflater = zlib.decompressobj()
total, channels, bit_depth, interlaced = 0, 0, 0, 0
while True:
head = fileobj.read(8)
if len(head) < 8:
raise ValueError("truncated")
length, kind = struct.unpack(">I4s", head)
if length > 0x7FFFFFFF:
raise ValueError("chunk")
if kind == b"IHDR":
data = fileobj.read(length)
bit_depth, color_type, _, _, interlaced = struct.unpack(">BBBBB", data[8:13])
channels = {0: 1, 2: 3, 3: 1, 4: 2, 6: 4}[color_type]
fileobj.read(4)
elif kind == b"IDAT":
remaining = length
while remaining:
piece = fileobj.read(min(remaining, 65536))
if not piece:
raise ValueError("truncated")
remaining -= len(piece)
pending = piece
while pending:
total += len(inflater.decompress(pending, 1 << 20))
pending = inflater.unconsumed_tail
fileobj.read(4)
elif kind == b"IEND":
break
else:
fileobj.seek(length + 4, 1)
if not inflater.eof:
raise ValueError("zlib stream incomplete")
if not interlaced:
row = (width * channels * bit_depth + 7) // 8
if total != height * (row + 1):
raise ValueError("unexpected image data size")
def validate(fileobj, declared_type):
"""Возвращает (content_type, width, height) либо бросает ImageRejected (415 — не то/повреждено, 413 — много пикселей)."""
fileobj.seek(0)
detected = sniff(fileobj.read(16))
if detected is None or detected != declared_type:
raise ImageRejected(415, "unsupported_media_type", "Содержимое не соответствует PNG/JPEG или заявленному Content-Type")
fileobj.seek(0)
try:
with Image.open(fileobj) as img:
width, height = img.size
fmt = img.format
except (UnidentifiedImageError, OSError, ValueError, SyntaxError, Image.DecompressionBombError):
raise ImageRejected(415, "unsupported_media_type", "Изображение повреждено или не поддерживается") from None
if fmt not in ("PNG", "JPEG") or width < 1 or height < 1:
raise ImageRejected(415, "unsupported_media_type", "Неподдерживаемый формат изображения")
if width * height > config.MAX_IMAGE_PIXELS:
raise ImageRejected(413, "payload_too_large", "Слишком много пикселей в изображении")
fileobj.seek(0)
try:
img = Image.open(fileobj)
if fmt == "PNG":
img.verify()
_check_png_stream(fileobj, width, height)
else:
img.draft("RGB", (max(1, width // 8), max(1, height // 8)))
img.load()
img.close()
except Exception: # любая ошибка декодера = повреждённый файл
raise ImageRejected(415, "unsupported_media_type", "Изображение повреждено") from None
return detected, width, height
+464
View File
@@ -0,0 +1,464 @@
"""ASGI-приложение мессенджера (lab2): REST API, PostgreSQL, сессии Bearer, кэш Valkey."""
import asyncio
import contextlib
from concurrent.futures import ThreadPoolExecutor
import base64
import binascii
import hashlib
import json
import os
import re
import time
import uuid
import asyncpg
from . import config, passwords, telemetry
from .attachments import AttachmentMixin, StreamResponse
from .db import DEPENDENCY_ERRORS, Database
from .broker import Broker
from .outbox import Relay
from .s3 import S3
from .sessions import SessionCache, Sessions
from .web import (ApiError, Request, bad, dt_to_us, encode_cursor, format_dt, log, new_request_id, page_params,
parse_json_object, parse_uuid, respond, unavailable, us_to_dt, valid_text, USERNAME_RE)
ROUTES = [
("health_live", re.compile(r"/health/live"), {"GET"}),
("health_ready", re.compile(r"/health/ready"), {"GET"}),
("users", re.compile(r"/api/v1/users"), {"POST"}),
("users_me", re.compile(r"/api/v1/users/me"), {"GET"}),
("create_session", re.compile(r"/api/v1/auth/sessions"), {"POST"}),
("session_current", re.compile(r"/api/v1/auth/sessions/current"), {"DELETE"}),
("chats", re.compile(r"/api/v1/chats"), {"GET", "POST"}),
("chat", re.compile(r"/api/v1/chats/(?P<chat_id>[^/]+)"), {"GET"}),
("messages", re.compile(r"/api/v1/chats/(?P<chat_id>[^/]+)/messages"), {"GET", "POST"}),
("upload_attachment", re.compile(r"/api/v1/chats/(?P<chat_id>[^/]+)/attachments"), {"POST"}),
("get_attachment", re.compile(r"/api/v1/attachments/(?P<attachment_id>[^/]+)"), {"GET"}),
("download_attachment", re.compile(r"/api/v1/attachments/(?P<attachment_id>[^/]+)/content"), {"GET"}),
]
ROUTE_LABELS = {
"health_live": "/health/live", "health_ready": "/health/ready", "users": "/api/v1/users",
"users_me": "/api/v1/users/me", "create_session": "/api/v1/auth/sessions",
"session_current": "/api/v1/auth/sessions/current", "chats": "/api/v1/chats",
"chat": "/api/v1/chats/{chat_id}", "messages": "/api/v1/chats/{chat_id}/messages",
"upload_attachment": "/api/v1/chats/{chat_id}/attachments", "get_attachment": "/api/v1/attachments/{attachment_id}",
"download_attachment": "/api/v1/attachments/{attachment_id}/content",
}
TOKEN_RE = re.compile(r"[A-Za-z0-9_-]{32,128}")
IDEMPOTENCY_KEY_RE = re.compile(r"[A-Za-z0-9._:-]{1,128}")
BEARER = 'Bearer realm="messenger"'
BASIC = 'Basic realm="messenger", charset="UTF-8"'
# Идентификатор реплики: имя хоста контейнера (12 hex, == короткий ID контейнера) -> легко сопоставить с docker ps.
INSTANCE_ID = os.environ.get("INSTANCE_ID") or (os.environ.get("HOSTNAME") or uuid.uuid4().hex)[:12]
MSG_ATTS = ("COALESCE((SELECT array_agg(ma.attachment_id ORDER BY ma.position) FROM message_attachments ma "
"WHERE ma.message_id = messages.id), '{}'::uuid[]) AS attachment_ids")
CHAT_SELECT = ("SELECT c.id, c.title, c.created_at, "
"(SELECT array_agg(m2.user_id ORDER BY m2.ord) FROM chat_members m2 WHERE m2.chat_id = c.id) AS member_ids ")
def user_json(row):
return {"id": str(row["id"]), "username": row["username"], "display_name": row["display_name"],
"created_at": format_dt(row["created_at"])}
def chat_json(row):
return {"id": str(row["id"]), "title": row["title"], "member_ids": [str(x) for x in row["member_ids"]],
"created_at": format_dt(row["created_at"])}
def message_json(row):
return {"id": str(row["id"]), "chat_id": str(row["chat_id"]), "sender_id": str(row["sender_id"]),
"text": row["text"], "created_at": format_dt(row["created_at"]),
"attachment_ids": [str(x) for x in (row["attachment_ids"] or [])]}
def unauthorized(challenge=BEARER, message="Требуется действительная сессия"):
return ApiError(401, "unauthorized", message, {"www-authenticate": challenge})
class Messenger(AttachmentMixin):
def __init__(self):
self.db = Database()
self.s3 = S3()
self.broker = Broker()
self.relay = Relay(self.db, self.broker)
self.image_pool = ThreadPoolExecutor(max_workers=config.IMAGE_THREADS, thread_name_prefix="image")
self._init_attachments()
self.cache = SessionCache()
self.sessions = Sessions(self.db, self.cache)
self.inflight = 0
self.draining = False
# Метрики очереди считает API (любая реплика), а не worker: при остановке worker алерт по возрасту задач продолжает работать.
self.queue_stats = (0, 0.0)
telemetry.gauge("messenger.queue.depth", lambda: [(self.queue_stats[0], {"queue.name": "attachment.process"})], "{job}", "queued + processing")
telemetry.gauge("messenger.queue.oldest_job_age", lambda: [(self.queue_stats[1], {"queue.name": "attachment.process"})], "s", "Возраст старейшего задания")
# --- аутентификация ------------------------------------------------------
@staticmethod
def bearer_token(req):
header = req.headers.get("authorization", "")
scheme, _, token = header.partition(" ")
if scheme.lower() != "bearer" or not TOKEN_RE.fullmatch(token.strip()):
raise unauthorized()
return token.strip()
async def authenticate(self, req):
token = self.bearer_token(req)
user_id = await self.sessions.resolve(token)
if user_id is None:
raise unauthorized()
req.user_id, req.token = user_id, token
return user_id
# --- обработчики -----------------------------------------------------------
async def health_live(self, req, params):
return 200, {"status": "ok"}
async def health_ready(self, req, params):
# Критичная зависимость — PostgreSQL. Valkey — необязательный кэш: его отказ не останавливает API.
if self.draining:
raise unavailable("Сервис останавливается")
try:
await self.db.ping()
except ApiError:
raise
except Exception:
raise unavailable() from None
return 200, {"status": "ok"}
async def users(self, req, params):
req.require_json_media_type()
data = parse_json_object(await req.body(), {"username", "display_name", "password"},
{"username", "display_name", "password"})
username = data["username"]
if not isinstance(username, str) or not USERNAME_RE.fullmatch(username):
raise bad("username должен соответствовать [a-z0-9_]{3,32}")
display_name = valid_text(data["display_name"], "display_name", 100)
password = valid_text(data["password"], "password", 128, allow_empty=True)
if len(password) < 12:
raise bad("password должен содержать от 12 до 128 символов")
pw_hash = await passwords.hash_password(password)
row = await self.db.fetchrow(
"INSERT INTO users (id, username, display_name, password_hash) VALUES ($1, $2, $3, $4) "
"ON CONFLICT (username) DO NOTHING RETURNING id, username, display_name, created_at",
uuid.uuid4(), username, display_name, pw_hash)
if row is None:
raise ApiError(409, "conflict", "username уже занят")
return 201, user_json(row)
async def users_me(self, req, params):
user_id = await self.authenticate(req)
row = await self.db.fetchrow("SELECT id, username, display_name, created_at FROM users WHERE id = $1", user_id)
if row is None:
raise unauthorized()
return 200, user_json(row)
async def create_session(self, req, params):
header = req.headers.get("authorization", "")
scheme, _, value = header.partition(" ")
if scheme.lower() != "basic" or not value.strip():
raise unauthorized(BASIC, "Неверные учётные данные")
try:
decoded = base64.b64decode(value.strip(), validate=True).decode("utf-8")
username, sep, password = decoded.partition(":")
if not sep:
raise ValueError
except (binascii.Error, ValueError, UnicodeDecodeError):
raise unauthorized(BASIC, "Неверные учётные данные") from None
row = None
if USERNAME_RE.fullmatch(username):
row = await self.db.fetchrow("SELECT id, username, display_name, created_at, password_hash FROM users "
"WHERE username = $1", username)
ok = await passwords.verify_password(row["password_hash"] if row else None, password)
if not ok:
raise unauthorized(BASIC, "Неверные учётные данные")
token, expires_at = await self.sessions.create(row["id"])
return 201, {"token": token, "token_type": "Bearer", "expires_at": format_dt(expires_at), "user": user_json(row)}
async def session_current(self, req, params):
await self.authenticate(req)
await self.sessions.revoke(req.token)
return 204, None
async def chats(self, req, params):
user_id = await self.authenticate(req)
if req.method == "GET":
limit, after = page_params(req.query, "c", user_id)
if after is None:
rows = await self.db.fetch(
CHAT_SELECT + "FROM chat_members m JOIN chats c ON c.id = m.chat_id WHERE m.user_id = $1 "
"ORDER BY m.chat_created_at, m.chat_id LIMIT $2", user_id, limit + 1)
else:
rows = await self.db.fetch(
CHAT_SELECT + "FROM chat_members m JOIN chats c ON c.id = m.chat_id WHERE m.user_id = $1 "
"AND (m.chat_created_at, m.chat_id) > ($2, $3) ORDER BY m.chat_created_at, m.chat_id LIMIT $4",
user_id, us_to_dt(after[0]), uuid.UUID(after[1]), limit + 1)
more = len(rows) > limit
rows = rows[:limit]
cursor = encode_cursor("c", user_id, (dt_to_us(rows[-1]["created_at"]), str(rows[-1]["id"]))) if more else None
return 200, {"items": [chat_json(r) for r in rows], "next_cursor": cursor}
req.require_json_media_type()
data = parse_json_object(await req.body(), {"title", "member_ids"}, {"title", "member_ids"})
title = valid_text(data["title"], "title", 100)
members = data["member_ids"]
if not isinstance(members, list) or not 1 <= len(members) <= 100:
raise bad("member_ids должен содержать от 1 до 100 элементов")
ids = [parse_uuid(m, "member_id") for m in members]
if len(set(ids)) != len(ids):
raise bad("member_ids содержит дубликаты")
if user_id not in ids:
raise bad("Создатель должен быть в member_ids")
chat_id = uuid.uuid4()
uuids = [uuid.UUID(i) for i in ids]
try:
async with self.db.conn() as c:
async with c.transaction(): # чат и участники создаются атомарно
# Существование участников проверяет внешний ключ chat_members.user_id: при неизвестном
# участнике вставка падает и откатывается вся транзакция, включая уже созданный чат.
created = await c.fetchval("INSERT INTO chats (id, title) VALUES ($1, $2) RETURNING created_at",
chat_id, title)
await c.execute(
"INSERT INTO chat_members (chat_id, user_id, chat_created_at, ord) "
"SELECT $1, u, $2, o FROM unnest($3::uuid[]) WITH ORDINALITY AS t(u, o)",
chat_id, created, uuids)
except asyncpg.ForeignKeyViolationError:
raise bad("Неизвестный участник") from None
return 201, {"id": str(chat_id), "title": title, "member_ids": ids, "created_at": format_dt(created)}
async def _require_member(self, user_id, params):
chat_id = parse_uuid(params["chat_id"], "chat_id")
ok = await self.db.fetchval("SELECT 1 FROM chat_members WHERE chat_id = $1 AND user_id = $2", chat_id, user_id)
if not ok:
raise ApiError(404, "not_found", "Чат не найден")
return chat_id
async def chat(self, req, params):
user_id = await self.authenticate(req)
chat_id = parse_uuid(params["chat_id"], "chat_id")
row = await self.db.fetchrow(
CHAT_SELECT + "FROM chats c JOIN chat_members m ON m.chat_id = c.id AND m.user_id = $2 WHERE c.id = $1",
chat_id, user_id)
if row is None:
raise ApiError(404, "not_found", "Чат не найден")
return 200, chat_json(row)
async def messages(self, req, params):
user_id = await self.authenticate(req)
if req.method == "POST":
row = await self._post_message(req, parse_uuid(params["chat_id"], "chat_id"), user_id)
return 201, message_json(row)
chat_id = await self._require_member(user_id, params)
if req.method == "GET":
limit, after = page_params(req.query, "m", chat_id)
if after is None:
rows = await self.db.fetch(
"SELECT id, chat_id, sender_id, text, created_at, " + MSG_ATTS + " FROM messages WHERE chat_id = $1 "
"ORDER BY created_at, id LIMIT $2", chat_id, limit + 1)
else:
rows = await self.db.fetch(
"SELECT id, chat_id, sender_id, text, created_at, " + MSG_ATTS + " FROM messages WHERE chat_id = $1 "
"AND (created_at, id) > ($2, $3) ORDER BY created_at, id LIMIT $4",
chat_id, us_to_dt(after[0]), uuid.UUID(after[1]), limit + 1)
more = len(rows) > limit
rows = rows[:limit]
cursor = encode_cursor("m", chat_id, (dt_to_us(rows[-1]["created_at"]), str(rows[-1]["id"]))) if more else None
return 200, {"items": [message_json(r) for r in rows], "next_cursor": cursor}
async def _post_message(self, req, chat_id, user_id):
# Сначала разбор запроса; ошибка валидации (400/415) возвращается только участнику чата — постороннему всегда 404.
try:
key = req.headers.get("idempotency-key", "")
if not IDEMPOTENCY_KEY_RE.fullmatch(key):
raise bad("Требуется заголовок Idempotency-Key ([A-Za-z0-9._:-]{1,128})")
req.require_json_media_type()
data = parse_json_object(await req.body(), {"text"}, {"text", "attachment_ids"})
raw_atts = data.get("attachment_ids", [])
if not isinstance(raw_atts, list) or len(raw_atts) > 10:
raise bad("attachment_ids: массив до 10 элементов")
atts = [parse_uuid(a, "attachment_id") for a in raw_atts]
if len(set(atts)) != len(atts):
raise bad("attachment_ids содержит дубликаты")
text = valid_text(data["text"], "text", 4000, allow_empty=bool(atts))
except ApiError:
await self._require_member(user_id, {"chat_id": chat_id})
raise
# Нормализованное представление запроса: порядок ключей JSON не важен, отсутствие attachment_ids == [].
request_hash = hashlib.sha256(json.dumps({"text": text, "attachment_ids": atts}, sort_keys=True,
ensure_ascii=False).encode("utf-8")).digest()
row = await self.db.fetchrow("SELECT * FROM send_message($1, $2, $3, $4, $5, $6, $7)", uuid.UUID(chat_id),
uuid.UUID(user_id), key, request_hash, text, float(config.IDEMPOTENCY_TTL_SECONDS),
[uuid.UUID(a) for a in atts], retry=False)
if row["outcome"] == "not_member":
raise ApiError(404, "not_found", "Чат не найден")
if row["outcome"] == "attachment_not_found":
raise ApiError(404, "not_found", "Вложение не найдено")
if row["outcome"] == "attachment_failed":
raise ApiError(409, "conflict", "Вложение не удалось обработать")
if bytes(row["request_hash"]) != request_hash:
raise ApiError(409, "conflict", "Idempotency-Key уже использован с другим телом запроса")
return {"id": row["msg_id"], "chat_id": row["msg_chat"], "sender_id": row["msg_sender"], "text": row["msg_text"],
"created_at": row["msg_created"], "attachment_ids": row["msg_atts"]}
# --- ASGI ----------------------------------------------------------------------
async def __call__(self, scope, receive, send):
if scope["type"] == "lifespan":
return await self.lifespan(receive, send)
if scope["type"] != "http":
return
started = time.perf_counter()
request_id = new_request_id()
route_name, status, headers = "unknown", 500, {}
raw_headers = {k.decode("latin-1").lower(): v.decode("latin-1") for k, v in scope["headers"]}
carrier = {k: raw_headers[k] for k in ("traceparent", "tracestate") if k in raw_headers}
is_health = scope.get("path", "").startswith("/health") # проверки LB раз в секунду: без спанов и access-log (метрики остаются)
span_cm = (contextlib.nullcontext() if is_health else
telemetry.span("HTTP %s" % scope.get("method"), "server", {"http.request.method": scope.get("method")}, carrier))
sp = span_cm.__enter__()
try:
req = Request(scope, receive, request_id)
handler, params, route_name = self.route(req.method, req.path)
telemetry.set_attr(sp, "http.route", ROUTE_LABELS.get(route_name, "unmatched"))
req.span = sp
if not route_name.startswith("health") and self.inflight >= config.MAX_INFLIGHT:
raise ApiError(429, "rate_limited", "Слишком много одновременных запросов",
{"retry-after": str(config.RETRY_AFTER_SECONDS)})
self.inflight += 1
try:
await asyncio.sleep(0)
if not route_name.startswith("health") and (config.HTTP_FAULT_500 or config.HTTP_FAULT_SLOW):
await self.inject_http_faults()
status, body = await handler(req, params)
finally:
self.inflight -= 1
except ApiError as exc:
status, headers = exc.status, exc.headers
body = {"error": {"code": exc.code, "message": exc.message, "request_id": request_id}}
except DEPENDENCY_ERRORS as exc:
log("warning", "dependency_error", request_id=request_id, error_type=type(exc).__name__)
status, headers = 503, {"retry-after": str(config.RETRY_AFTER_SECONDS)}
body = {"error": {"code": "unavailable", "message": "Зависимость временно недоступна", "request_id": request_id}}
except Exception as exc:
log("error", "unhandled_exception", request_id=request_id, error_type=type(exc).__name__)
status = 500
body = {"error": {"code": "internal_error", "message": "Внутренняя ошибка", "request_id": request_id}}
route_label = ROUTE_LABELS.get(route_name, "unmatched")
duration = time.perf_counter() - started
telemetry.set_attr(sp, "http.response.status_code", status)
telemetry.set_attr(sp, "http.request_id", request_id)
if sp is not None:
sp.update_name("HTTP %s %s" % (scope.get("method"), route_label))
if status >= 500:
telemetry.mark_error(sp, "HTTP %d" % status)
telemetry.record_http(scope.get("method"), route_label, status, duration)
if isinstance(body, StreamResponse):
try:
await body.send(send, request_id, INSTANCE_ID)
except Exception as exc: # клиент оборвал загрузку/ошибка S3 посреди потока
log("warning", "stream_aborted", request_id=request_id, error_type=type(exc).__name__)
else:
await respond(send, status, body, request_id, headers, INSTANCE_ID)
if os.environ.get("ACCESS_LOG", "1") != "0" and (not is_health or status >= 500):
log("info", "request", request_id=request_id, method=scope.get("method"),
route=route_label, status=status, duration_ms=round(duration * 1000, 3))
span_cm.__exit__(None, None, None)
async def janitor(self):
"""Удаляет давно просроченные ключи идемпотентности и сессии (безопасно запускать на каждой реплике)."""
import random
await asyncio.sleep(random.uniform(5, 30))
while True:
try:
await self.db.execute("DELETE FROM idempotency_keys WHERE ctid IN (SELECT ctid FROM idempotency_keys "
"WHERE created_at < now() - make_interval(secs => $1) LIMIT 5000)",
float(config.IDEMPOTENCY_TTL_SECONDS) + 3600)
await self.db.execute("DELETE FROM sessions WHERE ctid IN (SELECT ctid FROM sessions "
"WHERE expires_at < now() - interval '1 day' LIMIT 5000)")
except Exception as exc:
log("warning", "janitor_failed", error_type=type(exc).__name__)
await asyncio.sleep(3600)
async def queue_stats_loop(self):
while True:
try:
row = await self.db.fetchrow("SELECT count(*) AS n, coalesce(extract(epoch FROM now() - min(accepted_at)), 0) AS age "
"FROM attachments WHERE status IN ('queued', 'processing')")
self.queue_stats = (row["n"], float(row["age"]))
except Exception:
pass
await asyncio.sleep(2)
@staticmethod
async def inject_http_faults():
import random
if config.HTTP_FAULT_SLOW and random.random() < float(config.HTTP_FAULT_SLOW[0]):
await asyncio.sleep(float(config.HTTP_FAULT_SLOW[1]))
if config.HTTP_FAULT_500 and random.random() < float(config.HTTP_FAULT_500[0]):
raise RuntimeError("injected fault")
def route(self, method, path):
for name, pattern, methods in ROUTES:
m = pattern.fullmatch(path)
if m:
if method not in methods:
raise ApiError(405, "invalid_request", "Метод не поддерживается", {"allow": ", ".join(sorted(methods))})
return getattr(self, name), m.groupdict(), name
raise ApiError(404, "not_found", "Ресурс не найден")
async def lifespan(self, receive, send):
while True:
event = await receive()
if event["type"] == "lifespan.startup":
telemetry.setup(config.SERVICE_NAME)
telemetry.process_metrics()
await self.db.start()
await self.cache.start()
await self.s3.start()
self._relay_task = asyncio.create_task(self.relay.run())
self._queue_task = asyncio.create_task(self.queue_stats_loop())
self._janitor = asyncio.create_task(self.janitor())
log("info", "startup", env=config.APP_ENV, instance=INSTANCE_ID, revision=os.environ.get("APP_REVISION", "dev"), cache=bool(config.VALKEY_URL),
session_ttl=config.SESSION_TTL_SECONDS, pool_max=config.DB_POOL_MAX)
await send({"type": "lifespan.startup.complete"})
elif event["type"] == "lifespan.shutdown":
self.draining = True
self._janitor.cancel()
await self.cache.stop()
telemetry.shutdown()
self.relay.stop()
self._relay_task.cancel()
self._queue_task.cancel()
await self.broker.stop()
await self.s3.stop()
await self.db.stop()
log("info", "shutdown")
await send({"type": "lifespan.shutdown.complete"})
return
async def overloaded_app(scope, receive, send):
"""Ответ uvicorn при превышении --limit-concurrency в контрактном формате."""
if scope["type"] != "http":
return
request_id = new_request_id()
body = {"error": {"code": "unavailable", "message": "Сервер перегружен: слишком много соединений",
"request_id": request_id}}
await respond(send, 503, body, request_id, {"retry-after": str(config.RETRY_AFTER_SECONDS), "connection": "close"}, INSTANCE_ID)
def _install_overload_response():
for name in ("uvicorn.protocols.http.httptools_impl", "uvicorn.protocols.http.h11_impl"):
try:
module = __import__(name, fromlist=["service_unavailable"])
except ImportError:
continue
module.service_unavailable = overloaded_app
def create_app():
_install_overload_response()
return Messenger()
app = create_app()
+64
View File
@@ -0,0 +1,64 @@
"""Применение версионируемых SQL-миграций: python -m app.migrate
Свойства:
* повторный запуск ничего не делает (таблица schema_migrations);
* одновременный запуск нескольких экземпляров безопасен (advisory lock);
* каждая миграция — одна транзакция вместе с записью о применении;
* ждёт готовности PostgreSQL до MIGRATE_WAIT_SECONDS.
"""
import asyncio
import hashlib
import os
import sys
import time
import asyncpg
from . import config
from .web import log
LOCK_ID = 7352001
async def connect(deadline):
while True:
try:
return await asyncpg.connect(config.DATABASE_URL, timeout=3)
except (OSError, asyncpg.PostgresError, asyncio.TimeoutError) as exc:
if time.monotonic() > deadline:
raise
log("info", "waiting_for_database", error_type=type(exc).__name__)
await asyncio.sleep(1)
async def main():
wait = float(os.environ.get("MIGRATE_WAIT_SECONDS", "60"))
conn = await connect(time.monotonic() + wait)
try:
await conn.execute("SELECT pg_advisory_lock($1)", LOCK_ID)
await conn.execute("CREATE TABLE IF NOT EXISTS schema_migrations ("
"version text PRIMARY KEY, checksum text NOT NULL, applied_at timestamptz NOT NULL DEFAULT now())")
applied = {r["version"]: r["checksum"] for r in await conn.fetch("SELECT version, checksum FROM schema_migrations")}
directory = os.path.abspath(config.MIGRATIONS_DIR)
files = sorted(f for f in os.listdir(directory) if f.endswith(".sql"))
count = 0
for name in files:
sql = open(os.path.join(directory, name), encoding="utf-8").read()
checksum = hashlib.sha256(sql.encode("utf-8")).hexdigest()
if name in applied:
if applied[name] != checksum:
log("error", "migration_modified", version=name)
sys.exit(2)
continue
async with conn.transaction():
await conn.execute(sql)
await conn.execute("INSERT INTO schema_migrations (version, checksum) VALUES ($1, $2)", name, checksum)
count += 1
log("info", "migration_applied", version=name)
log("info", "migrations_done", applied=count, total=len(files))
finally:
await conn.close()
if __name__ == "__main__":
asyncio.run(main())
+82
View File
@@ -0,0 +1,82 @@
"""Transactional outbox: relay публикует задания из PostgreSQL в брокер.
Гарантия: задание, зафиксированное в БД (COMMIT), будет опубликовано даже если процесс упал между COMMIT и publish.
Relay запускается в каждой реплике API и в worker; строки берутся `FOR UPDATE SKIP LOCKED`, поэтому одну строку
публикует один relay. Если процесс упадёт после подтверждения брокера, но до отметки published_at, задание будет
опубликовано повторно — доставка at-least-once, дубликаты безвредны (обработка идемпотентна).
"""
import asyncio
from datetime import timezone
from . import config, telemetry
from .web import format_dt, log
JOB_TYPE = "attachment.process.v1"
def job_message(row, carrier=None):
msg = {"schema_version": 1, "job_id": str(row["job_id"]), "type": JOB_TYPE, "attachment_id": str(row["attachment_id"]),
"pipeline_version": row["pipeline_version"], "attempt": row["attempt"],
"created_at": format_dt(row["created_at"].astimezone(timezone.utc))}
msg.update(carrier or {}) # traceparent / tracestate (W3C Trace Context) — поля схемы jobs.schema.json
return msg
class Relay:
def __init__(self, db, broker):
self.db, self.broker = db, broker
self._kick = asyncio.Event()
self._stop = False
self.last_error = None
def kick(self):
self._kick.set()
def stop(self):
self._stop = True
self._kick.set()
async def relay_once(self):
published = 0
async with self.db.conn() as c:
async with c.transaction():
rows = await c.fetch(
"SELECT id, job_id, attachment_id, pipeline_version, attempt, created_at, traceparent, tracestate FROM outbox "
"WHERE published_at IS NULL AND available_at <= now() ORDER BY id LIMIT $1 FOR UPDATE SKIP LOCKED",
config.RELAY_BATCH)
for row in rows:
parent = {k: row[k] for k in ("traceparent", "tracestate") if row[k]}
# PRODUCER-спан — дочерний к спану запроса upload (контекст восстановлен из outbox)
with telemetry.span("attachment.process publish", "producer",
{"messaging.system": "rabbitmq", "messaging.destination.name": "attachment.process",
"messaging.operation": "publish", "outbox.attempt": row["attempt"]}, parent):
carrier = telemetry.current_carrier() or parent
await self.broker.publish(job_message(row, carrier), headers=carrier)
await c.execute("UPDATE outbox SET published_at = now() WHERE id = $1", row["id"])
published += 1
return published
def headers_for(self, row):
return {}
async def run(self):
delay = config.RELAY_INTERVAL
while not self._stop:
try:
n = await self.relay_once()
self.last_error = None
delay = config.RELAY_INTERVAL
if n >= config.RELAY_BATCH:
continue # возможно, есть ещё
except asyncio.CancelledError:
raise
except Exception as exc:
if self.last_error != type(exc).__name__:
log("warning", "outbox_relay_failed", error_type=type(exc).__name__)
self.last_error = type(exc).__name__
delay = min(delay * 2, 10.0) # брокер/БД недоступны: пауза с ростом до 10 с, задания остаются в outbox
try:
await asyncio.wait_for(self._kick.wait(), delay)
except asyncio.TimeoutError:
pass
self._kick.clear()
+31
View File
@@ -0,0 +1,31 @@
"""Хеширование паролей: Argon2id (библиотека argon2-cffi), вычисления вне event loop."""
import asyncio
from concurrent.futures import ThreadPoolExecutor
from argon2 import PasswordHasher
from argon2.exceptions import InvalidHashError, VerificationError, VerifyMismatchError
from . import config
_hasher = PasswordHasher(time_cost=config.ARGON2_TIME_COST, memory_cost=config.ARGON2_MEMORY_KIB,
parallelism=config.ARGON2_PARALLELISM)
_pool = ThreadPoolExecutor(max_workers=config.HASH_CONCURRENCY, thread_name_prefix="argon2")
# Для неизвестного username проверяем фиктивный хеш: время ответа не выдаёт существование пользователя.
_DUMMY = _hasher.hash("dummy-password-for-timing-equalisation")
async def hash_password(password):
return await asyncio.get_running_loop().run_in_executor(_pool, _hasher.hash, password)
def _verify(stored, password):
try:
return _hasher.verify(stored, password)
except (VerifyMismatchError, VerificationError, InvalidHashError):
return False
async def verify_password(stored, password):
"""stored=None -> проверка фиктивного хеша, результат всегда False."""
ok = await asyncio.get_running_loop().run_in_executor(_pool, _verify, stored or _DUMMY, password)
return ok and stored is not None
+97
View File
@@ -0,0 +1,97 @@
"""Обработка изображений (pipeline_version=1): thumbnail, crop, optimized.
Правила (contracts/README.md):
* thumbnail: JPEG, вписан в 256x256 с сохранением пропорций, БЕЗ увеличения, EXIF orientation применяется, alpha — на белый
фон; размеры округляются вниз, минимум 1 px (320x200 -> 256x160); качество 85, метаданные не сохраняются;
* crop: JPEG, центрированный квадрат 128x128 (маленькие исходники допускается увеличивать), качество 85;
* optimized: WebP, вписан в 1280x1280 без увеличения, quality=80 method=4, EXIF/ICC не сохраняются (ориентация применена).
Память: JPEG декодируется через draft (масштабирование при декодировании), число пикселей ограничено MAX_IMAGE_PIXELS.
"""
import hashlib
import io
import math
from PIL import Image, ImageOps
from . import config
Image.MAX_IMAGE_PIXELS = None
PIPELINE_VERSION = 1
JPEG_QUALITY = 85
WEBP_QUALITY = 80
class PermanentError(Exception):
"""Постоянная ошибка входных данных: повтор не поможет (-> failed, error_code)."""
def __init__(self, code):
super().__init__(code)
self.code = code
def _flatten(img):
"""RGBA/LA/P с прозрачностью -> RGB на белом фоне."""
if img.mode in ("RGBA", "LA") or (img.mode == "P" and "transparency" in img.info):
rgba = img.convert("RGBA")
background = Image.new("RGB", rgba.size, (255, 255, 255))
background.paste(rgba, mask=rgba.getchannel("A"))
return background
return img.convert("RGB")
def _fit_size(width, height, box, upscale=False):
scale = min(box / width, box / height)
if not upscale:
scale = min(scale, 1.0)
return max(1, math.floor(width * scale)), max(1, math.floor(height * scale))
def _encode(img, fmt, **params):
buf = io.BytesIO()
img.save(buf, fmt, **params)
data = buf.getvalue()
return data, hashlib.sha256(data).hexdigest()
def process(fileobj):
"""Возвращает список вариантов: dict(name, content_type, data, sha256, width, height). Бросает PermanentError."""
try:
fileobj.seek(0)
src = Image.open(fileobj)
fmt = src.format
width, height = src.size
if fmt not in ("PNG", "JPEG"):
raise PermanentError("invalid_image")
if width * height > config.MAX_IMAGE_PIXELS:
raise PermanentError("image_too_large")
if fmt == "JPEG" and width > 2560 and height > 2560:
# экономия памяти: JPEG декодируется сразу в масштабе 1/2 (результат всё равно вписывается в 1280x1280)
src.draft("RGB", (width // 2, height // 2))
src.load()
img = ImageOps.exif_transpose(src)
except PermanentError:
raise
except MemoryError:
raise
except Exception:
raise PermanentError("invalid_image") from None
base = _flatten(img)
out = []
# thumbnail
tw, th = _fit_size(base.width, base.height, 256)
thumb = base.resize((tw, th), Image.LANCZOS) if (tw, th) != base.size else base
data, sha = _encode(thumb, "JPEG", quality=JPEG_QUALITY, optimize=True)
out.append({"name": "thumbnail", "content_type": "image/jpeg", "data": data, "sha256": sha, "width": tw, "height": th})
# crop 128x128 (центрированный квадрат)
side = min(base.width, base.height)
left, top = (base.width - side) // 2, (base.height - side) // 2
crop = base.crop((left, top, left + side, top + side)).resize((128, 128), Image.LANCZOS)
data, sha = _encode(crop, "JPEG", quality=JPEG_QUALITY, optimize=True)
out.append({"name": "crop", "content_type": "image/jpeg", "data": data, "sha256": sha, "width": 128, "height": 128})
# optimized WebP (прозрачность сохраняется)
opt_src = img.convert("RGBA") if (img.mode in ("RGBA", "LA") or (img.mode == "P" and "transparency" in img.info)) else base
ow, oh = _fit_size(opt_src.width, opt_src.height, 1280)
opt = opt_src.resize((ow, oh), Image.LANCZOS) if (ow, oh) != opt_src.size else opt_src
data, sha = _encode(opt, "WEBP", quality=WEBP_QUALITY, method=4)
out.append({"name": "optimized", "content_type": "image/webp", "data": data, "sha256": sha, "width": ow, "height": oh})
return out
+118
View File
@@ -0,0 +1,118 @@
"""Сверка S3 и PostgreSQL (reconcile): python -m app.reconcile [--apply] [--grace СЕКУНДЫ] [--prefix att/]
Находит и (с --apply) безопасно исправляет расхождения двух хранилищ:
* orphan_object — объект в S3 без строки в attachments (сбой между записью в S3 и COMMIT метаданных, прерванная загрузка);
удаляется ТОЛЬКО если старше grace (по умолчанию ORPHAN_GRACE_SECONDS=900 с): свежий объект может быть
загрузкой, метаданные которой ещё не зафиксированы;
* missing_object — строка attachments (ready) без объекта в S3 (потеря/удаление объекта): строка НЕ удаляется,
статус -> failed, error_code=object_missing (сообщения со ссылкой остаются, скачивание даёт 409);
* size_mismatch — размер объекта не совпадает с size_bytes: статус -> failed, error_code=size_mismatch.
Режим по умолчанию — dry-run (ничего не меняет). Повторный запуск идемпотентен. Используемые (привязанные к сообщению)
вложения и свежие объекты не удаляются никогда: удаляются только объекты, у которых нет строки в БД.
Выход: JSON-отчёт со счётчиками и временем обхода.
"""
import argparse
import asyncio
import json
import time
import uuid
from datetime import datetime, timedelta, timezone
import asyncpg
from . import config
from .s3 import S3
def object_owner(key):
"""Recognize server-generated prefixes, including effects not committed yet."""
parts = key.split("/")
if (parts[0] == "att" and len(parts) >= 3) or (parts[0] == "staging" and len(parts) == 2):
try:
return uuid.UUID(parts[1])
except ValueError:
pass
return None
async def reconcile(apply, grace, prefix):
started = time.perf_counter()
s3 = S3()
await s3.start()
conn = await asyncpg.connect(config.DATABASE_URL, timeout=5)
try:
t0 = time.perf_counter()
rows = await conn.fetch("SELECT id, s3_key, staging_key, size_bytes, status, created_at FROM attachments")
variants = await conn.fetch(
"SELECT a.id, v.s3_key, v.size_bytes, a.status, a.created_at "
"FROM attachment_variants v JOIN attachments a ON a.id = v.attachment_id")
db_ms = (time.perf_counter() - t0) * 1000
by_key = {r["s3_key"]: dict(r, check=True) for r in [*rows, *variants]}
# Staging may already be removed after ready, or retained for failed/replay.
by_key.update({r["staging_key"]: dict(r, check=False) for r in rows if r["staging_key"]})
owners = {r["id"] for r in rows}
t0 = time.perf_counter()
objects = {}
async for key, size, modified in s3.list_keys(prefix):
objects[key] = (size, modified)
list_ms = (time.perf_counter() - t0) * 1000
now = datetime.now(timezone.utc)
cutoff = now - timedelta(seconds=grace)
report = {"mode": "apply" if apply else "dry-run", "grace_seconds": grace, "db_rows": len(rows), "s3_objects": len(objects),
"orphan_objects": [], "orphans_skipped_fresh": 0, "owned_objects_skipped": 0,
"missing_objects": [], "size_mismatch": [], "fixed": {"deleted_orphans": 0, "marked_failed": 0}}
for key, (size, modified) in objects.items():
row = by_key.get(key)
if row is None:
if object_owner(key) in owners:
# Worker can write S3 variants before committing their metadata.
report["owned_objects_skipped"] += 1
elif modified < cutoff:
report["orphan_objects"].append(key)
else:
report["orphans_skipped_fresh"] += 1
elif row["check"] and row["status"] == "ready" and row["size_bytes"] != size:
report["size_mismatch"].append(str(row["id"]))
for key, row in by_key.items():
if row["check"] and key.startswith(prefix) and key not in objects and row["status"] == "ready" and row["created_at"] < cutoff:
report["missing_objects"].append(str(row["id"]))
for field in ("missing_objects", "size_mismatch"):
report[field] = sorted(set(report[field]))
if apply:
for key in report["orphan_objects"]:
# повторная проверка перед удалением: за время обхода строка могла появиться
referenced = await conn.fetchval(
"SELECT EXISTS (SELECT 1 FROM attachments WHERE s3_key = $1 OR staging_key = $1 OR id = $2) "
"OR EXISTS (SELECT 1 FROM attachment_variants WHERE s3_key = $1)", key, object_owner(key))
if not referenced:
await s3.delete(key)
report["fixed"]["deleted_orphans"] += 1
for ids, code in ((report["missing_objects"], "object_missing"), (report["size_mismatch"], "size_mismatch")):
for att_id in ids:
n = await conn.execute("UPDATE attachments SET status = 'failed', error_code = $2, updated_at = now() "
"WHERE id = $1 AND status = 'ready'", att_id, code)
report["fixed"]["marked_failed"] += int(n.split()[-1])
report["timing_ms"] = {"db_query": round(db_ms, 1), "s3_listing": round(list_ms, 1),
"total": round((time.perf_counter() - started) * 1000, 1)}
return report
finally:
await conn.close()
await s3.stop()
def main():
ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
ap.add_argument("--apply", action="store_true", help="исправлять (по умолчанию dry-run)")
ap.add_argument("--grace", type=int, default=config.ORPHAN_GRACE_SECONDS)
ap.add_argument("--prefix", default="att/")
ap.add_argument("--max-listed", type=int, default=20, help="сколько идентификаторов печатать в отчёте")
args = ap.parse_args()
report = asyncio.run(reconcile(args.apply, args.grace, args.prefix))
for k in ("orphan_objects", "missing_objects", "size_mismatch"):
report[k + "_count"] = len(report[k])
report[k] = report[k][:args.max_listed]
print(json.dumps(report, ensure_ascii=False, indent=2))
if __name__ == "__main__":
main()
+70
View File
@@ -0,0 +1,70 @@
"""Операторский replay и реестр неудавшихся заданий: python -m app.replay [list | replay <attachment_id> | replay-all]
list — реестр failed-вложений: id, error_code, число попыток, возраст, наличие staging;
replay <attachment_id> — возвращает ТОЛЬКО failed-вложение в queued: новая попытка (attempt сбрасывается на 1), новый job_id,
запись в job_events (аудит), задание в outbox. Если у вложения уже есть варианты (ready/успешный
эффект), повторного исполнения не будет: вложение не в статусе failed -> replay отклоняется.
Эффекты защищены ключом (attachment_id, pipeline_version, name): даже повторное выполнение не создаёт второй набор вариантов.
"""
import argparse
import asyncio
import json
import sys
import uuid
import asyncpg
from . import config
async def list_failed(conn):
rows = await conn.fetch(
"SELECT id, error_code, attempt, staging_key IS NOT NULL AS has_staging, updated_at FROM attachments "
"WHERE status = 'failed' ORDER BY updated_at DESC LIMIT 200")
return [{"attachment_id": str(r["id"]), "error_code": r["error_code"], "attempt": r["attempt"],
"staging_present": r["has_staging"], "failed_at": r["updated_at"].isoformat()} for r in rows]
async def replay(conn, att_id):
async with conn.transaction():
row = await conn.fetchrow("SELECT id, status, pipeline_version, staging_key FROM attachments WHERE id = $1 FOR UPDATE", att_id)
if row is None:
return {"attachment_id": str(att_id), "result": "not_found"}
if row["status"] != "failed":
return {"attachment_id": str(att_id), "result": "skipped", "reason": "status=%s (replay только для failed)" % row["status"]}
if not row["staging_key"]:
return {"attachment_id": str(att_id), "result": "skipped", "reason": "staging удалён по retention, исходник недоступен"}
job_id = uuid.uuid4()
await conn.execute("UPDATE attachments SET status = 'queued', error_code = NULL, attempt = 0, job_id = $2, lease_owner = NULL, "
"lease_expires_at = NULL, updated_at = now() WHERE id = $1", att_id, job_id)
await conn.execute("INSERT INTO outbox (job_id, attachment_id, pipeline_version, attempt) VALUES ($1, $2, $3, 1)",
job_id, att_id, row["pipeline_version"])
await conn.execute("INSERT INTO job_events (attachment_id, job_id, event, attempt, detail) VALUES ($1, $2, 'operator_replay', 1, 'failed -> queued')",
att_id, job_id)
return {"attachment_id": str(att_id), "result": "requeued", "job_id": str(job_id)}
async def main_async(args):
conn = await asyncpg.connect(config.DATABASE_URL, timeout=5)
try:
if args.command == "list":
return await list_failed(conn)
if args.command == "replay":
return await replay(conn, uuid.UUID(args.attachment_id))
return [await replay(conn, uuid.UUID(f["attachment_id"])) for f in await list_failed(conn)]
finally:
await conn.close()
def main():
ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
sub = ap.add_subparsers(dest="command", required=True)
sub.add_parser("list")
r = sub.add_parser("replay")
r.add_argument("attachment_id")
sub.add_parser("replay-all")
print(json.dumps(asyncio.run(main_async(ap.parse_args())), ensure_ascii=False, indent=2))
if __name__ == "__main__":
sys.exit(main())
+92
View File
@@ -0,0 +1,92 @@
"""Приватное S3-хранилище (MinIO) через aiobotocore: потоковая загрузка/скачивание, таймауты, перевод сбоев в 503."""
import asyncio
from aiobotocore.config import AioConfig
from aiobotocore.session import get_session
from botocore.exceptions import BotoCoreError, ClientError
from . import config, telemetry
from .web import log, unavailable
S3_ERRORS = (BotoCoreError, ClientError, asyncio.TimeoutError, OSError)
class S3:
def __init__(self):
self._ctx = None
self.client = None
async def start(self):
if not config.S3_ENDPOINT:
log("error", "s3_endpoint_missing")
return
session = get_session()
self._ctx = session.create_client(
"s3", endpoint_url=config.S3_ENDPOINT, region_name=config.S3_REGION,
aws_access_key_id=config.S3_ACCESS_KEY, aws_secret_access_key=config.S3_SECRET_KEY,
config=AioConfig(connect_timeout=config.S3_CONNECT_TIMEOUT, read_timeout=config.S3_READ_TIMEOUT,
retries={"max_attempts": 2, "mode": "standard"}, max_pool_connections=config.S3_POOL,
s3={"addressing_style": "path"}))
self.client = await self._ctx.__aenter__()
async def stop(self):
if self._ctx:
await self._ctx.__aexit__(None, None, None)
def _check(self):
if self.client is None:
raise unavailable("Хранилище файлов не настроено")
async def put(self, key, fileobj, content_type, size):
self._check()
try:
with telemetry.child_span("s3.put_object", "client", {"rpc.system": "aws-api", "rpc.method": "PutObject"}):
await self.client.put_object(Bucket=config.S3_BUCKET, Key=key, Body=fileobj, ContentType=content_type,
ContentLength=size)
except S3_ERRORS as exc:
log("warning", "s3_put_failed", error_type=type(exc).__name__)
raise unavailable("Хранилище файлов недоступно") from exc
async def put_bytes(self, key, data, content_type):
self._check()
with telemetry.child_span("s3.put_object", "client", {"rpc.system": "aws-api", "rpc.method": "PutObject"}):
await self.client.put_object(Bucket=config.S3_BUCKET, Key=key, Body=data, ContentType=content_type)
async def open_stream(self, key):
"""Возвращает (stream, content_length). Вызывающий обязан закрыть stream."""
self._check()
try:
with telemetry.child_span("s3.get_object", "client", {"rpc.system": "aws-api", "rpc.method": "GetObject"}):
resp = await self.client.get_object(Bucket=config.S3_BUCKET, Key=key)
except S3_ERRORS as exc:
log("warning", "s3_get_failed", error_type=type(exc).__name__)
raise unavailable("Хранилище файлов недоступно") from exc
return resp["Body"], resp["ContentLength"]
async def delete(self, key):
self._check()
await self.client.delete_object(Bucket=config.S3_BUCKET, Key=key)
async def head(self, key):
"""Метаданные объекта или None, если его нет."""
self._check()
try:
return await self.client.head_object(Bucket=config.S3_BUCKET, Key=key)
except ClientError as exc:
if exc.response.get("Error", {}).get("Code") in ("404", "NoSuchKey", "NotFound"):
return None
raise unavailable("Хранилище файлов недоступно") from exc
except S3_ERRORS as exc:
raise unavailable("Хранилище файлов недоступно") from exc
async def ping(self):
self._check()
await asyncio.wait_for(self.client.head_bucket(Bucket=config.S3_BUCKET), 2.0)
async def list_keys(self, prefix):
"""Асинхронный итератор (key, size, last_modified) по префиксу — для сверки (reconcile)."""
self._check()
paginator = self.client.get_paginator("list_objects_v2")
async for page in paginator.paginate(Bucket=config.S3_BUCKET, Prefix=prefix):
for obj in page.get("Contents", []):
yield obj["Key"], obj["Size"], obj["LastModified"]
+85
View File
@@ -0,0 +1,85 @@
"""Идемпотентный bootstrap S3: python -m app.s3_bootstrap (one-shot сервис s3-init при каждом `docker compose up`).
1) bucket создаётся администратором, если его ещё нет (повтор ничего не меняет);
2) проверяется, что анонимный запрос к bucket отклоняется (private);
3) проверяется, что отдельный ключ API (права только на этот bucket) может писать/читать/удалять и НЕ может создавать bucket.
Ключи читаются из файлов секретов; в вывод не попадают. Код выхода != 0, если любая проверка не пройдена.
"""
import asyncio
import os
import sys
import time
import urllib.error
import urllib.request
from aiobotocore.config import AioConfig
from aiobotocore.session import get_session
from botocore.exceptions import ClientError
from . import config
def secret(name):
path = os.environ.get(name + "_FILE")
with open(path, encoding="utf-8") as fh:
return fh.read().strip()
def client(session, access, secret_key):
return session.create_client("s3", endpoint_url=config.S3_ENDPOINT, region_name=config.S3_REGION, aws_access_key_id=access,
aws_secret_access_key=secret_key,
config=AioConfig(connect_timeout=3, read_timeout=10, retries={"max_attempts": 1},
s3={"addressing_style": "path"}))
async def main():
bucket = config.S3_BUCKET
session = get_session()
deadline = time.time() + 60
async with client(session, secret("S3_ADMIN_ACCESS_KEY"), secret("S3_ADMIN_SECRET_KEY")) as admin:
while True:
try:
await admin.list_buckets()
break
except Exception as exc:
if time.time() > deadline:
print("S3 недоступен:", type(exc).__name__)
return 1
await asyncio.sleep(1)
try:
await admin.head_bucket(Bucket=bucket)
print("bucket %s уже существует" % bucket)
except ClientError:
await admin.create_bucket(Bucket=bucket)
print("bucket %s создан" % bucket)
# приватность: запрос без подписи отклоняется
try:
urllib.request.urlopen("%s/%s/" % (config.S3_ENDPOINT, bucket), timeout=5)
print("ОШИБКА: bucket доступен анонимно")
return 2
except urllib.error.HTTPError as exc:
if exc.code not in (401, 403):
print("ОШИБКА: анонимный доступ вернул %d, ожидался 401/403" % exc.code)
return 2
print("анонимный доступ запрещён (HTTP %d)" % exc.code)
# ключ API: рабочие права на bucket, нет прав администратора
async with client(session, config.S3_ACCESS_KEY, config.S3_SECRET_KEY) as api:
key = "bootstrap-check/%d" % int(time.time())
await api.put_object(Bucket=bucket, Key=key, Body=b"ok")
got = await (await api.get_object(Bucket=bucket, Key=key))["Body"].read()
await api.delete_object(Bucket=bucket, Key=key)
if got != b"ok":
print("ОШИБКА: чтение/запись ключом API")
return 3
try:
await api.create_bucket(Bucket="not-allowed-" + str(int(time.time())))
print("ОШИБКА: ключ API смог создать bucket (избыточные права)")
return 4
except ClientError as exc:
print("ключ API ограничен одним bucket (create_bucket отклонён: %s)" % exc.response["Error"]["Code"])
print("bootstrap завершён")
return 0
if __name__ == "__main__":
sys.exit(asyncio.run(main()))
+77
View File
@@ -0,0 +1,77 @@
"""Read-only readiness probe after restart: ranged GET of recent ready originals.
An open S3 socket / HEAD bucket does not prove that SeaweedFS volume locations
have been registered. Read one byte from up to 16 recent ready originals.
This is a bounded recovery hook, not an audit of every historical object.
"""
import asyncio
import sys
import asyncpg
from . import config
from .s3 import S3
async def probe(s3):
count = 0
batch = []
async def read(key, size):
params = {"Bucket": config.S3_BUCKET, "Key": key}
if size:
params["Range"] = "bytes=0-0"
response = await s3.client.get_object(**params)
async with response["Body"] as stream:
data = await stream.read()
if len(data) != min(size, 1):
raise ValueError("unexpected ranged read size")
async def flush():
results = await asyncio.gather(*(read(key, size) for key, size in batch), return_exceptions=True)
for result in results:
if isinstance(result, BaseException):
raise result
await s3.ping()
conn = await asyncpg.connect(config.DATABASE_URL, timeout=5)
try:
rows = await conn.fetch("SELECT s3_key, size_bytes FROM attachments "
"WHERE status = 'ready' ORDER BY accepted_at DESC LIMIT 16")
finally:
await conn.close()
for row in rows:
key, size = row["s3_key"], row["size_bytes"]
batch.append((key, size))
count += 1
if len(batch) == 8:
await flush()
batch.clear()
if batch:
await flush()
return count
async def main():
s3 = S3()
await s3.start()
try:
async with asyncio.timeout(45):
while True:
try:
count = await probe(s3)
print(f"PASS: S3 recovery, {count} recent ready originals readable")
return 0
except Exception as exc:
# Do not print SDK requests/credentials or object contents.
print(f"Waiting for S3 data volumes: {type(exc).__name__}", flush=True)
await asyncio.sleep(1)
except TimeoutError:
print("FAIL: S3 data recovery exceeded 45 seconds")
return 1
finally:
await s3.stop()
if __name__ == "__main__":
sys.exit(asyncio.run(main()))
+40
View File
@@ -0,0 +1,40 @@
"""Точка входа: uvicorn с управляемым завершением (drain).
SIGTERM -> readiness сразу 503 -> DRAIN_SECONDS продолжаем обслуживать запросы, пока балансировщик
исключает реплику -> uvicorn прекращает приём и дожидается текущих запросов (timeout_graceful_shutdown).
Так плановая остановка/rolling update не даёт ошибок клиентам.
"""
import signal
import threading
import uvicorn
from . import config
from .main import app
from .web import log
class DrainingServer(uvicorn.Server):
_draining = False
def handle_exit(self, sig, frame):
if sig == signal.SIGTERM and not self._draining and config.DRAIN_SECONDS > 0:
self._draining = True
app.draining = True
log("info", "draining", seconds=config.DRAIN_SECONDS)
timer = threading.Timer(config.DRAIN_SECONDS, super().handle_exit, args=(sig, frame))
timer.daemon = True
timer.start()
return
super().handle_exit(sig, frame)
def main():
cfg = uvicorn.Config(app, host=config.APP_HOST, port=config.APP_PORT, workers=1, access_log=False,
log_level="warning", timeout_graceful_shutdown=config.GRACEFUL_SHUTDOWN_SECONDS,
timeout_keep_alive=config.KEEPALIVE_SECONDS, limit_concurrency=config.LIMIT_CONCURRENCY)
DrainingServer(cfg).run()
if __name__ == "__main__":
main()
+131
View File
@@ -0,0 +1,131 @@
"""Сессии: PostgreSQL — источник истины; Valkey — кэш с TTL (read-through).
Политики:
* токен — 256 бит из secrets; в БД и Valkey хранится только SHA-256 токена;
* expiry: expires_at в БД (по часам БД). В кэше TTL = min(остаток жизни, SESSION_CACHE_TTL_SECONDS);
* инвалидация: logout помечает revoked_at в БД и удаляет ключ в Valkey;
* Valkey недоступен -> кэш обходится (чтение из БД), анонимный запрос никогда не считается доверенным;
circuit breaker не даёт каждому запросу ждать таймаут;
* окно устаревания: если Valkey был недоступен во время logout, ключ может прожить до SESSION_CACHE_TTL_SECONDS.
"""
import hashlib
import secrets
import time
from . import config
from .web import log
try:
import redis.asyncio as aioredis
except ImportError: # кэш опционален
aioredis = None
def token_hash(token):
return hashlib.sha256(token.encode("ascii")).digest()
class SessionCache:
def __init__(self):
self.client = None
self.down_until = 0.0
self.stats = {"hit": 0, "miss": 0, "error": 0}
async def start(self):
if config.VALKEY_URL and aioredis:
self.client = aioredis.Redis.from_url(config.VALKEY_URL, socket_timeout=config.VALKEY_TIMEOUT,
socket_connect_timeout=config.VALKEY_TIMEOUT)
async def stop(self):
if self.client:
await self.client.aclose()
def _usable(self):
return self.client is not None and time.monotonic() >= self.down_until
def _fail(self, exc):
self.stats["error"] += 1
if time.monotonic() >= self.down_until:
log("warning", "session_cache_unavailable", error_type=type(exc).__name__)
self.down_until = time.monotonic() + config.VALKEY_BREAKER_SECONDS
async def get(self, key):
if not self._usable():
return None
try:
value = await self.client.get(b"s:" + key)
except Exception as exc:
self._fail(exc)
return None
self.stats["hit" if value is not None else "miss"] += 1
return value
async def put(self, key, value, ttl):
if not self._usable() or ttl <= 0:
return
try:
await self.client.set(b"s:" + key, value, ex=max(1, int(ttl)))
except Exception as exc:
self._fail(exc)
async def delete(self, key):
if not self._usable():
return
try:
await self.client.delete(b"s:" + key)
except Exception as exc:
self._fail(exc)
async def ping(self):
if self.client is None:
return None
try:
return bool(await self.client.ping())
except Exception:
return False
class Sessions:
def __init__(self, db, cache):
self.db, self.cache = db, cache
@staticmethod
def _cache_value(user_id, expires_at):
return ("%s|%d" % (user_id, int(expires_at.timestamp() * 1000))).encode()
async def create(self, user_id):
token = secrets.token_urlsafe(32) # 256 бит энтропии
digest = token_hash(token)
row = await self.db.fetchrow(
"INSERT INTO sessions (token_hash, user_id, expires_at) "
"VALUES ($1, $2, now() + make_interval(secs => $3)) RETURNING expires_at",
digest, user_id, float(config.SESSION_TTL_SECONDS))
remaining = row["expires_at"].timestamp() - time.time()
await self.cache.put(digest, self._cache_value(user_id, row["expires_at"]),
min(remaining, config.SESSION_CACHE_TTL_SECONDS))
return token, row["expires_at"]
async def resolve(self, token):
"""token -> user_id (str) либо None, если сессии нет/истекла/отозвана."""
digest = token_hash(token)
cached = await self.cache.get(digest)
if cached is not None:
user_id, _, expires_ms = cached.decode().partition("|")
if int(expires_ms) > time.time() * 1000:
return user_id
await self.cache.delete(digest)
row = await self.db.fetchrow(
"SELECT user_id, expires_at FROM sessions "
"WHERE token_hash = $1 AND revoked_at IS NULL AND expires_at > now()", digest)
if row is None:
return None
user_id = str(row["user_id"])
remaining = row["expires_at"].timestamp() - time.time()
await self.cache.put(digest, self._cache_value(user_id, row["expires_at"]),
min(remaining, config.SESSION_CACHE_TTL_SECONDS))
return user_id
async def revoke(self, token):
digest = token_hash(token)
await self.db.execute("UPDATE sessions SET revoked_at = now() WHERE token_hash = $1 AND revoked_at IS NULL", digest)
await self.cache.delete(digest)
+229
View File
@@ -0,0 +1,229 @@
"""Наблюдаемость (lab6): OpenTelemetry SDK — трассы, метрики, логи по OTLP/HTTP в Collector.
Выключается переменной OTEL_ENABLED=0 (нужно для эксперимента «цена наблюдаемости»): тогда все функции — no-op и
накладных расходов на экспорт нет. Секреты, тела сообщений, токены и изображения в телеметрию не попадают:
в атрибутах только шаблон маршрута, метод, статус, тип задания и исход; ID объектов — только в логах/спанах
(не в labels метрик).
Соответствие сигналов contracts/observability.md и имён метрик (после Prometheus-экспортёра Collector):
http.server.request.duration (s) -> http_server_request_duration_seconds_{bucket,sum,count} (HTTP requests = _count)
messenger.jobs.processed -> messenger_jobs_processed_total{job_type,outcome}
messenger.job.duration (s) -> messenger_job_duration_seconds_*
messenger.job.time_to_ready (s) -> messenger_job_time_to_ready_seconds_* (от приёма до ready)
messenger.queue.depth / .oldest_age -> messenger_queue_depth{queue_name}, messenger_queue_oldest_job_age_seconds
db.client.connections.usage/waiting -> db_client_connections_usage{state}, db_client_connections_waiting
messenger.worker.busy / .capacity -> messenger_worker_busy, messenger_worker_capacity
process.memory.rss / process.cpu.time -> process_memory_rss_bytes, process_cpu_time_seconds_total
"""
import contextlib
import json
import logging
import os
import time
from . import config
ENABLED = config.OTEL_ENABLED
_state = {"tracer": None, "meter": None, "logger": None, "propagator": None, "inst": {}}
HTTP_BUCKETS = [0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0]
JOB_BUCKETS = [0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30, 60]
TTR_BUCKETS = [0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30, 60, 120, 300]
def setup(service_name):
"""Инициализирует провайдеры. Вызывается один раз при старте процесса (API или worker)."""
if not ENABLED:
return
from opentelemetry import metrics, trace
from opentelemetry.exporter.otlp.proto.http._log_exporter import OTLPLogExporter
from opentelemetry.exporter.otlp.proto.http.metric_exporter import OTLPMetricExporter
from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter
from opentelemetry.propagate import get_global_textmap
from opentelemetry.sdk._logs import LoggerProvider, LoggingHandler
from opentelemetry.sdk._logs.export import BatchLogRecordProcessor
from opentelemetry.sdk.metrics import MeterProvider
from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader
from opentelemetry.sdk.metrics.view import ExplicitBucketHistogramAggregation, View
from opentelemetry.sdk.resources import Resource
from opentelemetry.sdk.trace import TracerProvider
from opentelemetry.sdk.trace.export import BatchSpanProcessor
from opentelemetry.sdk.trace.sampling import ParentBased, TraceIdRatioBased
endpoint = config.OTEL_ENDPOINT.rstrip("/")
resource = Resource.create({"service.name": service_name, "service.version": config.SERVICE_VERSION,
"service.instance.id": config.INSTANCE_ID, "deployment.environment.name": "lab"})
# --- трассы: head-sampling по доле (ParentBased: решение родителя, в т.ч. из traceparent брокера, сохраняется)
tp = TracerProvider(resource=resource, sampler=ParentBased(TraceIdRatioBased(config.OTEL_TRACE_RATIO)))
tp.add_span_processor(BatchSpanProcessor(
OTLPSpanExporter(endpoint=endpoint + "/v1/traces", timeout=3),
max_queue_size=config.OTEL_QUEUE_SIZE, max_export_batch_size=config.OTEL_BATCH_SIZE, schedule_delay_millis=2000, export_timeout_millis=3000))
trace.set_tracer_provider(tp)
# --- метрики (histogram с явными границами, секунды)
views = [View(instrument_name="http.server.request.duration", aggregation=ExplicitBucketHistogramAggregation(HTTP_BUCKETS)),
View(instrument_name="messenger.job.duration", aggregation=ExplicitBucketHistogramAggregation(JOB_BUCKETS)),
View(instrument_name="messenger.job.time_to_ready", aggregation=ExplicitBucketHistogramAggregation(TTR_BUCKETS))]
reader = PeriodicExportingMetricReader(OTLPMetricExporter(endpoint=endpoint + "/v1/metrics", timeout=3),
export_interval_millis=config.OTEL_METRIC_INTERVAL_MS, export_timeout_millis=3000)
mp = MeterProvider(resource=resource, metric_readers=[reader], views=views)
metrics.set_meter_provider(mp)
# --- логи: стандартный logging -> OTLP (trace_id/span_id добавляются автоматически из активного спана)
lp = LoggerProvider(resource=resource)
lp.add_log_record_processor(BatchLogRecordProcessor(OTLPLogExporter(endpoint=endpoint + "/v1/logs", timeout=3),
max_queue_size=config.OTEL_QUEUE_SIZE, schedule_delay_millis=2000))
logger = logging.getLogger("messenger")
logger.setLevel(logging.INFO)
logger.propagate = False
logger.addHandler(LoggingHandler(level=logging.INFO, logger_provider=lp))
_state.update(tracer=trace.get_tracer("messenger"), meter=metrics.get_meter("messenger"), logger=logger,
propagator=get_global_textmap(), providers=(tp, mp, lp))
m = _state["meter"]
_state["inst"] = {
"http": m.create_histogram("http.server.request.duration", unit="s", description="Длительность HTTP-запроса"),
"jobs": m.create_counter("messenger.jobs.processed", description="Обработанные задания по исходам"),
"job_dur": m.create_histogram("messenger.job.duration", unit="s", description="Длительность обработки задания"),
"ttr": m.create_histogram("messenger.job.time_to_ready", unit="s", description="От приёма загрузки до ready"),
}
for args in list(_pending_gauges):
(observable_counter if len(args) == 5 else gauge)(*args[:4])
_pending_gauges.clear()
def shutdown():
for p in _state.get("providers", ()):
try:
p.shutdown()
except Exception:
pass
# --- трассы ----------------------------------------------------------------------------------
@contextlib.contextmanager
def span(name, kind="internal", attributes=None, carrier=None):
"""Контекстный менеджер спана. carrier — dict с traceparent/tracestate для продолжения чужого trace."""
if not ENABLED or _state["tracer"] is None:
yield None
return
from opentelemetry import trace
from opentelemetry.trace import SpanKind, Status, StatusCode
ctx = _state["propagator"].extract(carrier) if carrier else None
kinds = {"server": SpanKind.SERVER, "client": SpanKind.CLIENT, "producer": SpanKind.PRODUCER,
"consumer": SpanKind.CONSUMER, "internal": SpanKind.INTERNAL}
with _state["tracer"].start_as_current_span(name, context=ctx, kind=kinds[kind], attributes=attributes or {},
record_exception=False, set_status_on_exception=False) as sp:
try:
yield sp
except Exception as exc:
sp.set_status(Status(StatusCode.ERROR, type(exc).__name__))
sp.set_attribute("error.type", type(exc).__name__)
raise
def child_span(name, kind="internal", attributes=None):
"""Спан создаётся только внутри активного спана (запрос/задание); фоновые опросы БД не порождают корневых трасс."""
if not ENABLED or _state["tracer"] is None:
return contextlib.nullcontext()
from opentelemetry import trace
if not trace.get_current_span().get_span_context().is_valid:
return contextlib.nullcontext()
return span(name, kind, attributes)
def set_attr(sp, key, value):
if sp is not None:
sp.set_attribute(key, value)
def mark_error(sp, description):
if sp is not None:
from opentelemetry.trace import Status, StatusCode
sp.set_status(Status(StatusCode.ERROR, description))
def current_carrier():
"""traceparent/tracestate текущего спана (для передачи через outbox/брокер)."""
if not ENABLED or _state["propagator"] is None:
return {}
carrier = {}
_state["propagator"].inject(carrier)
return carrier
def trace_ids():
if not ENABLED:
return None, None
from opentelemetry import trace
ctx = trace.get_current_span().get_span_context()
if not ctx.is_valid:
return None, None
return "%032x" % ctx.trace_id, "%016x" % ctx.span_id
# --- метрики ----------------------------------------------------------------------------------
def record_http(method, route, status, seconds):
if ENABLED and _state["inst"]:
_state["inst"]["http"].record(seconds, {"http.request.method": method, "http.route": route,
"http.response.status_code": status})
def record_job(job_type, outcome, seconds, time_to_ready=None):
if ENABLED and _state["inst"]:
attrs = {"job.type": job_type, "outcome": outcome}
_state["inst"]["jobs"].add(1, attrs)
_state["inst"]["job_dur"].record(seconds, attrs)
if time_to_ready is not None:
_state["inst"]["ttr"].record(time_to_ready, {"job.type": job_type})
_pending_gauges = []
def gauge(name, callback, unit="1", description=""):
"""Observable gauge: callback() -> [(value, {attrs})]. Регистрация откладывается до setup(), если провайдер ещё не создан."""
if not ENABLED:
return
if _state["meter"] is None:
_pending_gauges.append((name, callback, unit, description))
return
from opentelemetry.metrics import Observation
def cb(_options):
try:
return [Observation(v, a) for v, a in callback()]
except Exception:
return []
_state["meter"].create_observable_gauge(name, callbacks=[cb], unit=unit, description=description)
def observable_counter(name, callback, unit="1", description=""):
"""Монотонный счётчик, значение которого читается при экспорте (например, накопленное CPU-время)."""
if not ENABLED or _state["meter"] is None:
_pending_gauges.append((name, callback, unit, description, "counter"))
return
from opentelemetry.metrics import Observation
def cb(_options):
try:
return [Observation(v, a) for v, a in callback()]
except Exception:
return []
_state["meter"].create_observable_counter(name, callbacks=[cb], unit=unit, description=description)
def process_metrics():
def rss():
try:
with open("/proc/self/statm") as fh:
return [(int(fh.read().split()[1]) * os.sysconf("SC_PAGE_SIZE"), {})]
except (OSError, ValueError):
return []
gauge("process.memory.rss", rss, "By", "Резидентная память процесса")
observable_counter("process.cpu.time", lambda: [(time.process_time(), {})], "s", "Накопленное CPU-время процесса")
# --- логи ---------------------------------------------------------------------------------------
def emit_log(level, record):
"""Отправляет запись в OTLP (если включено). record — dict без секретов."""
if not ENABLED or _state["logger"] is None:
return
lvl = {"info": logging.INFO, "warning": logging.WARNING, "error": logging.ERROR}.get(level, logging.INFO)
_state["logger"].log(lvl, record.get("msg", ""), extra={k: v for k, v in record.items() if k not in ("msg", "level", "ts")})
+192
View File
@@ -0,0 +1,192 @@
"""Общий HTTP-слой: ошибки, запрос, валидация, курсоры, ответы, журнал."""
import base64
import json
import re
import sys
import time
import uuid
from datetime import datetime, timedelta, timezone
from urllib.parse import parse_qsl
from . import config, telemetry
UUID_RE = re.compile(r"[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}")
USERNAME_RE = re.compile(r"[a-z0-9_]{3,32}")
LIMIT_RE = re.compile(r"[0-9]{1,9}")
EPOCH = datetime(1970, 1, 1, tzinfo=timezone.utc)
class ApiError(Exception):
def __init__(self, status, code, message, headers=None):
super().__init__(message)
self.status, self.code, self.message, self.headers = status, code, message, headers or {}
def bad(message):
return ApiError(400, "invalid_request", message)
def unavailable(message="Зависимость временно недоступна"):
return ApiError(503, "unavailable", message, {"retry-after": str(config.RETRY_AFTER_SECONDS)})
def log(level, msg, **fields):
"""Структурный лог: JSON в stdout + OTLP (если включено). trace_id/span_id — из активного спана; тела и токены не логируются."""
record = {"ts": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), "level": level, "service": config.SERVICE_NAME, "msg": msg}
trace_id, span_id = telemetry.trace_ids()
if trace_id:
record["trace_id"], record["span_id"] = trace_id, span_id
record.update(fields)
sys.stdout.write(json.dumps(record, ensure_ascii=False) + "\n")
sys.stdout.flush()
telemetry.emit_log(level, record)
# --- время -------------------------------------------------------------------
def format_dt(value):
value = value.astimezone(timezone.utc)
return value.strftime("%Y-%m-%dT%H:%M:%S.") + "%06dZ" % value.microsecond
def dt_to_us(value):
return (value - EPOCH) // timedelta(microseconds=1)
def us_to_dt(us):
return EPOCH + timedelta(microseconds=us)
# --- валидация ---------------------------------------------------------------
def parse_uuid(value, what="id"):
if not isinstance(value, str) or not UUID_RE.fullmatch(value):
raise bad("Некорректный %s" % what)
return value.lower()
def valid_text(value, field, maximum, allow_empty=False):
if not isinstance(value, str):
raise bad("Поле %s должно быть строкой" % field)
try:
value.encode("utf-8")
except UnicodeEncodeError:
raise bad("Поле %s содержит некорректные символы" % field) from None
if "\x00" in value:
raise bad("Поле %s содержит NUL" % field)
if len(value) > maximum:
raise bad("Поле %s слишком длинное" % field)
if not allow_empty and not value.strip():
raise bad("Поле %s не может быть пустым" % field)
return value
def _reject_constant(name):
raise ValueError(name)
def parse_json_object(raw, required, allowed):
try:
data = json.loads(raw.decode("utf-8"), parse_constant=_reject_constant)
except (UnicodeDecodeError, ValueError, RecursionError):
raise bad("Тело запроса — некорректный JSON") from None
if not isinstance(data, dict):
raise bad("Ожидается JSON-объект")
if set(data) - allowed:
raise bad("Лишние поля в запросе")
if any(key not in data for key in required):
raise bad("Не хватает обязательных полей")
return data
def encode_cursor(kind, resource, key):
payload = json.dumps({"k": kind, "r": resource, "t": key[0], "i": key[1]}, separators=(",", ":"))
return base64.urlsafe_b64encode(payload.encode()).decode().rstrip("=")
def decode_cursor(text, kind, resource):
try:
data = json.loads(base64.urlsafe_b64decode(text + "=" * (-len(text) % 4)).decode())
if (not isinstance(data, dict) or data.get("k") != kind or data.get("r") != resource
or not isinstance(data.get("t"), int) or isinstance(data.get("t"), bool)
or not isinstance(data.get("i"), str) or not UUID_RE.fullmatch(data["i"])
or not -62135596800000000 < data["t"] < 253402300799000000):
raise ValueError
return data["t"], data["i"]
except (ValueError, UnicodeError):
raise bad("Некорректный cursor") from None
def page_params(query, kind, resource):
limit, cursor = 50, None
if "limit" in query:
if len(query["limit"]) != 1 or not LIMIT_RE.fullmatch(query["limit"][0]):
raise bad("limit должен быть целым числом 1–100")
limit = int(query["limit"][0])
if not 1 <= limit <= 100:
raise bad("limit должен быть целым числом 1–100")
if "cursor" in query:
if len(query["cursor"]) != 1 or not query["cursor"][0]:
raise bad("Некорректный cursor")
cursor = decode_cursor(query["cursor"][0], kind, resource)
return limit, cursor
def parse_query(raw):
result = {}
if raw:
for key, value in parse_qsl(raw.decode("latin-1"), keep_blank_values=True):
result.setdefault(key, []).append(value)
return result
class Request:
def __init__(self, scope, receive, request_id):
self.method = scope["method"]
self.path = scope["path"]
self.query = parse_query(scope.get("query_string", b""))
self.headers = {k.decode("latin-1").lower(): v.decode("latin-1") for k, v in scope["headers"]}
self.receive = receive
self.request_id = request_id
self.user_id = None
async def body(self):
declared = self.headers.get("content-length")
if declared is not None:
if not declared.isdigit():
raise bad("Некорректный Content-Length")
if int(declared) > config.MAX_BODY_BYTES:
raise ApiError(413, "payload_too_large", "Тело запроса слишком большое", {"connection": "close"})
chunks, size = [], 0
while True:
event = await self.receive()
if event["type"] == "http.disconnect":
raise bad("Соединение закрыто клиентом")
chunk = event.get("body", b"")
size += len(chunk)
if size > config.MAX_BODY_BYTES:
raise ApiError(413, "payload_too_large", "Тело запроса слишком большое", {"connection": "close"})
chunks.append(chunk)
if not event.get("more_body"):
return b"".join(chunks)
def require_json_media_type(self):
media = self.headers.get("content-type", "").split(";")[0].strip().lower()
if media != "application/json":
raise ApiError(415, "unsupported_media_type", "Ожидается Content-Type: application/json")
async def respond(send, status, body, request_id, extra=None, instance_id=None):
payload = b"" if status == 204 else json.dumps(body, ensure_ascii=False, separators=(",", ":")).encode("utf-8")
headers = [(b"x-request-id", request_id.encode())]
if status != 204:
headers += [(b"content-type", b"application/json"), (b"content-length", str(len(payload)).encode())]
else:
headers.append((b"content-length", b"0"))
if instance_id:
headers.append((b"x-instance-id", instance_id.encode()))
headers += [(k.encode(), v.encode()) for k, v in (extra or {}).items()]
await send({"type": "http.response.start", "status": status, "headers": headers})
await send({"type": "http.response.body", "body": payload})
def new_request_id():
return uuid.uuid4().hex
+345
View File
@@ -0,0 +1,345 @@
"""Worker вложений: python -m app.worker
Конвейер задания (jobs.schema.json, состояния — contracts/attachment-states.md):
claim (queued -> processing, lease) -> чтение staging -> варианты (Pillow, в потоках) -> запись в конечный S3-prefix
-> ОДНА транзакция: варианты + ready -> ack -> очистка staging.
Ack отправляется только после устойчивого эффекта (COMMIT результата). Повторная доставка уже готовой задачи — no-op с ack.
Временные ошибки: возврат в queued с backoff (экспоненциальный + jitter) через outbox; после MAX_ATTEMPTS — failed/retry_exhausted.
Постоянные ошибки (невалидное изображение): failed с error_code сразу. Зависшие processing возвращает sweeper по истечении lease.
"""
import asyncio
import json
import os
import random
import signal
import tempfile
import time
import uuid
from concurrent.futures import ThreadPoolExecutor
import aio_pika
from . import config, pipeline, telemetry
from .broker import QUEUE, Broker
from .db import Database
from .outbox import Relay
from .s3 import S3
from .web import log
WORKER_ID = "%s-%s" % ((os.environ.get("HOSTNAME") or "worker")[:12], uuid.uuid4().hex[:6])
def backoff_seconds(attempt):
"""Экспоненциальная задержка с полным jitter: случайное значение в [0, min(cap, base * 2^(attempt-1))]."""
ceiling = min(config.RETRY_MAX_DELAY, config.RETRY_BASE_DELAY * (2 ** max(0, attempt - 1)))
return random.uniform(ceiling * 0.5, ceiling)
class Worker:
def __init__(self):
self.db, self.s3, self.broker = Database(), S3(), Broker()
self.relay = Relay(self.db, self.broker)
self.pool = ThreadPoolExecutor(max_workers=config.WORKER_CONCURRENCY, thread_name_prefix="img")
self.busy = 0
self.stopping = False
self.tasks = []
telemetry.gauge("messenger.worker.busy", lambda: [(self.busy, {})], "{job}", "Задания в обработке")
telemetry.gauge("messenger.worker.capacity", lambda: [(config.WORKER_CONCURRENCY, {})], "{job}", "Ёмкость worker")
# --- claim / finalize ---------------------------------------------------------
async def claim(self, job):
return await self.db.fetchrow(
"UPDATE attachments SET status = 'processing', lease_owner = $2, processing_started_at = now(), "
"lease_expires_at = now() + make_interval(secs => $3), attempt = $4, job_id = $5, updated_at = now() "
"WHERE id = $1 AND status = 'queued' AND pipeline_version <= $6 "
"RETURNING id, chat_id, staging_key, s3_key, content_type, size_bytes, attempt",
uuid.UUID(job["attachment_id"]), WORKER_ID, float(config.LEASE_SECONDS), job["attempt"],
uuid.UUID(job["job_id"]), job["pipeline_version"], retry=False)
async def fail(self, att_id, job, code, event="failed"):
await self.db.execute(
"WITH u AS (UPDATE attachments SET status = 'failed', error_code = $2, lease_owner = NULL, lease_expires_at = NULL, "
"updated_at = now() WHERE id = $1 AND status = 'processing' AND lease_owner = $3 RETURNING id) "
"INSERT INTO job_events (attachment_id, job_id, event, attempt, detail) SELECT id, $4, $5, $6, $2 FROM u",
att_id, code, WORKER_ID, uuid.UUID(job["job_id"]), event, job["attempt"])
async def reschedule(self, att_id, job, reason):
"""Временная ошибка: processing -> queued, новая запись outbox с отложенной публикацией (backoff + jitter)."""
nxt = job["attempt"] + 1
if nxt > config.MAX_ATTEMPTS:
log("warning", "job_retry_exhausted", attachment_id=str(att_id), attempts=job["attempt"])
await self.fail(att_id, job, "retry_exhausted", "retry_exhausted")
return
delay = backoff_seconds(job["attempt"])
await self.db.execute(
"WITH u AS (UPDATE attachments SET status = 'queued', lease_owner = NULL, lease_expires_at = NULL, updated_at = now() "
"WHERE id = $1 AND status = 'processing' AND lease_owner = $2 RETURNING id, pipeline_version), "
"o AS (INSERT INTO outbox (job_id, attachment_id, pipeline_version, attempt, available_at, traceparent, tracestate) "
"SELECT $3, id, pipeline_version, $4, now() + make_interval(secs => $5), $7, $8 FROM u RETURNING 1) "
"INSERT INTO job_events (attachment_id, job_id, event, attempt, detail) SELECT id, $3, 'retry_scheduled', $4, $6 FROM u",
att_id, WORKER_ID, uuid.UUID(job["job_id"]), nxt, delay, reason, job.get("traceparent"), job.get("tracestate"))
log("info", "job_rescheduled", attachment_id=str(att_id), next_attempt=nxt, delay_seconds=round(delay, 2), reason=reason)
async def publish_result(self, row, job, variants):
"""Одна транзакция: варианты (идемпотентно) + ready. Только владелец lease может опубликовать результат."""
async with self.db.conn() as c:
async with c.transaction():
owner = await c.fetchval("SELECT 1 FROM attachments WHERE id = $1 AND status = 'processing' AND lease_owner = $2 "
"FOR UPDATE", row["id"], WORKER_ID)
if owner is None:
return False # lease потерян (sweeper вернул задачу): результат не публикуем, чужой повтор доделает
for v in variants:
await c.execute(
"INSERT INTO attachment_variants (attachment_id, pipeline_version, name, content_type, size_bytes, width, "
"height, sha256, s3_key) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9) "
"ON CONFLICT (attachment_id, pipeline_version, name) DO NOTHING",
row["id"], job["pipeline_version"], v["name"], v["content_type"], len(v["data"]), v["width"], v["height"],
v["sha256"], "att/%s/v%d/%s" % (row["id"], job["pipeline_version"], v["name"]))
await c.execute("UPDATE attachments SET status = 'ready', error_code = NULL, ready_at = now(), updated_at = now(), "
"lease_owner = NULL, lease_expires_at = NULL WHERE id = $1", row["id"])
await c.execute("INSERT INTO job_events (attachment_id, job_id, event, attempt) VALUES ($1, $2, 'ready', $3)",
row["id"], uuid.UUID(job["job_id"]), job["attempt"])
return True
# --- обработка ----------------------------------------------------------------
async def process(self, job):
att_id = uuid.UUID(job["attachment_id"])
row = await self.claim(job)
if row is None:
state = await self.db.fetchval("SELECT status FROM attachments WHERE id = $1", att_id)
log("info", "job_noop", attachment_id=job["attachment_id"], status=state or "unknown")
telemetry.record_job("attachment.process.v1", "noop", 0.0)
return # ready/failed/processing у другого владельца/неизвестное вложение: идемпотентный no-op, ack
started = time.perf_counter()
outcome = "ready"
try:
await self.fault("after_claim")
stream, _ = await self.s3.open_stream(row["staging_key"])
spool = tempfile.SpooledTemporaryFile(max_size=2 * 1024 * 1024)
try:
try:
while True:
chunk = await stream.read(256 * 1024)
if not chunk:
break
spool.write(chunk)
finally:
stream.close()
loop = asyncio.get_running_loop()
variants = await asyncio.wait_for(loop.run_in_executor(self.pool, pipeline.process, spool), config.JOB_TIMEOUT)
finally:
spool.close()
# оригинал и варианты в конечный prefix: ключи детерминированы, повторная запись перезаписывает те же байты
await self.copy_original(row)
for v in variants:
await self.s3.put_bytes("att/%s/v%d/%s" % (row["id"], job["pipeline_version"], v["name"]), v["data"], v["content_type"])
await self.fault("after_s3_before_commit")
if not await self.publish_result(row, job, variants):
log("warning", "lease_lost", attachment_id=job["attachment_id"])
return
await self.fault("after_commit_before_ack")
log("info", "job_ready", attachment_id=job["attachment_id"], attempt=job["attempt"],
duration_ms=round((time.perf_counter() - started) * 1000, 1))
try:
await self.s3.delete(row["staging_key"]) # staging безопасно удалять после COMMIT результата
except Exception as exc:
log("warning", "staging_cleanup_failed", error_type=type(exc).__name__)
except pipeline.PermanentError as exc:
outcome = "failed"
await self.fail(att_id, job, exc.code)
log("warning", "job_failed", attachment_id=job["attachment_id"], error_code=exc.code)
except MemoryError:
outcome = "failed"
await self.fail(att_id, job, "processing_failed")
log("error", "job_failed_oom", attachment_id=job["attachment_id"])
except asyncio.CancelledError:
raise
except InjectedCrash:
raise
except Exception as exc: # S3/БД/таймаут/неожиданное: временная ошибка
outcome = "retried"
await self.reschedule(att_id, job, type(exc).__name__)
finally:
ttr = None
if outcome == "ready":
ttr = await self.db.fetchval("SELECT extract(epoch FROM ready_at - accepted_at) FROM attachments WHERE id = $1 "
"AND status = 'ready'", att_id)
telemetry.record_job("attachment.process.v1", outcome, time.perf_counter() - started, float(ttr) if ttr is not None else None)
async def copy_original(self, row):
stream, size = await self.s3.open_stream(row["staging_key"])
spool = tempfile.SpooledTemporaryFile(max_size=2 * 1024 * 1024)
try:
while True:
chunk = await stream.read(256 * 1024)
if not chunk:
break
spool.write(chunk)
stream.close()
spool.seek(0)
await self.s3.put(row["s3_key"], spool, row["content_type"], size)
finally:
spool.close()
async def fault(self, point):
"""Точки внедрения сбоев для экспериментов (FAULT_INJECTION=crash:<точка> => процесс умирает мгновенно)."""
if "crash_once:" + point in config.FAULT_INJECTION:
marker = "/tmp/fault-%s.done" % point # сохраняется при рестарте того же контейнера -> сбой ровно один раз
if not os.path.exists(marker):
open(marker, "w").close()
log("error", "fault_crash_once", point=point)
os._exit(137)
if "crash:" + point in config.FAULT_INJECTION:
log("error", "fault_crash", point=point)
os._exit(137)
if "fail:" + point in config.FAULT_INJECTION:
raise OSError("injected failure at " + point)
async def on_message(self, message):
try:
job = json.loads(message.body)
if (job.get("schema_version") != 1 or job.get("type") != "attachment.process.v1"
or not isinstance(job.get("attempt"), int) or not isinstance(job.get("pipeline_version"), int)):
raise ValueError("bad job")
uuid.UUID(job["attachment_id"]), uuid.UUID(job["job_id"])
except Exception:
log("error", "job_unreadable")
await message.reject(requeue=False) # -> DLQ
return
self.busy += 1
carrier = {k: job[k] for k in ("traceparent", "tracestate") if k in job}
try:
with telemetry.span("attachment.process process", "consumer",
{"messaging.system": "rabbitmq", "messaging.destination.name": "attachment.process",
"messaging.operation": "process", "job.type": job["type"], "job.attempt": job["attempt"]}, carrier):
await self.process(job)
await message.ack()
except InjectedCrash:
raise
except Exception as exc:
log("error", "job_unhandled", error_type=type(exc).__name__)
await message.nack(requeue=True)
finally:
self.busy -= 1
# --- sweeper -------------------------------------------------------------------
async def sweep_once(self):
"""Истёкший lease -> queued (новая попытка) или failed/retry_exhausted; чистка старого staging."""
rows = await self.db.fetch(
"WITH exp AS (SELECT id FROM attachments WHERE status = 'processing' AND lease_expires_at < now() FOR UPDATE SKIP LOCKED), "
"u AS (UPDATE attachments a SET status = CASE WHEN a.attempt >= $1 THEN 'failed' ELSE 'queued' END, "
"error_code = CASE WHEN a.attempt >= $1 THEN 'retry_exhausted' ELSE NULL END, lease_owner = NULL, lease_expires_at = NULL, "
"updated_at = now() FROM exp WHERE a.id = exp.id RETURNING a.id, a.status, a.pipeline_version, a.attempt, a.job_id), "
"o AS (INSERT INTO outbox (job_id, attachment_id, pipeline_version, attempt, traceparent, tracestate) "
"SELECT COALESCE(job_id, gen_random_uuid()), id, pipeline_version, attempt + 1, "
"(SELECT traceparent FROM outbox WHERE attachment_id = u.id ORDER BY id DESC LIMIT 1), "
"(SELECT tracestate FROM outbox WHERE attachment_id = u.id ORDER BY id DESC LIMIT 1) FROM u WHERE status = 'queued' RETURNING 1), "
"e AS (INSERT INTO job_events (attachment_id, job_id, event, attempt, detail) SELECT id, job_id, 'lease_expired', attempt, status FROM u RETURNING 1) "
"SELECT id, status FROM u", config.MAX_ATTEMPTS)
for r in rows:
log("warning", "lease_expired", attachment_id=str(r["id"]), new_status=r["status"])
if rows:
self.relay.kick()
async def sweeper(self):
while not self.stopping:
try:
await self.sweep_once()
except Exception as exc:
log("warning", "sweeper_failed", error_type=type(exc).__name__)
await asyncio.sleep(config.SWEEP_INTERVAL)
async def staging_janitor(self):
"""Удаляет staging для ready (если не удалён после COMMIT) и для failed старше retention; текущую работу не трогает."""
while not self.stopping:
await asyncio.sleep(max(30, config.SWEEP_INTERVAL * 6))
try:
rows = await self.db.fetch(
"SELECT id, staging_key FROM attachments WHERE staging_key IS NOT NULL AND (status = 'ready' OR "
"(status = 'failed' AND updated_at < now() - make_interval(secs => $1))) LIMIT 200",
float(config.STAGING_FAILED_RETENTION_SECONDS))
for r in rows:
await self.s3.delete(r["staging_key"])
await self.db.execute("UPDATE attachments SET staging_key = NULL WHERE id = $1", r["id"])
except Exception as exc:
log("warning", "staging_janitor_failed", error_type=type(exc).__name__)
# --- жизненный цикл ------------------------------------------------------------
async def connect_broker(self):
while not self.stopping:
try:
await self.broker.ensure(prefetch=config.WORKER_CONCURRENCY)
return
except Exception as exc:
log("warning", "broker_unavailable", error_type=type(exc).__name__)
await asyncio.sleep(2)
async def consume(self):
while not self.stopping:
await self.connect_broker()
try:
queue = await self.broker.channel.declare_queue(QUEUE, durable=True, passive=True)
tag = await queue.consume(self.on_message)
log("info", "consuming", worker=WORKER_ID, prefetch=config.WORKER_CONCURRENCY)
while not self.stopping and not self.broker.channel.is_closed:
await asyncio.sleep(1)
if self.stopping:
await queue.cancel(tag)
return
except Exception as exc:
log("warning", "consume_failed", error_type=type(exc).__name__)
await asyncio.sleep(2)
async def run(self):
telemetry.setup(config.SERVICE_NAME)
telemetry.process_metrics()
await self.db.start()
await self.s3.start()
log("info", "worker_started", worker=WORKER_ID, concurrency=config.WORKER_CONCURRENCY, lease=config.LEASE_SECONDS,
max_attempts=config.MAX_ATTEMPTS)
loop = asyncio.get_running_loop()
for sig in (signal.SIGTERM, signal.SIGINT):
try:
loop.add_signal_handler(sig, self.request_stop)
except NotImplementedError:
signal.signal(sig, lambda *_: self.request_stop())
self.tasks = [asyncio.create_task(self.consume()), asyncio.create_task(self.relay.run()),
asyncio.create_task(self.sweeper()), asyncio.create_task(self.staging_janitor())]
beat = 0.0
while not self.stopping:
if time.monotonic() - beat > 5: # heartbeat для HEALTHCHECK контейнера
beat = time.monotonic()
try:
with open("/tmp/worker.alive", "w") as fh:
fh.write(str(time.time()))
except OSError:
pass
await asyncio.sleep(0.5)
# graceful: перестаём брать новые задания, ждём текущие до GRACEFUL_SHUTDOWN_SECONDS
deadline = time.monotonic() + config.GRACEFUL_SHUTDOWN_SECONDS
while self.busy and time.monotonic() < deadline:
await asyncio.sleep(0.2)
self.relay.stop()
for t in self.tasks:
t.cancel()
await self.broker.stop()
await self.s3.stop()
await self.db.stop()
log("info", "worker_stopped", unfinished=self.busy)
telemetry.shutdown()
def request_stop(self):
log("info", "worker_stopping")
self.stopping = True
class InjectedCrash(Exception):
pass
def main():
asyncio.run(Worker().run())
if __name__ == "__main__":
main()
+32
View File
@@ -0,0 +1,32 @@
# Собственные проверки ЛР6
Запустите стенд по REPORT.md. Python 3.11+ с `python -m pip install -r research/requirements.txt` включает Pillow для пиксельных проверок. Публичным tests/ достаточно Python 3.9+. В Windows используйте `PYTHONUTF8=1`.
Все сценарии выполняются **последовательно** на учебном стенде: часть тестов явно останавливает сервисы, внедряет отказы и восстанавливает настройки. Volumes не удаляются. Профиль для проверок наблюдаемости — OTEL_ENABLED=1, head sampling 1.0, обычный collector.yaml.
```sh
python scripts/check_contract.py
python scripts/validate_openapi.py # предварительно установить requirements-dev.txt
python tests/smoke.py --base-url http://127.0.0.1:8080
for suite in test_api test_idempotency test_attachments test_topology test_pipeline test_observability test_observability_retry test_reconcile5 test_resilience5 test_expiry test_dependencies; do
python -m unittest discover -s student-tests -p "$suite.py" -v || exit 1
done
python tests/resilience.py --mode persistence --action-script scripts/restart.sh --base-url http://127.0.0.1:8080
python tests/resilience.py --mode failover --action-script scripts/stop-one.sh --base-url http://127.0.0.1:8080
sh scripts/restore-replicas.sh
```
В Windows публичным resilience передаются restart.cmd / stop-one.cmd; восстановление API — `docker compose up -d --wait --no-recreate --no-deps api`. Failover повторяется с восстановлением, чтобы проверить обе реплики. Фактические остановленные контейнеры и перезапуск всех сервисов сохранены в evidence/recheck_2026-11-12/.
Набор содержит 66 тестов: API 12, идемпотентность 9, вложения 11, топология 6, pipeline 9, наблюдаемость 5, retry trace 1, reconciler 2, отказы pipeline 7, TTL 2, зависимости 2. Проверки наблюдаемости ищут реальные metrics/logs/traces, связь API → producer → consumer → S3/DB, retry в той же трассе, низкую кардинальность и отсутствие синтетических секретов в сигналах.
Исследования для выбранного трека (запускать после тестов, не одновременно):
```sh
mkdir -p evidence/my-run
python research/incidents.py --out evidence/my-run
python research/obs_cost.py --out evidence/my-run --profiles off on100 head10 tail --repeat 2
python research/plot_obs_cost.py --out evidence/my-run
```
Четыре контролируемых инцидента проверяют firing/resolved; off/on100/head10/tail сравниваются при одинаковых ресурсах, 64 клиентах, 8 процессах, 5 с прогрева и 30 с измерения (2 повтора). Эти прогоны занимают около 20 минут. Секреты создаются scripts/init-secrets.sh и в Git не входят. Все команды и реальные результаты текущей проверки — evidence/checks_2026-11-12.txt и evidence/recheck_2026-11-12/.
Binary file not shown.

After

Width:  |  Height:  |  Size: 344 B

Binary file not shown.

After

Width:  |  Height:  |  Size: 1.6 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 151 B

Binary file not shown.

After

Width:  |  Height:  |  Size: 24 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 109 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 132 B

+208
View File
@@ -0,0 +1,208 @@
"""Собственные тесты lab2 против запущенного стенда (docker compose up -d).
python -m unittest discover -s student-tests -v # BASE_URL по умолчанию http://127.0.0.1:8080
Тесты используют только синтетические данные и ничего не удаляют. Проверки expiry и недоступности
зависимостей лежат отдельно (test_expiry.py, test_dependencies.py): они управляют контейнерами.
"""
import base64
import json
import os
import subprocess
import unittest
import urllib.error
import urllib.request
import uuid
from concurrent.futures import ThreadPoolExecutor
from urllib.parse import quote
BASE = os.environ.get("BASE_URL", "http://127.0.0.1:8080").rstrip("/")
ROOT = os.path.join(os.path.dirname(__file__), "..")
def call(method, path, body=None, token=None, headers=None, raw=None):
h = dict(headers or {})
if method == "POST" and path.endswith("/messages") and headers is None:
h["Idempotency-Key"] = uuid.uuid4().hex # lab3: ключ обязателен; тесты без ключа — в test_idempotency.py
data = raw
if body is not None:
data = json.dumps(body, ensure_ascii=False).encode("utf-8")
h.setdefault("Content-Type", "application/json")
if token:
h["Authorization"] = "Bearer " + token
req = urllib.request.Request(BASE + path, data=data, headers=h, method=method)
try:
with urllib.request.urlopen(req, timeout=10) as resp:
status, hdrs, payload = resp.status, resp.headers, resp.read()
except urllib.error.HTTPError as err:
status, hdrs, payload = err.code, err.headers, err.read()
return status, hdrs, (json.loads(payload) if payload else None)
def basic(username, password):
return {"Authorization": "Basic " + base64.b64encode((username + ":" + password).encode("utf-8")).decode()}
def register(prefix="u"):
name = "%s_%s" % (prefix, uuid.uuid4().hex[:12])
password = "Pa55:word-" + uuid.uuid4().hex # двоеточие в пароле допустимо
status, _, user = call("POST", "/api/v1/users", {"username": name, "display_name": "Имя " + name, "password": password})
assert status == 201, user
status, _, session = call("POST", "/api/v1/auth/sessions", headers=basic(name, password))
assert status == 201, session
return {"user": user, "password": password, "token": session["token"], "session": session}
def psql(sql):
out = subprocess.run(["docker", "compose", "--env-file", ".env", "exec", "-T", "postgres", "psql", "-U", "messenger",
"-d", "messenger", "-Atc", sql], cwd=ROOT, capture_output=True, text=True, check=True)
return out.stdout.strip()
class AuthTest(unittest.TestCase):
def assertError(self, result, status, code):
self.assertEqual(result[0], status, result[2])
self.assertEqual(result[2]["error"]["code"], code)
self.assertEqual(result[2]["error"]["request_id"], result[1]["X-Request-Id"])
def test_register_validation(self):
base = {"username": "val_" + uuid.uuid4().hex[:8], "display_name": "x", "password": "p" * 12}
for patch in ({"password": "short"}, {"password": "p" * 129}, {"password": 12345678901234}, {"extra": 1},
{"username": "BAD"}, {"display_name": " "}):
body = dict(base, **patch)
self.assertError(call("POST", "/api/v1/users", body), 400, "invalid_request")
missing = {k: v for k, v in base.items() if k != "password"}
self.assertError(call("POST", "/api/v1/users", missing), 400, "invalid_request")
self.assertEqual(call("POST", "/api/v1/users", dict(base, password="я" * 12))[0], 201) # 12 code points
def test_password_is_not_exposed_or_stored_plain(self):
actor = register()
self.assertEqual(set(actor["user"]), {"id", "username", "display_name", "created_at"})
stored = psql("SELECT password_hash FROM users WHERE username = '%s'" % actor["user"]["username"])
self.assertTrue(stored.startswith("$argon2id$"), stored[:12])
self.assertNotIn(actor["password"], stored)
def test_wrong_password_and_unknown_user_are_indistinguishable(self):
actor = register()
wrong = call("POST", "/api/v1/auth/sessions", headers=basic(actor["user"]["username"], "wrong-password-123"))
unknown = call("POST", "/api/v1/auth/sessions", headers=basic("nobody_" + uuid.uuid4().hex[:8], "wrong-password-123"))
for r in (wrong, unknown):
self.assertError(r, 401, "unauthorized")
self.assertEqual(r[1]["WWW-Authenticate"], 'Basic realm="messenger", charset="UTF-8"')
self.assertEqual(wrong[2]["error"]["message"], unknown[2]["error"]["message"])
for bad in ({}, {"Authorization": "Basic !!!"}, {"Authorization": "Basic " + base64.b64encode(b"nocolon").decode()},
{"Authorization": "Bearer abc"}):
self.assertEqual(call("POST", "/api/v1/auth/sessions", headers=bad)[0], 401)
def test_bearer_rules(self):
actor = register()
self.assertEqual(call("GET", "/api/v1/users/me", token=actor["token"])[2], actor["user"])
for headers in ({}, basic(actor["user"]["username"], actor["password"]), {"X-User-Id": actor["user"]["id"]},
{"Authorization": "Bearer " + "A" * 43}, {"Authorization": "Bearer short"}):
r = call("GET", "/api/v1/users/me", headers=headers)
self.assertError(r, 401, "unauthorized")
self.assertEqual(r[1]["WWW-Authenticate"], 'Bearer realm="messenger"')
other = register()
me = call("GET", "/api/v1/users/me", token=actor["token"], headers={"X-User-Id": other["user"]["id"]})[2]
self.assertEqual(me["id"], actor["user"]["id"]) # X-User-Id игнорируется
def test_session_shape_entropy_and_logout(self):
actor = register()
s = actor["session"]
self.assertEqual(s["token_type"], "Bearer")
self.assertGreaterEqual(len(s["token"]), 43) # 32 байта в base64url = 256 бит
self.assertNotIn(actor["user"]["id"], s["token"])
self.assertNotEqual(register()["token"], s["token"])
self.assertEqual(call("DELETE", "/api/v1/auth/sessions/current", token=s["token"])[0], 204)
self.assertError(call("GET", "/api/v1/users/me", token=s["token"]), 401, "unauthorized")
self.assertError(call("DELETE", "/api/v1/auth/sessions/current", token=s["token"]), 401, "unauthorized")
def test_logout_revokes_in_database_and_cache(self):
actor = register()
call("GET", "/api/v1/users/me", token=actor["token"]) # прогреваем кэш
call("DELETE", "/api/v1/auth/sessions/current", token=actor["token"])
revoked = psql("SELECT count(*) FROM sessions s JOIN users u ON u.id = s.user_id "
"WHERE u.username = '%s' AND s.revoked_at IS NOT NULL" % actor["user"]["username"])
self.assertEqual(revoked, "1")
def test_token_only_stored_as_hash(self):
actor = register()
count = psql("SELECT count(*) FROM sessions WHERE token_hash = '\\x00'::bytea OR encode(token_hash,'escape') = '%s'" % actor["token"])
self.assertEqual(count, "0")
def test_concurrent_registration_of_same_username(self):
name = "race_" + uuid.uuid4().hex[:10]
body = {"username": name, "display_name": "x", "password": "racing-password-1"}
with ThreadPoolExecutor(12) as pool:
codes = sorted(pool.map(lambda _: call("POST", "/api/v1/users", body)[0], range(12)))
self.assertEqual(codes, [201] + [409] * 11)
self.assertEqual(psql("SELECT count(*) FROM users WHERE username = '%s'" % name), "1")
class DataTest(unittest.TestCase):
def test_chat_creation_is_atomic(self):
a = register()
before = psql("SELECT (SELECT count(*) FROM chats) || '/' || (SELECT count(*) FROM chat_members)")
ghost = str(uuid.uuid4())
r = call("POST", "/api/v1/chats", {"title": "откат", "member_ids": [a["user"]["id"], ghost]}, token=a["token"])
self.assertEqual(r[0], 400)
after = psql("SELECT (SELECT count(*) FROM chats) || '/' || (SELECT count(*) FROM chat_members)")
self.assertEqual(before, after, "после неудачного создания не должно остаться чата или участников")
ok = call("POST", "/api/v1/chats", {"title": "ок", "member_ids": [a["user"]["id"]]}, token=a["token"])
self.assertEqual(ok[0], 201)
def test_foreign_chat_empty_message_bad_json(self):
a, b, eve = register(), register(), register()
chat = call("POST", "/api/v1/chats", {"title": "ЛС", "member_ids": [a["user"]["id"], b["user"]["id"]]}, a["token"])[2]
path = "/api/v1/chats/%s/messages" % chat["id"]
self.assertEqual(call("POST", path, {"text": "чужое"}, eve["token"])[0], 404)
self.assertEqual(call("GET", path, token=eve["token"])[0], 404)
self.assertEqual(call("POST", path, {"text": " "}, a["token"])[0], 400)
self.assertEqual(call("POST", path, raw=b"{oops", token=a["token"], headers={"Content-Type": "application/json"})[0], 400)
self.assertEqual(call("POST", path, {"text": "x", "sender_id": b["user"]["id"]}, a["token"])[0], 400)
sent = call("POST", path, {"text": "Привет 👋"}, a["token"])[2]
self.assertEqual(sent["sender_id"], a["user"]["id"])
def test_multi_page_history_and_cursor_binding(self):
a, b = register(), register()
chat = call("POST", "/api/v1/chats", {"title": "стр", "member_ids": [a["user"]["id"], b["user"]["id"]]}, a["token"])[2]
other = call("POST", "/api/v1/chats", {"title": "др", "member_ids": [a["user"]["id"]]}, a["token"])[2]
path = "/api/v1/chats/%s/messages" % chat["id"]
sent = [call("POST", path, {"text": "m%d" % i}, (a if i % 2 else b)["token"])[2] for i in range(23)]
walked, cursor, pages = [], None, 0
while True:
q = "?limit=%d" % (5 if pages % 2 == 0 else 8) + ("&cursor=" + quote(cursor, safe="") if cursor else "")
page = call("GET", path + q, token=b["token"])[2]
walked += page["items"]
pages += 1
cursor = page["next_cursor"]
if not cursor:
break
self.assertEqual(walked, sent)
self.assertGreaterEqual(pages, 3)
first = call("GET", path + "?limit=1", token=a["token"])[2]["next_cursor"]
r = call("GET", "/api/v1/chats/%s/messages?cursor=%s" % (other["id"], quote(first, safe="")), token=a["token"])
self.assertEqual(r[0], 400)
def test_parallel_writes_from_many_clients(self):
users = [register() for _ in range(5)]
chat = call("POST", "/api/v1/chats", {"title": "гонки", "member_ids": [u["user"]["id"] for u in users]}, users[0]["token"])[2]
path = "/api/v1/chats/%s/messages" % chat["id"]
def client(u):
return [call("POST", path, {"text": "%s-%d" % (u["user"]["username"], i)}, u["token"])[0] for i in range(20)]
with ThreadPoolExecutor(5) as pool:
codes = [c for part in pool.map(client, users) for c in part]
self.assertEqual(codes, [201] * 100)
total, cursor = [], None
while True:
page = call("GET", path + "?limit=100" + ("&cursor=" + quote(cursor, safe="") if cursor else ""), token=users[0]["token"])[2]
total += page["items"]
cursor = page["next_cursor"]
if not cursor:
break
self.assertEqual(len({m["id"] for m in total}), 100)
if __name__ == "__main__":
unittest.main()
+205
View File
@@ -0,0 +1,205 @@
"""Собственные тесты вложений lab6 (негативные сценарии и права). Запуск:
python -m unittest discover -s student-tests -p "test_attachments.py" -v (BASE_URL по умолчанию http://127.0.0.1:8080)
"""
import hashlib
import json
import os
import socket
import struct
import sys
import time
import unittest
import urllib.error
import urllib.parse
import urllib.request
import uuid
import zlib
sys.path.insert(0, os.path.dirname(__file__))
from test_api import BASE, ROOT, call, psql, register # noqa: E402
FIX = os.path.join(os.path.dirname(__file__), "fixtures")
PNG = open(os.path.join(FIX, "sample.png"), "rb").read()
JPG = open(os.path.join(FIX, "sample.jpg"), "rb").read()
def upload(token, chat_id, data, ctype="image/png", filename="sample.png", raw_filename=None):
qs = raw_filename if raw_filename is not None else "filename=" + filename
req = urllib.request.Request("%s/api/v1/chats/%s/attachments?%s" % (BASE, chat_id, qs), data=data, method="POST",
headers={"Authorization": "Bearer " + token, "Content-Type": ctype})
try:
with urllib.request.urlopen(req, timeout=30) as r:
return r.status, r.headers, json.loads(r.read())
except urllib.error.HTTPError as err:
body = err.read()
return err.code, err.headers, (json.loads(body) if body else None)
def download(token, att_id, suffix="/content"):
req = urllib.request.Request("%s/api/v1/attachments/%s%s" % (BASE, att_id, suffix),
headers={"Authorization": "Bearer " + token if token else ""})
try:
with urllib.request.urlopen(req, timeout=30) as r:
return r.status, r.headers, r.read()
except urllib.error.HTTPError as err:
return err.code, err.headers, err.read()
def wait_ready(token, att_id, seconds=60):
"""lab6: загрузка возвращает 202 queued; готовность определяется опросом метаданных."""
deadline = time.time() + seconds
while time.time() < deadline:
code, _, body = download(token, att_id, "")
att = json.loads(body)
if att["status"] == "ready":
return att
assert att["status"] != "failed", att
time.sleep(0.3)
raise AssertionError("not ready")
def png_chunk(kind, data):
return struct.pack(">I", len(data)) + kind + data + struct.pack(">I", zlib.crc32(kind + data) & 0xFFFFFFFF)
def bomb_png(width=20000, height=20000):
"""Корректный по структуре PNG, заявляющий 400 млн пикселей, без данных изображения (отклоняется по заголовку)."""
ihdr = struct.pack(">IIBBBBB", width, height, 8, 2, 0, 0, 0)
return b"\x89PNG\r\n\x1a\n" + png_chunk(b"IHDR", ihdr) + png_chunk(b"IDAT", zlib.compress(b"\x00" * 16)) + png_chunk(b"IEND", b"")
class AttachmentTest(unittest.TestCase):
def setUp(self):
self.a, self.b, self.eve = register("att"), register("att"), register("att")
self.chat = call("POST", "/api/v1/chats", {"title": "файлы", "member_ids": [self.a["user"]["id"], self.b["user"]["id"]]}, self.a["token"])[2]
self.other = call("POST", "/api/v1/chats", {"title": "другой", "member_ids": [self.a["user"]["id"]]}, self.a["token"])[2]
def assertError(self, result, status, code=None):
self.assertEqual(result[0], status, result[2])
if code:
self.assertEqual(result[2]["error"]["code"], code)
def test_png_and_jpeg_roundtrip_with_checksum(self):
for data, ctype, name in ((PNG, "image/png", "фото.png"), (JPG, "image/jpeg", "photo.jpg")):
status, headers, att = upload(self.a["token"], self.chat["id"], data, ctype, urllib.parse.quote(name))
self.assertEqual(status, 202, att)
self.assertEqual((att["status"], att["variants"], att["error_code"]), ("queued", [], None))
wait_ready(self.b["token"], att["id"])
self.assertEqual((att["size_bytes"], att["sha256"], att["content_type"]), (len(data), hashlib.sha256(data).hexdigest(), ctype))
self.assertEqual((att["chat_id"], att["owner_id"], att["filename"]), (self.chat["id"], self.a["user"]["id"], name))
code, hdr, body = download(self.b["token"], att["id"]) # другой участник чата
self.assertEqual((code, body, hdr.get_content_type()), (200, data, ctype))
self.assertEqual(download(self.b["token"], att["id"], "")[0], 200)
def test_too_large_is_413_and_nothing_is_stored(self):
before = psql("SELECT count(*) FROM attachments")
r = upload(self.a["token"], self.chat["id"], PNG[:8] + b"\x00" * (10 * 1024 * 1024 + 1))
self.assertError(r, 413, "payload_too_large")
self.assertEqual(psql("SELECT count(*) FROM attachments"), before)
def test_corrupted_and_spoofed_files_are_415_not_500(self):
cases = {
"truncated png": (PNG[: len(PNG) // 2], "image/png"),
"garbage png": (b"\x89PNG\r\n\x1a\n" + b"\x00" * 200, "image/png"),
"random bytes": (os.urandom(500), "image/png"),
"truncated jpeg": (JPG[: len(JPG) // 2], "image/jpeg"),
"garbage jpeg": (b"\xff\xd8\xff\xe0" + b"\x00" * 200, "image/jpeg"),
"png as jpeg": (PNG, "image/jpeg"),
"jpeg as png": (JPG, "image/png"),
"text as png": (b"hello, i am not an image", "image/png"),
"gif": (b"GIF89a" + b"\x00" * 50, "image/png"),
}
for name, (data, ctype) in cases.items():
with self.subTest(name):
self.assertError(upload(self.a["token"], self.chat["id"], data, ctype), 415, "unsupported_media_type")
self.assertError(upload(self.a["token"], self.chat["id"], PNG, "text/plain"), 415)
self.assertError(upload(self.a["token"], self.chat["id"], PNG, "multipart/form-data; boundary=x"), 415)
def test_decompression_bomb_header_is_413(self):
self.assertError(upload(self.a["token"], self.chat["id"], bomb_png()), 413, "payload_too_large")
# 6000x6600 = 39.6 млн пикселей < лимита: по размерам допустимо, но данных изображения нет -> файл повреждён (415)
self.assertEqual(upload(self.a["token"], self.chat["id"], bomb_png(6000, 6600))[0], 415)
def test_dangerous_filenames_are_400(self):
bad = ["a%2Fb.png", "..%5Cx.png", "x%00.png", "x%0A.png", "x%7F.png", "", "%20%20", "n" * 256]
for name in bad:
with self.subTest(name):
self.assertError(upload(self.a["token"], self.chat["id"], PNG, raw_filename="filename=" + name), 400, "invalid_request")
self.assertError(upload(self.a["token"], self.chat["id"], PNG, raw_filename=""), 400)
self.assertEqual(upload(self.a["token"], self.chat["id"], PNG, raw_filename="filename=" + "n" * 255)[0], 202)
def test_empty_body_is_400(self):
self.assertError(upload(self.a["token"], self.chat["id"], b""), 400)
def test_foreign_user_cannot_upload_read_or_download(self):
self.assertError(upload(self.eve["token"], self.chat["id"], PNG), 404)
att = upload(self.a["token"], self.chat["id"], PNG)[2]
wait_ready(self.a["token"], att["id"])
for suffix in ("", "/content"):
self.assertEqual(download(self.eve["token"], att["id"], suffix)[0], 404)
self.assertEqual(download(None, att["id"], suffix)[0], 401)
self.assertEqual(download(self.a["token"], str(uuid.uuid4()))[0], 404)
self.assertEqual(download(self.a["token"], "not-a-uuid")[0], 400)
self.assertEqual(download(self.a["token"], att["id"], "/content?variant=thumbnail")[0], 200) # после ready вариант доступен
self.assertEqual(download(self.a["token"], att["id"], "/content?variant=bogus")[0], 400)
def test_message_with_attachments_and_chat_binding(self):
att = upload(self.a["token"], self.chat["id"], PNG)[2]
foreign_chat_att = upload(self.a["token"], self.other["id"], JPG, "image/jpeg")[2]
path = "/api/v1/chats/%s/messages" % self.chat["id"]
sent = call("POST", path, {"text": "", "attachment_ids": [att["id"]]}, self.a["token"])
self.assertEqual(sent[0], 201, sent[2])
self.assertEqual(sent[2]["attachment_ids"], [att["id"]])
self.assertEqual(call("POST", path, {"text": "ещё раз то же вложение", "attachment_ids": [att["id"]]}, self.b["token"])[0], 201)
listed = call("GET", path + "?limit=10", token=self.b["token"])[2]["items"]
self.assertEqual([m["attachment_ids"] for m in listed], [[att["id"]], [att["id"]]])
# вложение другого чата / несуществующее -> 404; дубликаты, >10, пустое сообщение без вложений -> 400
self.assertEqual(call("POST", path, {"text": "x", "attachment_ids": [foreign_chat_att["id"]]}, self.a["token"])[0], 404)
self.assertEqual(call("POST", path, {"text": "x", "attachment_ids": [str(uuid.uuid4())]}, self.a["token"])[0], 404)
self.assertEqual(call("POST", path, {"text": "x", "attachment_ids": [att["id"], att["id"]]}, self.a["token"])[0], 400)
self.assertEqual(call("POST", path, {"text": "x", "attachment_ids": [str(uuid.uuid4()) for _ in range(11)]}, self.a["token"])[0], 400)
self.assertEqual(call("POST", path, {"text": " ", "attachment_ids": []}, self.a["token"])[0], 400)
self.assertEqual(call("POST", path, {"text": "x", "attachment_ids": "nope"}, self.a["token"])[0], 400)
# чужой пользователь не может сослаться на вложение
self.assertEqual(call("POST", path, {"text": "x", "attachment_ids": [att["id"]]}, self.eve["token"])[0], 404)
def test_idempotent_replay_with_attachments(self):
att = upload(self.a["token"], self.chat["id"], PNG)[2]
path = "/api/v1/chats/%s/messages" % self.chat["id"]
key = uuid.uuid4().hex
h = {"Idempotency-Key": key}
first = call("POST", path, {"text": "t", "attachment_ids": [att["id"]]}, self.a["token"], headers=h)
again = call("POST", path, {"attachment_ids": [att["id"]], "text": "t"}, self.a["token"], headers=h) # другой порядок ключей
self.assertEqual((first[0], again[0], first[2]), (201, 201, again[2]))
self.assertEqual(call("POST", path, {"text": "t"}, self.a["token"], headers=h)[0], 409) # другое тело
text_key = uuid.uuid4().hex
m1 = call("POST", path, {"text": "без вложений"}, self.a["token"], headers={"Idempotency-Key": text_key})
m2 = call("POST", path, {"text": "без вложений", "attachment_ids": []}, self.a["token"], headers={"Idempotency-Key": text_key})
self.assertEqual(m1[2], m2[2]) # отсутствие == []
self.assertEqual(m1[2]["attachment_ids"], [])
def test_aborted_upload_leaves_nothing(self):
before = (psql("SELECT count(*) FROM attachments"))
host, port = BASE.split("//")[1].split(":")
with socket.create_connection((host, int(port))) as s:
s.sendall(("POST /api/v1/chats/%s/attachments?filename=a.png HTTP/1.1\r\nHost: x\r\nAuthorization: Bearer %s\r\n"
"Content-Type: image/png\r\nContent-Length: 200000\r\n\r\n" % (self.chat["id"], self.a["token"])).encode() + PNG[:5000])
import time
time.sleep(1)
self.assertEqual(psql("SELECT count(*) FROM attachments"), before)
def test_bucket_is_private_outside_the_api(self):
att = upload(self.a["token"], self.chat["id"], PNG)[2]
wait_ready(self.a["token"], att["id"])
key = psql("SELECT s3_key FROM attachments WHERE id = '%s'" % att["id"])
import subprocess
code = ("import urllib.request,urllib.error,sys\n"
"try:\n urllib.request.urlopen('http://seaweedfs:8333/messenger-attachments/%s', timeout=5); print('OPEN')\n"
"except urllib.error.HTTPError as e: print(e.code)\n" % key)
out = subprocess.run(["docker", "compose", "--env-file", ".env", "exec", "-T", "api", "python", "-c", code], cwd=ROOT,
capture_output=True, text=True).stdout.strip()
self.assertEqual(out, "403", "анонимный GET объекта напрямую из S3 должен быть запрещён")
if __name__ == "__main__":
unittest.main()
+82
View File
@@ -0,0 +1,82 @@
"""Поведение при недоступности зависимостей: Valkey (кэш) и PostgreSQL (источник истины).
python student-tests/test_dependencies.py
Контейнеры останавливаются и запускаются обратно (volumes сохраняются). Ожидания:
* Valkey остановлен -> кэш обходится, API работает (чтение сессии из PostgreSQL), logout работает;
* PostgreSQL остановлен -> 503 в контрактном формате с Retry-After (не 500, не «анонимный» доступ),
/health/ready = 503, запрос без токена по-прежнему 401; после запуска БД API восстанавливается сам.
"""
import os
import subprocess
import sys
import time
import unittest
sys.path.insert(0, os.path.dirname(__file__))
from test_api import ROOT, basic, call, register # noqa: E402
def compose(*args):
return subprocess.run(["docker", "compose", "--env-file", ".env", *args], cwd=ROOT, check=True, capture_output=True, text=True)
def wait(predicate, seconds=60):
deadline = time.time() + seconds
while time.time() < deadline:
try:
if predicate():
return
except Exception:
pass
time.sleep(0.5)
raise RuntimeError("условие не выполнено за %d с" % seconds)
class DependencyTest(unittest.TestCase):
def tearDown(self):
compose("start", "postgres", "valkey")
wait(lambda: call("GET", "/health/ready")[0] == 200)
def test_valkey_down_is_bypassed(self):
actor = register("dep")
self.assertEqual(call("GET", "/api/v1/users/me", token=actor["token"])[0], 200)
compose("stop", "valkey")
started = time.time()
for _ in range(5):
self.assertEqual(call("GET", "/api/v1/users/me", token=actor["token"])[0], 200)
self.assertLess(time.time() - started, 3, "circuit breaker не должен давать каждому запросу ждать таймаут")
# вход и выход работают без кэша
s = call("POST", "/api/v1/auth/sessions", headers=basic(actor["user"]["username"], actor["password"]))
self.assertEqual(s[0], 201)
self.assertEqual(call("DELETE", "/api/v1/auth/sessions/current", token=s[2]["token"])[0], 204)
self.assertEqual(call("GET", "/api/v1/users/me", token=s[2]["token"])[0], 401) # отзыв виден без кэша
self.assertEqual(call("GET", "/health/ready")[0], 200) # кэш не критичен для readiness
def test_postgres_down_gives_bounded_503_never_anonymous_access(self):
actor = register("dep")
self.assertEqual(call("GET", "/api/v1/users/me", token=actor["token"])[0], 200) # токен в кэше
compose("stop", "postgres")
# Wait past HAProxy's fall=2 health checks: cover the all-unready state,
# not just the short interval while the load balancer still routes normally.
time.sleep(5)
for method, path, body, token in (("GET", "/api/v1/users/me", None, actor["token"]),
("GET", "/api/v1/chats", None, actor["token"]),
("POST", "/api/v1/chats", {"title": "x", "member_ids": [actor["user"]["id"]]}, actor["token"]),
("GET", "/api/v1/users/me", None, "Z" * 43)): # неизвестный токен: кэш пуст -> БД -> 503
r = call(method, path, body, token)
self.assertEqual(r[0], 503, (path, r))
self.assertEqual(r[2]["error"]["code"], "unavailable")
self.assertGreaterEqual(int(r[1]["Retry-After"]), 1)
self.assertEqual(r[2]["error"]["request_id"], r[1]["X-Request-Id"])
r = call("POST", "/api/v1/auth/sessions", headers=basic(actor["user"]["username"], actor["password"]))
self.assertEqual(r[0], 503)
self.assertEqual(call("GET", "/health/ready")[0], 503)
self.assertEqual(call("GET", "/health/live")[0], 200)
self.assertEqual(call("GET", "/api/v1/users/me")[0], 401) # без токена — по-прежнему 401
compose("start", "postgres")
wait(lambda: call("GET", "/health/ready")[0] == 200)
self.assertEqual(call("GET", "/api/v1/users/me", token=actor["token"])[0], 200) # сессия пережила простой БД
if __name__ == "__main__":
unittest.main()
+69
View File
@@ -0,0 +1,69 @@
"""Проверка срока жизни сессии при коротком SESSION_TTL_SECONDS (по умолчанию 2).
python student-tests/test_expiry.py
Скрипт пересоздаёт только контейнер api с SESSION_TTL_SECONDS=2, проверяет expiry и возвращает настройки
по умолчанию. Данные и volumes не затрагиваются.
"""
import os
import subprocess
import sys
import time
import unittest
sys.path.insert(0, os.path.dirname(__file__))
from test_api import ROOT, call, register # noqa: E402
TTL = 2
def compose(*args, env=None):
return subprocess.run(["docker", "compose", "--env-file", ".env", *args], cwd=ROOT, check=True,
capture_output=True, text=True, env=dict(os.environ, **(env or {})))
def wait_ready(seconds=60):
deadline = time.time() + seconds
while time.time() < deadline:
try:
if call("GET", "/health/ready")[0] == 200:
return
except Exception:
pass
time.sleep(0.5)
raise RuntimeError("api не стал ready")
class ExpiryTest(unittest.TestCase):
@classmethod
def setUpClass(cls):
compose("up", "-d", "--no-deps", "--force-recreate", "api", env={"SESSION_TTL_SECONDS": str(TTL)})
wait_ready()
@classmethod
def tearDownClass(cls):
compose("up", "-d", "--no-deps", "--force-recreate", "api")
wait_ready()
def test_session_expires_after_ttl(self):
actor = register("exp")
session = actor["session"]
self.assertEqual(call("GET", "/api/v1/users/me", token=actor["token"])[0], 200) # валидна, кэш прогрет
time.sleep(TTL + 1.5)
r = call("GET", "/api/v1/users/me", token=actor["token"])
self.assertEqual(r[0], 401, "после TTL токен должен быть отвергнут (и кэшем, и БД)")
self.assertEqual(r[1]["WWW-Authenticate"], 'Bearer realm="messenger"')
self.assertEqual(r[2]["error"]["code"], "unauthorized")
self.assertEqual(call("DELETE", "/api/v1/auth/sessions/current", token=actor["token"])[0], 401)
self.assertIn("expires_at", session)
def test_expiry_is_not_extended_by_use(self):
actor = register("exp")
for _ in range(3):
call("GET", "/api/v1/users/me", token=actor["token"])
time.sleep(0.5)
time.sleep(1.5)
self.assertEqual(call("GET", "/api/v1/users/me", token=actor["token"])[0], 401)
if __name__ == "__main__":
unittest.main()
+121
View File
@@ -0,0 +1,121 @@
"""Идемпотентность отправки сообщений (lab3) и распределение по репликам. Запуск:
python student-tests/test_idempotency.py (BASE_URL по умолчанию http://127.0.0.1:8080, через LB)
"""
import os
import sys
import unittest
import uuid
from concurrent.futures import ThreadPoolExecutor
from urllib.parse import quote
sys.path.insert(0, os.path.dirname(__file__))
from test_api import call, psql, register # noqa: E402
def chat_with(owner, *others):
ids = [owner["user"]["id"]] + [o["user"]["id"] for o in others]
return call("POST", "/api/v1/chats", {"title": "идемпотентность", "member_ids": ids}, owner["token"])[2]
def history(token, chat_id):
items, cursor = [], None
while True:
page = call("GET", "/api/v1/chats/%s/messages?limit=100%s" % (chat_id, "&cursor=" + quote(cursor, safe="") if cursor else ""), token=token)[2]
items += page["items"]
cursor = page["next_cursor"]
if not cursor:
return items
class IdempotencyTest(unittest.TestCase):
def setUp(self):
self.a, self.b, self.eve = register(), register(), register()
self.chat = chat_with(self.a, self.b)
self.path = "/api/v1/chats/%s/messages" % self.chat["id"]
def post(self, actor, body, key, path=None):
return call("POST", path or self.path, body, actor["token"], headers={"Idempotency-Key": key})
def test_key_validation(self):
for headers in ({}, {"Idempotency-Key": ""}, {"Idempotency-Key": "a b"},
{"Idempotency-Key": "x" * 129}, {"Idempotency-Key": "a/b"}):
h = dict(headers)
h.setdefault("Content-Type", "application/json")
r = call("POST", self.path, raw=b'{"text":"x"}', token=self.a["token"], headers=h)
self.assertEqual(r[0], 400, headers)
self.assertEqual(r[2]["error"]["code"], "invalid_request")
self.assertEqual(self.post(self.a, {"text": "ok"}, "A-z_0.9:-" + "k" * 100)[0], 201)
def test_replay_returns_same_message_and_creates_no_duplicate(self):
key = uuid.uuid4().hex
first = self.post(self.a, {"text": "один раз"}, key)
second = self.post(self.a, {"text": "один раз"}, key)
self.assertEqual((first[0], second[0]), (201, 201))
self.assertEqual(first[2], second[2])
self.assertEqual(len([m for m in history(self.b["token"], self.chat["id"]) if m["text"] == "один раз"]), 1)
def test_different_body_same_key_is_409(self):
key = uuid.uuid4().hex
self.post(self.a, {"text": "первое"}, key)
r = self.post(self.a, {"text": "другое"}, key)
self.assertEqual(r[0], 409)
self.assertEqual(r[2]["error"]["code"], "conflict")
def test_scope_is_user_and_chat(self):
key = uuid.uuid4().hex
m1 = self.post(self.a, {"text": "общий ключ"}, key)[2]
m2 = self.post(self.b, {"text": "общий ключ"}, key)[2] # другой пользователь
other = chat_with(self.a)
m3 = self.post(self.a, {"text": "общий ключ"}, key, "/api/v1/chats/%s/messages" % other["id"])[2] # другой чат
self.assertEqual(len({m1["id"], m2["id"], m3["id"]}), 3)
def test_replay_rechecks_session_and_membership(self):
key = uuid.uuid4().hex
self.post(self.a, {"text": "защищено"}, key)
self.assertEqual(self.post(self.eve, {"text": "защищено"}, key)[0], 404) # посторонний
r = call("POST", self.path, {"text": "защищено"}, "Z" * 43, headers={"Idempotency-Key": key})
self.assertEqual(r[0], 401) # недействительный токен
logout = register()
chat = chat_with(logout)
path = "/api/v1/chats/%s/messages" % chat["id"]
self.post(logout, {"text": "x"}, key, path)
call("DELETE", "/api/v1/auth/sessions/current", token=logout["token"])
self.assertEqual(call("POST", path, {"text": "x"}, logout["token"], headers={"Idempotency-Key": key})[0], 401)
def test_semantically_equal_body_is_replayed(self):
key = uuid.uuid4().hex
first = self.post(self.a, {"text": "эквивалент"}, key)
raw = call("POST", self.path, raw=b'{ "text" : "\u044d\u043a\u0432\u0438\u0432\u0430\u043b\u0435\u043d\u0442" }',
token=self.a["token"], headers={"Idempotency-Key": key, "Content-Type": "application/json"})
self.assertEqual((raw[0], raw[2]), (201, first[2]))
def test_concurrent_same_key_creates_exactly_one_message_across_replicas(self):
key = uuid.uuid4().hex
with ThreadPoolExecutor(24) as pool:
results = list(pool.map(lambda _: self.post(self.a, {"text": "гонка ключа"}, key), range(24)))
self.assertEqual({r[0] for r in results}, {201})
self.assertEqual(len({r[2]["id"] for r in results}), 1)
instances = {r[1]["X-Instance-Id"] for r in results}
self.assertGreaterEqual(len(instances), 2, "повторы должны обслуживаться разными репликами: %s" % instances)
found = [m for m in history(self.b["token"], self.chat["id"]) if m["text"] == "гонка ключа"]
self.assertEqual(len(found), 1)
def test_expired_key_is_reusable_after_ttl(self):
key = uuid.uuid4().hex
first = self.post(self.a, {"text": "ttl"}, key)[2]
psql("UPDATE idempotency_keys SET created_at = now() - interval '25 hours' WHERE idem_key = '%s'" % key)
again = self.post(self.a, {"text": "ttl"}, key)
self.assertEqual(again[0], 201)
self.assertNotEqual(again[2]["id"], first["id"], "после 24 ч ключ считается новым")
def test_session_is_shared_between_replicas_without_relogin(self):
seen = set()
for _ in range(30):
r = call("GET", "/api/v1/users/me", token=self.a["token"])
self.assertEqual(r[0], 200)
seen.add(r[1]["X-Instance-Id"])
self.assertGreaterEqual(len(seen), 2, seen)
if __name__ == "__main__":
unittest.main()
+142
View File
@@ -0,0 +1,142 @@
"""Проверка наблюдаемости lab6: метрики, логи, трассы, связь API -> брокер -> worker, отсутствие секретов в телеметрии.
python -m unittest discover -s student-tests -p test_observability.py -v (стенд запущен; нужен ~1 минута)
Запросы к Prometheus/Loki/Tempo выполняются изнутри сети стенда (их порты наружу не опубликованы).
"""
import json
import os
import subprocess
import sys
import time
import unittest
import urllib.parse
import uuid
sys.path.insert(0, os.path.dirname(__file__))
from test_api import BASE, ROOT, call, register # noqa: E402
from test_attachments import PNG, upload # noqa: E402
from test_pipeline import wait_status # noqa: E402
def net_get(url, accept=None):
cmd = ["docker", "compose", "--env-file", ".env", "exec", "-T", "prometheus", "wget", "-q", "-O", "-"]
if accept:
cmd += ["--header", "Accept: " + accept]
out = subprocess.run(cmd + [url], cwd=ROOT, capture_output=True, text=True, encoding="utf-8")
return out.stdout
def prom(query):
data = json.loads(net_get("http://127.0.0.1:9090/api/v1/query?query=" + urllib.parse.quote(query)))
return data["data"]["result"]
def loki(query, limit=200):
now = int(time.time() * 1e9)
url = "http://loki:3100/loki/api/v1/query_range?limit=%d&start=%d&end=%d&query=%s" % (limit, now - 600 * 10 ** 9, now, urllib.parse.quote(query))
data = json.loads(net_get(url))
return [(s["stream"], v) for s in data["data"]["result"] for v in s["values"]]
def tempo_trace(trace_id):
raw = net_get("http://tempo:3200/api/traces/" + trace_id, "application/json")
return json.loads(raw) if raw.strip() else None
def spans_of(trace):
out = []
for batch in trace.get("batches", trace.get("resourceSpans", [])):
service = next((a["value"].get("stringValue") for a in batch["resource"]["attributes"] if a["key"] == "service.name"), None)
for scope in batch.get("scopeSpans", batch.get("instrumentationLibrarySpans", [])):
for sp in scope["spans"]:
attrs = {a["key"]: next(iter(a["value"].values())) for a in sp.get("attributes", [])}
out.append({"service": service, "name": sp["name"], "id": sp["spanId"], "parent": sp.get("parentSpanId", ""), "kind": sp.get("kind"), "attrs": attrs})
return out
class ObservabilityTest(unittest.TestCase):
@classmethod
def setUpClass(cls):
cls.a = register("obs")
cls.chat = call("POST", "/api/v1/chats", {"title": "наблюдаемость", "member_ids": [cls.a["user"]["id"]]}, cls.a["token"])[2]
cls.trace_id = uuid.uuid4().hex
cls.secret_text = "СЕКРЕТНЫЙ-ТЕКСТ-" + uuid.uuid4().hex
# загрузка с входящим traceparent: так мы знаем trace_id заранее
import urllib.request
req = urllib.request.Request("%s/api/v1/chats/%s/attachments?filename=obs.png" % (BASE, cls.chat["id"]), data=PNG, method="POST",
headers={"Authorization": "Bearer " + cls.a["token"], "Content-Type": "image/png",
"traceparent": "00-%s-%s-01" % (cls.trace_id, uuid.uuid4().hex[:16])})
with urllib.request.urlopen(req, timeout=30) as r:
cls.att = json.loads(r.read())
wait_status(cls.a["token"], cls.att["id"], ("ready",), 60)
call("POST", "/api/v1/chats/%s/messages" % cls.chat["id"], {"text": cls.secret_text, "attachment_ids": [cls.att["id"]]}, cls.a["token"])
deadline = time.time() + 40 # экспорт SDK (2 с) + пакетная обработка Collector + индексация
cls.trace = None
while time.time() < deadline:
cls.trace = tempo_trace(cls.trace_id)
if cls.trace and {s["service"] for s in spans_of(cls.trace)} >= {"messenger-api", "messenger-worker"}:
break
time.sleep(2)
def test_one_trace_spans_api_outbox_broker_and_worker(self):
self.assertIsNotNone(self.trace, "трасса не найдена в Tempo")
spans = spans_of(self.trace)
by_name = {s["name"]: s for s in spans}
self.assertEqual({s["service"] for s in spans} & {"messenger-api", "messenger-worker"}, {"messenger-api", "messenger-worker"})
server = by_name["HTTP POST /api/v1/chats/{chat_id}/attachments"]
producer, consumer = by_name["attachment.process publish"], by_name["attachment.process process"]
self.assertEqual((server["service"], producer["service"], consumer["service"]), ("messenger-api", "messenger-api", "messenger-worker"))
# цепочка parent-child: server (API) -> producer (publish из outbox) -> consumer (worker)
self.assertEqual(producer["parent"], server["id"])
self.assertEqual(consumer["parent"], producer["id"])
names = {s["name"] for s in spans}
self.assertIn("s3.put_object", names) # запись staging (API) и результатов (worker)
self.assertIn("s3.get_object", names) # чтение staging worker-ом
self.assertIn("db.connection", names)
self.assertEqual(server["attrs"].get("http.route"), "/api/v1/chats/{chat_id}/attachments")
self.assertEqual(int(server["attrs"].get("http.response.status_code")), 202)
def test_logs_are_structured_and_linked_by_trace_id(self):
rows = loki('{service_name=~"messenger-.*"} | trace_id="%s"' % self.trace_id)
services = {stream["service_name"] for stream, _ in rows}
messages = {v[1] for _, v in rows}
self.assertIn("messenger-api", services)
self.assertIn("messenger-worker", services)
self.assertIn("job_ready", messages)
self.assertTrue(all(stream.get("span_id") for stream, _ in rows))
self.assertTrue(all(stream.get("service_version") is not None and stream.get("service_instance_id") for stream, _ in rows))
def test_metrics_for_http_jobs_queue_pool_worker(self):
for query in ('http_server_request_duration_seconds_count{http_route="/api/v1/chats/{chat_id}/attachments"}',
'messenger_jobs_processed_total{outcome="ready"}',
'messenger_job_duration_seconds_bucket', 'messenger_job_time_to_ready_seconds_bucket',
'messenger_queue_depth', 'messenger_queue_oldest_job_age_seconds', 'messenger_worker_capacity', 'messenger_worker_busy',
'db_client_connections_usage{state="used"}', 'db_client_connections_waiting', 'process_memory_rss_bytes',
'process_cpu_time_seconds_total'):
with self.subTest(query):
deadline = time.time() + 40 # экспорт SDK (5 с) + scrape Prometheus (5 с)
while not prom(query) and time.time() < deadline:
time.sleep(2)
self.assertTrue(prom(query), "нет данных: " + query)
def test_no_high_cardinality_or_secret_labels(self):
forbidden = ("user_id", "chat_id", "message_id", "attachment_id", "request_id", "filename")
series = json.loads(net_get("http://127.0.0.1:9090/api/v1/series?match[]=%s" % urllib.parse.quote('{__name__=~"http_server.*|messenger_.*|db_client.*|process_.*"}')))["data"]
labels = {k for s in series for k in s}
self.assertFalse(labels & set(forbidden), labels & set(forbidden))
text = json.dumps(series)
self.assertNotIn(self.a["token"], text)
route_values = {s.get("http_route") for s in series if "http_route" in s}
import re
self.assertFalse([r for r in route_values if re.search(r"[0-9a-f]{8}-[0-9a-f]{4}-", r)], route_values) # шаблоны, не реальные UUID
def test_secrets_and_message_bodies_are_not_exported(self):
self.assertEqual(loki('{service_name=~"messenger-.*"} |= "%s"' % self.secret_text), [])
self.assertEqual(loki('{service_name=~"messenger-.*"} |= "%s"' % self.a["token"]), [])
self.assertEqual(loki('{service_name=~"messenger-.*"} |= "%s"' % self.a["password"]), [])
raw = json.dumps(self.trace)
for secret in (self.secret_text, self.a["token"], self.a["password"]):
self.assertNotIn(secret, raw)
if __name__ == "__main__":
unittest.main()
+58
View File
@@ -0,0 +1,58 @@
"""Цепочка трассировки с повторами (уровень 4): все попытки одного задания находятся в ОДНОЙ трассе.
python student-tests/test_observability_retry.py
Worker пересоздаётся с внедрением сбоя (fail:after_claim, 3 попытки), затем возвращается к штатной конфигурации.
"""
import json
import os
import sys
import time
import unittest
import urllib.request
import uuid
sys.path.insert(0, os.path.dirname(__file__))
from test_api import BASE, call, register # noqa: E402
from test_attachments import PNG # noqa: E402
from test_observability import loki, spans_of, tempo_trace # noqa: E402
from test_pipeline import wait_status # noqa: E402
from test_resilience5 import compose, healthy, recreate_worker, wait # noqa: E402
class RetryTraceTest(unittest.TestCase):
@classmethod
def tearDownClass(cls):
recreate_worker()
def test_all_attempts_in_one_trace_and_logs(self):
recreate_worker({"WORKER_FAULT_INJECTION": "fail:after_claim", "MAX_ATTEMPTS": "3", "RETRY_BASE_DELAY": "0.3"})
a = register("obsr")
chat = call("POST", "/api/v1/chats", {"title": "retry-trace", "member_ids": [a["user"]["id"]]}, a["token"])[2]
trace_id = uuid.uuid4().hex
req = urllib.request.Request("%s/api/v1/chats/%s/attachments?filename=r.png" % (BASE, chat["id"]), data=PNG, method="POST",
headers={"Authorization": "Bearer " + a["token"], "Content-Type": "image/png",
"traceparent": "00-%s-%s-01" % (trace_id, uuid.uuid4().hex[:16])})
with urllib.request.urlopen(req, timeout=30) as r:
att = json.loads(r.read())
failed = wait_status(a["token"], att["id"], ("failed",), 60)
self.assertEqual(failed["error_code"], "retry_exhausted")
spans, deadline = [], time.time() + 40
while time.time() < deadline:
trace = tempo_trace(trace_id)
spans = spans_of(trace) if trace else []
consumers = [s for s in spans if s["name"] == "attachment.process process"]
if len(consumers) >= 3:
break
time.sleep(2)
consumers = sorted([s for s in spans if s["name"] == "attachment.process process"], key=lambda s: int(s["attrs"]["job.attempt"]))
publishers = {s["id"]: s for s in spans if s["name"] == "attachment.process publish"}
self.assertEqual([int(c["attrs"]["job.attempt"]) for c in consumers], [1, 2, 3])
for c in consumers:
self.assertIn(c["parent"], publishers, "consumer-спан каждой попытки — потомок publish-спана той же трассы")
logs = loki('{service_name="messenger-worker"} | trace_id="%s"' % trace_id)
messages = [v[1] for _, v in logs]
self.assertEqual(messages.count("job_rescheduled"), 2)
self.assertIn("job_retry_exhausted", messages)
if __name__ == "__main__":
unittest.main()
+182
View File
@@ -0,0 +1,182 @@
"""Тесты асинхронного конвейера вложений lab6 (контракт вариантов, состояния, права). Запуск:
python -m unittest discover -s student-tests -p "test_pipeline.py" -v
Пиксельные проверки выполняются, если установлен Pillow (иначе пропускаются); размеры проверяются по заголовкам файлов.
"""
import hashlib
import io
import os
import struct
import sys
import time
import unittest
import uuid
sys.path.insert(0, os.path.dirname(__file__))
from test_api import call, register # noqa: E402
from test_attachments import FIX, JPG, PNG, download, upload # noqa: E402
try:
from PIL import Image
except ImportError: # pragma: no cover
Image = None
def fixture(name):
return open(os.path.join(FIX, name), "rb").read()
def jpeg_size(data):
i = 2
while i < len(data):
while data[i] == 0xFF:
i += 1
marker = data[i]
i += 1
if marker in (0xD8, 0x01) or 0xD0 <= marker <= 0xD7:
continue
length = int.from_bytes(data[i:i + 2], "big")
if marker in (0xC0, 0xC1, 0xC2):
return int.from_bytes(data[i + 5:i + 7], "big"), int.from_bytes(data[i + 3:i + 5], "big")
i += length
raise AssertionError("JPEG dimensions not found")
def webp_size(data):
assert data[:4] == b"RIFF" and data[8:12] == b"WEBP", "not WebP"
kind = data[12:16]
if kind == b"VP8X":
return 1 + int.from_bytes(data[24:27], "little"), 1 + int.from_bytes(data[27:30], "little")
if kind == b"VP8L":
bits = int.from_bytes(data[21:25], "little")
return (bits & 0x3FFF) + 1, ((bits >> 14) & 0x3FFF) + 1
if kind == b"VP8 ":
return struct.unpack("<HH", data[26:30])[0] & 0x3FFF, struct.unpack("<HH", data[26:30])[1] & 0x3FFF
raise AssertionError("unknown WebP chunk")
def wait_status(token, att_id, wanted=("ready",), seconds=60):
deadline = time.time() + seconds
while time.time() < deadline:
code, _, body = download(token, att_id, "")
att = __import__("json").loads(body)
if att["status"] in wanted:
return att
time.sleep(0.3)
raise AssertionError("вложение не достигло %s за %d с: %s" % (wanted, seconds, att["status"]))
class PipelineTest(unittest.TestCase):
def setUp(self):
self.a, self.b, self.eve = register("pipe"), register("pipe"), register("pipe")
self.chat = call("POST", "/api/v1/chats", {"title": "конвейер", "member_ids": [self.a["user"]["id"], self.b["user"]["id"]]}, self.a["token"])[2]
def ready(self, data, ctype="image/png", name="x.png"):
status, _, att = upload(self.a["token"], self.chat["id"], data, ctype, name)
self.assertEqual(status, 202, att)
self.assertEqual((att["status"], att["variants"], att["error_code"]), ("queued", [], None))
return wait_status(self.b["token"], att["id"])
def variants(self, att):
return {v["name"]: v for v in att["variants"]}
def fetch(self, att, variant):
code, hdr, body = download(self.b["token"], att["id"], "/content?variant=" + variant)
self.assertEqual(code, 200)
return hdr.get_content_type(), body
def test_sample_png_produces_all_variants_per_contract(self):
att = self.ready(PNG)
v = self.variants(att)
self.assertEqual(set(v), {"thumbnail", "crop", "optimized"})
self.assertEqual((att["sha256"], att["size_bytes"]), (hashlib.sha256(PNG).hexdigest(), len(PNG))) # описывают оригинал
self.assertEqual((v["thumbnail"]["width"], v["thumbnail"]["height"], v["thumbnail"]["content_type"]), (256, 160, "image/jpeg"))
self.assertEqual((v["crop"]["width"], v["crop"]["height"], v["crop"]["content_type"]), (128, 128, "image/jpeg"))
self.assertEqual(v["optimized"]["content_type"], "image/webp")
self.assertEqual((v["optimized"]["width"], v["optimized"]["height"]), (320, 200)) # без увеличения
ctype, thumb = self.fetch(att, "thumbnail")
self.assertEqual((ctype, jpeg_size(thumb)), ("image/jpeg", (256, 160)))
self.assertEqual((v["thumbnail"]["size_bytes"], v["thumbnail"]["sha256"]), (len(thumb), hashlib.sha256(thumb).hexdigest()))
_, crop = self.fetch(att, "crop")
self.assertEqual(jpeg_size(crop), (128, 128))
ctype, opt = self.fetch(att, "optimized")
self.assertEqual((ctype, webp_size(opt)), ("image/webp", (320, 200)))
code, hdr, original = download(self.b["token"], att["id"])
self.assertEqual((code, original, hdr.get_content_type()), (200, PNG, "image/png")) # оригинал неизменен
def test_exif_orientation_is_applied(self):
att = self.ready(fixture("exif_rot6.jpg"), "image/jpeg", "rot.jpg")
v = self.variants(att)
# 300x100 с Orientation=6 -> 100x300 -> вписывается в 256: 85x256 (округление вниз)
self.assertEqual((v["thumbnail"]["width"], v["thumbnail"]["height"]), (85, 256))
self.assertEqual(jpeg_size(self.fetch(att, "thumbnail")[1]), (85, 256))
def test_small_image_is_not_upscaled_but_crop_is_128(self):
att = self.ready(fixture("small.png"), "image/png", "small.png")
v = self.variants(att)
self.assertEqual((v["thumbnail"]["width"], v["thumbnail"]["height"]), (50, 30))
self.assertEqual((v["crop"]["width"], v["crop"]["height"]), (128, 128))
self.assertEqual((v["optimized"]["width"], v["optimized"]["height"]), (50, 30))
@unittest.skipIf(Image is None, "нужен Pillow")
def test_alpha_is_composited_on_white(self):
att = self.ready(fixture("alpha.png"), "image/png", "alpha.png")
thumb = Image.open(io.BytesIO(self.fetch(att, "thumbnail")[1])).convert("RGB")
left, right = thumb.getpixel((10, thumb.height // 2)), thumb.getpixel((thumb.width - 10, thumb.height // 2))
self.assertTrue(all(c > 235 for c in left), "прозрачная область должна стать белой: %s" % (left,))
self.assertTrue(right[0] > 200 and right[1] < 60, "непрозрачная красная область сохранена: %s" % (right,))
def test_message_can_reference_attachment_before_ready_and_failed_cannot(self):
status, _, att = upload(self.a["token"], self.chat["id"], PNG)
path = "/api/v1/chats/%s/messages" % self.chat["id"]
sent = call("POST", path, {"text": "", "attachment_ids": [att["id"]]}, self.a["token"])
self.assertEqual((status, sent[0], sent[2]["attachment_ids"]), (202, 201, [att["id"]]))
poison = upload(self.a["token"], self.chat["id"], fixture("poison.png"))
self.assertEqual(poison[0], 202, poison[2])
failed = wait_status(self.b["token"], poison[2]["id"], ("failed",))
self.assertEqual((failed["status"], failed["error_code"], failed["variants"]), ("failed", "invalid_image", []))
self.assertEqual(call("POST", path, {"text": "x", "attachment_ids": [failed["id"]]}, self.a["token"])[0], 409)
self.assertEqual(download(self.b["token"], failed["id"], "/content")[0], 409)
self.assertEqual(download(self.b["token"], failed["id"], "/content?variant=thumbnail")[0], 409)
def test_content_is_409_while_not_ready_and_original_is_untouched(self):
status, _, att = upload(self.a["token"], self.chat["id"], PNG)
code = download(self.b["token"], att["id"], "/content")[0]
self.assertIn(code, (200, 409)) # быстрый worker мог успеть: 200 допустим, но не 404/500
done = wait_status(self.b["token"], att["id"])
self.assertEqual(download(self.b["token"], done["id"], "/content")[2], PNG)
def test_other_variants_visible_only_to_chat_members(self):
att = self.ready(PNG)
for variant in ("thumbnail", "crop", "optimized", "original"):
self.assertEqual(download(self.eve["token"], att["id"], "/content?variant=" + variant)[0], 404)
self.assertEqual(download(self.b["token"], att["id"], "/content?variant=bogus")[0], 400)
def test_ready_job_redelivery_does_not_duplicate_variants(self):
from test_api import psql
att = self.ready(PNG)
before = psql("SELECT count(*) FROM attachment_variants WHERE attachment_id = '%s'" % att["id"])
self.assertEqual(before, "3")
psql("INSERT INTO outbox (job_id, attachment_id, pipeline_version, attempt) VALUES (gen_random_uuid(), '%s', 1, 2)" % att["id"])
time.sleep(4)
self.assertEqual(psql("SELECT count(*) FROM attachment_variants WHERE attachment_id = '%s'" % att["id"]), "3")
self.assertEqual(wait_status(self.b["token"], att["id"])["status"], "ready")
def test_staging_is_cleaned_after_ready_and_kept_for_failed(self):
import subprocess
from test_api import ROOT, psql
att = self.ready(PNG)
poison = upload(self.a["token"], self.chat["id"], fixture("poison.png"))[2]
wait_status(self.b["token"], poison["id"], ("failed",))
def exists(att_id):
code = "\n".join(["import asyncio", "from app.s3 import S3", "async def m():",
" s = S3(); await s.start(); print((await s.head('staging/%s')) is not None); await s.stop()" % att_id,
"asyncio.run(m())"])
return subprocess.run(["docker", "compose", "--env-file", ".env", "exec", "-T", "api", "python", "-c", code], cwd=ROOT,
capture_output=True, text=True).stdout.strip()
self.assertEqual(exists(att["id"]), "False", "staging готового вложения должен быть удалён после COMMIT результата")
self.assertEqual(exists(poison["id"]), "True", "staging неудавшегося вложения хранится для разбора/replay (retention)")
if __name__ == "__main__":
unittest.main()

Some files were not shown because too many files have changed in this diff Show More