forked from mtusi2026/mtusi2026lab6
Compare commits
1
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
79ef8807d5 |
+45
-5
@@ -6,25 +6,65 @@ APP_ENV=lab
|
|||||||
SESSION_TTL_SECONDS=3600
|
SESSION_TTL_SECONDS=3600
|
||||||
# Choose credentials locally. File paths contain no secret values.
|
# Choose credentials locally. File paths contain no secret values.
|
||||||
DATABASE_URL_FILE=/run/secrets/database_url
|
DATABASE_URL_FILE=/run/secrets/database_url
|
||||||
# Optional on 3, required use on 4 in lab2; Redis OR Valkey:
|
# Optional on 3, required use on 4 in lab2; Redis OR Valkey (this solution: Redis since lab2):
|
||||||
SESSION_STORE=postgres
|
SESSION_STORE=redis
|
||||||
CACHE_URL_FILE=/run/secrets/cache_url
|
CACHE_URL_FILE=/run/secrets/cache_url
|
||||||
API_REPLICAS=2
|
API_REPLICAS=2
|
||||||
IDEMPOTENCY_TTL_SECONDS=86400
|
IDEMPOTENCY_TTL_SECONDS=86400
|
||||||
S3_ENDPOINT=http://s3:9000
|
# SeaweedFS S3 (since lab4)
|
||||||
|
S3_ENDPOINT=http://s3:8333
|
||||||
S3_REGION=us-east-1
|
S3_REGION=us-east-1
|
||||||
S3_BUCKET=messenger-attachments
|
S3_BUCKET=messenger-attachments
|
||||||
S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id
|
S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id
|
||||||
S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key
|
S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key
|
||||||
MAX_UPLOAD_BYTES=10485760
|
MAX_UPLOAD_BYTES=10485760
|
||||||
MAX_IMAGE_PIXELS=40000000
|
MAX_IMAGE_PIXELS=40000000
|
||||||
BROKER_URL_FILE=/run/secrets/broker_url
|
|
||||||
|
# lab5: broker and worker. The API has NO broker credentials (it writes the transactional outbox);
|
||||||
|
# compose.yaml passes per-service files: dispatcher -> /run/secrets/broker_url_dispatcher,
|
||||||
|
# worker -> /run/secrets/broker_url_worker. This variable is therefore empty for api/janitor.
|
||||||
|
BROKER_URL_FILE=
|
||||||
WORKER_CONCURRENCY=2
|
WORKER_CONCURRENCY=2
|
||||||
|
WORKER_PREFETCH=1
|
||||||
JOB_MAX_ATTEMPTS=5
|
JOB_MAX_ATTEMPTS=5
|
||||||
JOB_TIMEOUT_SECONDS=60
|
JOB_TIMEOUT_SECONDS=60
|
||||||
|
JOB_LEASE_SECONDS=15
|
||||||
|
JOB_BACKOFF_BASE_SECONDS=2
|
||||||
|
JOB_BACKOFF_MAX_SECONDS=60
|
||||||
|
WORKER_MEMORY_LIMIT_MB=512
|
||||||
STAGING_RETENTION_SECONDS=86400
|
STAGING_RETENTION_SECONDS=86400
|
||||||
|
PIPELINE_VERSION=1
|
||||||
|
# Fault injection (worker/dispatcher only, triggered by files in the faults volume): compose.yaml enables it on the
|
||||||
|
# training stand by default; set FAULT_INJECTION=0 here to disable.
|
||||||
|
|
||||||
|
# Pools and timeouts (same as lab3)
|
||||||
|
DB_POOL_MIN=1
|
||||||
|
DB_POOL_MAX=10
|
||||||
|
DB_POOL_WAIT_SECONDS=3
|
||||||
|
DB_CONNECT_TIMEOUT_SECONDS=3
|
||||||
|
DB_STATEMENT_TIMEOUT_MS=5000
|
||||||
|
REDIS_POOL_MAX=20
|
||||||
|
REDIS_TIMEOUT_MS=500
|
||||||
|
SHUTDOWN_DRAIN_SECONDS=3
|
||||||
|
SHUTDOWN_TIMEOUT_SECONDS=10
|
||||||
|
|
||||||
|
# lab4: S3 client and uploads
|
||||||
|
S3_CONNECT_TIMEOUT_SECONDS=1
|
||||||
|
S3_READ_TIMEOUT_SECONDS=5
|
||||||
|
S3_MAX_ATTEMPTS=2
|
||||||
|
S3_POOL_MAX=20
|
||||||
|
UPLOAD_TIMEOUT_SECONDS=60
|
||||||
|
UPLOAD_IDLE_TIMEOUT_SECONDS=10
|
||||||
|
DECODE_CONCURRENCY=2
|
||||||
|
# Cleanup of abandoned uploads and staging (janitor service): grace period and pass interval
|
||||||
|
ATTACHMENT_GRACE_SECONDS=3600
|
||||||
|
JANITOR_INTERVAL_SECONDS=300
|
||||||
|
|
||||||
|
# lab6: OpenTelemetry. Standard SDK variables; compose.yaml sets the same values for every app service
|
||||||
|
# and a distinct OTEL_SERVICE_NAME (messenger-api / messenger-worker / messenger-dispatcher / messenger-janitor).
|
||||||
OTEL_SERVICE_NAME=messenger-api
|
OTEL_SERVICE_NAME=messenger-api
|
||||||
# Worker must set a distinct service.name.
|
|
||||||
OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4318
|
OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4318
|
||||||
OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf
|
OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf
|
||||||
OTEL_RESOURCE_ATTRIBUTES=deployment.environment.name=lab
|
OTEL_RESOURCE_ATTRIBUTES=deployment.environment.name=lab
|
||||||
|
# true -> SDK off in all app processes (logs still go to stderr as JSON)
|
||||||
|
OTEL_SDK_DISABLED=false
|
||||||
|
|||||||
@@ -0,0 +1,11 @@
|
|||||||
|
*.sh text eol=lf
|
||||||
|
*.sql text eol=lf
|
||||||
|
*.conf text eol=lf
|
||||||
|
*.cfg text eol=lf
|
||||||
|
*.example text eol=lf
|
||||||
|
*.cmd text eol=crlf
|
||||||
|
*.yaml text eol=lf
|
||||||
|
*.yml text eol=lf
|
||||||
|
*.json text eol=lf
|
||||||
|
*.py text eol=lf
|
||||||
|
*.md text eol=lf
|
||||||
@@ -1,18 +1,18 @@
|
|||||||
## Лабораторная 6
|
## Лабораторная 6
|
||||||
|
|
||||||
Автор / группа: TODO
|
Автор / группа: Бабиченко Максим, БВТ2303
|
||||||
Целевая оценка (3/4/5): TODO
|
Целевая оценка (3/4/5): 4
|
||||||
Трек 5, если заявлен: TODO
|
Трек 5, если заявлен: не заявлен
|
||||||
|
|
||||||
Что реализовано и что перенесено: TODO
|
Что реализовано и что перенесено: перенесено всё решение lab5 (API, outbox, dispatcher, worker, janitor, тесты lab1–5). Добавлены: OpenTelemetry SDK во всех процессах (traces/metrics/logs по OTLP), W3C trace context API → outbox → RabbitMQ → worker (parent-child, replay — link), OTel Collector (memory_limiter, batch, ограниченные очереди), Prometheus (OTLP receiver, recording/alert rules + unit-тесты), Loki, Tempo, Grafana с provisioning источников и dashboard из Git, SLI/SLO, 6 алертов с runbooks, сценарии инцидентов.
|
||||||
Команды запуска из чистого clone: TODO
|
Команды запуска из чистого clone: REPORT.md → «Запуск из чистого clone» (`python scripts/init_secrets.py`, `docker compose up -d --build --wait`, `python student-tests/collect_evidence.py ...`).
|
||||||
Результаты проверок и ссылка на REPORT.md: TODO
|
Результаты проверок и ссылка на REPORT.md: REPORT.md → «Доказательства критериев», `evidence/summary.md`, `evidence/incident-*.json`; прогон без Docker — `evidence/native-sandbox/`.
|
||||||
Ограничения / известные ошибки: TODO
|
Ограничения / известные ошибки: REPORT.md → «Ограничения и следующие шаги» (одна копия каждого хранилища, буфер Collector в памяти, без TLS до lab7).
|
||||||
Источники и использование ИИ: TODO
|
Источники и использование ИИ: REPORT.md, шапка (документация OTel/Prometheus/Loki/Tempo/Grafana; ИИ-ассистент Claude).
|
||||||
|
|
||||||
- [ ] Весь обязательный минимум текущей и предыдущих работ сохранён с объявленными переходами.
|
- [x] Весь обязательный минимум текущей и предыдущих работ сохранён с объявленными переходами.
|
||||||
- [ ] Контракт и публичные тесты не ослаблены.
|
- [x] Контракт и публичные тесты не ослаблены.
|
||||||
- [ ] make check и make test выполнены; приложен реальный вывод.
|
- [ ] make check и make test выполнены; приложен реальный вывод (Docker-прогон: `evidence/`).
|
||||||
- [ ] Сценарии отказов/безопасности своего уровня воспроизведены.
|
- [ ] Сценарии отказов/безопасности своего уровня воспроизведены (Docker-прогон: `evidence/incident-*.json`).
|
||||||
- [ ] Все секреты и приватные ключи находятся вне Git; отчёт не содержит tokens.
|
- [x] Все секреты и приватные ключи находятся вне Git; отчёт не содержит tokens.
|
||||||
- [ ] README задания сохранён, REPORT.md содержит инструкцию именно моего решения.
|
- [x] README задания сохранён, REPORT.md содержит инструкцию именно моего решения.
|
||||||
|
|||||||
+31
@@ -0,0 +1,31 @@
|
|||||||
|
FROM python:3.11.13-slim
|
||||||
|
|
||||||
|
ENV PYTHONDONTWRITEBYTECODE=1 \
|
||||||
|
PYTHONUNBUFFERED=1
|
||||||
|
|
||||||
|
WORKDIR /app
|
||||||
|
|
||||||
|
COPY requirements.txt .
|
||||||
|
|
||||||
|
RUN pip install --no-cache-dir --only-binary=:all: -r requirements.txt
|
||||||
|
|
||||||
|
COPY main.py settings.py errors.py db.py auth.py migrate.py storage.py uploads.py attachments.py janitor.py s3_init.py \
|
||||||
|
pipeline.py broker.py dispatcher.py worker.py imaging.py faults.py ops.py telemetry.py ./
|
||||||
|
COPY migrations/ ./migrations/
|
||||||
|
|
||||||
|
RUN useradd --create-home --shell /usr/sbin/nologin appuser \
|
||||||
|
&& mkdir -p /faults && chown appuser:appuser /faults
|
||||||
|
# /faults — точка монтирования volume faults (учебное внедрение сбоев worker/dispatcher); владелец appuser
|
||||||
|
# копируется в пустой named volume при первом монтировании.
|
||||||
|
|
||||||
|
USER appuser
|
||||||
|
|
||||||
|
ARG APP_VERSION=dev
|
||||||
|
ENV APP_VERSION=${APP_VERSION}
|
||||||
|
|
||||||
|
EXPOSE 8080
|
||||||
|
|
||||||
|
HEALTHCHECK --interval=10s --timeout=5s --start-period=20s --retries=3 \
|
||||||
|
CMD python -c "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=4)"
|
||||||
|
|
||||||
|
CMD ["python", "main.py"]
|
||||||
@@ -1,37 +1,368 @@
|
|||||||
# Отчёт — лабораторная 6
|
# Отчёт — лабораторная 6
|
||||||
|
|
||||||
- Автор, группа: TODO
|
- Автор, группа: Бабиченко Максим, БВТ2303
|
||||||
- Целевая оценка; выбранный трек 5, если нужен: TODO
|
- Целевая оценка; выбранный трек 5, если нужен: **4**, трек 5 не выбран
|
||||||
- Commit/PR: TODO после публикации
|
- Commit/PR: <<ссылка на MR в Gitea после публикации>>
|
||||||
- Использованные источники, библиотеки и помощь ИИ: TODO
|
- Использованные источники, библиотеки и помощь ИИ: всё из lab5 (FastAPI, psycopg 3, pika, boto3, Pillow, RabbitMQ 4.1, SeaweedFS 3.80, PostgreSQL 17, Redis 7.4, HAProxy 3.2). Новое: OpenTelemetry Python SDK 1.45.2 (`opentelemetry-sdk`, `opentelemetry-exporter-otlp-proto-http`) и `opentelemetry-instrumentation-psycopg` 0.66b2; образы `otel/opentelemetry-collector-contrib:0.164.0`, `prom/prometheus:v3.15.1`, `grafana/loki:3.7.9`, `grafana/tempo:3.1.1`, `grafana/grafana:12.4.14`, `busybox:1.37.0` (bootstrap прав volumes). Документация: OTel Collector configuration (memory_limiter, batch, exporterhelper sending_queue/retry_on_failure, internal telemetry), OTel semantic conventions (HTTP, database, messaging, `otel.sdk.*`), W3C Trace Context, Prometheus OTLP receiver и histograms/`histogram_quantile`, promtool unit testing, Loki OTLP ingestion и structured metadata, Tempo 3 single binary, Grafana provisioning (datasources, dashboards, derived fields, traces-to-logs). При написании кода, конфигураций и тестов использовался ИИ-ассистент (Claude); весь код я прогнал и могу объяснить и изменить на защите.
|
||||||
|
|
||||||
## Запуск из чистого clone
|
## Запуск из чистого clone
|
||||||
|
|
||||||
TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных.
|
Проверяю на Windows (PowerShell, Docker Desktop). Python на хосте нужен только для тестов (стандартная библиотека). Перед запуском остановить стенд lab5 (`docker compose down` в его папке): заняты порты 8080/8404. Имя проекта Compose — `messenger-lab6`, volumes отдельные от lab5.
|
||||||
|
|
||||||
|
```powershell
|
||||||
|
git clone https://gitea.privcloud.tech/Kalibr/mtusi2026lab6.git
|
||||||
|
cd mtusi2026lab6
|
||||||
|
git checkout solution/bvt2303-babichenko
|
||||||
|
$env:PYTHONUTF8 = "1"
|
||||||
|
Copy-Item .env.example .env
|
||||||
|
python scripts/init_secrets.py # как в lab5 + secrets/grafana_admin_password (папка secrets/ в .gitignore)
|
||||||
|
docker compose --env-file .env config --quiet
|
||||||
|
docker compose up -d --build --wait # postgres, s3, rabbitmq, otel-collector, prometheus, loki, tempo ->
|
||||||
|
# migrate, s3-init, telemetry-init -> api x2, janitor, dispatcher, worker,
|
||||||
|
# grafana -> lb
|
||||||
|
docker compose ps -a # migrate, s3-init, telemetry-init: Exited (0); остальные running/healthy
|
||||||
|
python tests/smoke.py --base-url http://127.0.0.1:8080 # make test при включённой телеметрии
|
||||||
|
|
||||||
|
# доказательства lab6 (~35 мин) в evidence/ и таблица замеров в этот отчёт:
|
||||||
|
python student-tests/collect_evidence.py check stack smoke configs telemetry dashboard telemetry-history `
|
||||||
|
incident-worker incident-db incident-telemetry incident-collector load
|
||||||
|
```
|
||||||
|
|
||||||
|
| Что | Где |
|
||||||
|
| --- | --- |
|
||||||
|
| Grafana (dashboard «Messenger — lab6 observability», Explore, Alerting) | http://127.0.0.1:3000, логин `admin`, пароль — `secrets/grafana_admin_password` |
|
||||||
|
| Prometheus: алерты pending/firing, запросы | http://127.0.0.1:9090/alerts, http://127.0.0.1:9090/graph |
|
||||||
|
| Loki / Tempo HTTP API (для тестов) | http://127.0.0.1:3100, http://127.0.0.1:3200 |
|
||||||
|
| Трасса конкретного ответа API | заголовок ответа `X-Trace-Id` (или свой `traceparent` в запросе) → Grafana → Explore → Tempo |
|
||||||
|
|
||||||
|
Отдельные проверки (make на Windows нет):
|
||||||
|
|
||||||
|
| Команда | Что проверяет |
|
||||||
|
| --- | --- |
|
||||||
|
| `python scripts/check_contract.py` | make check |
|
||||||
|
| `python student-tests/check_configs.py` | конфиги Collector/Prometheus/Loki/Tempo их же инструментами, unit-тесты алертов (promtool), provisioning Grafana через её API |
|
||||||
|
| `python student-tests/test_telemetry.py [request upload-chain retry-chain replay-link history]` | запрос ↔ лог ↔ трасса, цепочка upload → worker, retry, replay с link, сохранность истории |
|
||||||
|
| `python student-tests/check_dashboard.py` | каждый запрос dashboard и правил выполняется на живых данных, запрещённых меток нет |
|
||||||
|
| `python student-tests/incidents.py [worker-stop db-slow telemetry-down collector-down]` | инциденты: firing → диагностика → resolved |
|
||||||
|
| `docker compose exec prometheus promtool test rules /etc/prometheus/tests/alerts_test.yml` | unit-тесты порогов/окон/`for` алертов |
|
||||||
|
| `python scripts/build_dashboard.py` | пересобрать `infra/grafana/dashboards/messenger.json` из описания панелей |
|
||||||
|
|
||||||
|
Bootstrap: как в lab5 (`migrate`, `s3-init`, definitions RabbitMQ) плюс `telemetry-init` (одноразовый `chown 10001` volumes Loki и Tempo: оба процесса работают не от root). Provisioning Grafana — файлы `infra/grafana/provisioning` и `infra/grafana/dashboards`, монтируются только для чтения: источники данных и dashboard появляются при старте, ручного набора нет. Всё идемпотентно: повторный `up` ничего не пересоздаёт и не дублирует.
|
||||||
|
|
||||||
|
Остановка без потери данных: `docker compose down` (volumes `pgdata`, `redisdata`, `s3data`, `rabbitmqdata`, `faults`, `promdata`, `lokidata`, `tempodata`, `grafanadata` остаются). Намеренное удаление учебных данных: `docker compose down -v`, затем удалить папку `secrets` и снова `python scripts/init_secrets.py`. Выключить телеметрию целиком (для сравнения): `OTEL_SDK_DISABLED=true` в `.env` и `docker compose up -d`.
|
||||||
|
|
||||||
## Архитектура и решения
|
## Архитектура и решения
|
||||||
|
|
||||||
TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab5, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации.
|
```text
|
||||||
|
+--------------------------- сеть telemetry (internal) ----------------------------+
|
||||||
|
Клиент -> lb -> api x2 ---|-- OTLP/HTTP :4318 --> otel-collector --+-- traces OTLP gRPC --> tempo (tempodata) |
|
||||||
|
| | (traces, metrics, memory_limiter |-- logs OTLP HTTP --> loki (lokidata) |
|
||||||
|
| | logs) batch, queues +-- metrics OTLP HTTP --> prometheus(promdata)|
|
||||||
|
PostgreSQL <-+ outbox | ^ pull :8888 (метрики Collector)|
|
||||||
|
(источник истины) | dispatcher, worker, janitor -- OTLP --^ | |
|
||||||
|
| grafana -------+--> prometheus, loki, tempo |
|
||||||
|
+----------------------------------------------------------------------------------+
|
||||||
|
W3C traceparent: API (спан create, outbox) -> dispatcher (спан publish, AMQP headers + поле сообщения)
|
||||||
|
-> worker (спан process, consumer) -> S3/PostgreSQL; retry/replay -> новая строка outbox
|
||||||
|
127.0.0.1: 8080 (lb), 8404 (stats), 3000 (grafana), 9090 (prometheus), 3100 (loki), 3200 (tempo)
|
||||||
|
```
|
||||||
|
|
||||||
|
Перенесено из lab5 без изменения бизнес-логики и контракта: всё приложение (API, outbox, dispatcher, worker, janitor, варианты изображений, повторы, DLQ, replay), миграции, тесты lab1–5, hooks. Бизнес-API не менялся; единственное расширение — необязательный заголовок ответа `X-Trace-Id` (trace_id запроса) и приём входящего `traceparent` (обратно совместимо, клиент может не знать о них).
|
||||||
|
|
||||||
|
Что добавлено в lab6:
|
||||||
|
- `telemetry.py` — настройка OpenTelemetry SDK для всех процессов: ресурс, три провайдера (traces/metrics/logs) с OTLP/HTTP exporter, ограниченные очереди, сэмплер, JSON-логи в stderr + OTLP, метрики процесса/контейнера, W3C propagator, помощники спанов.
|
||||||
|
- Инструментирование: HTTP server-спаны и метрики в middleware `main.py`; спаны PostgreSQL (`opentelemetry-instrumentation-psycopg`, текст SQL с плейсхолдерами, без значений), S3 (`storage.py`), Redis и Argon2id (`auth.py`); messaging-спаны и перенос контекста через outbox/RabbitMQ (`pipeline.py`, `dispatcher.py`, `broker.py`, `worker.py`); метрики очереди/outbox (dispatcher), пула БД (`db.py`), занятости worker, времени до ready; структурные события логов (`event=...`).
|
||||||
|
- Конфигурации в Git: `infra/otel/collector.yaml`, `infra/prometheus/prometheus.yml`, `infra/prometheus/rules/{recording,alerts}.yml`, `infra/prometheus/tests/alerts_test.yml`, `infra/loki/loki.yaml`, `infra/tempo/tempo.yaml`, `infra/grafana/provisioning/**`, `infra/grafana/dashboards/messenger.json` (генерирует `scripts/build_dashboard.py`), `docs/runbooks.md`.
|
||||||
|
- `compose.yaml`: сервисы `otel-collector`, `telemetry-init`, `prometheus`, `loki`, `tempo`, `grafana`; сети `telemetry` (internal) и `ops` (только чтобы опубликовать UI на 127.0.0.1); volumes `promdata`, `lokidata`, `tempodata`, `grafanadata`; секрет `grafana_admin_password`. Restart hook перезапускает и хранилища телеметрии.
|
||||||
|
- Тесты: `student-tests/{obs.py, test_telemetry.py, check_dashboard.py, check_configs.py, incidents.py}`, обновлены `check_stack.py`, `run_persistence.py`, `collect_evidence.py`.
|
||||||
|
|
||||||
|
### Роли и назначение хранилищ
|
||||||
|
|
||||||
|
| Роль | Сервис | Версия | Зачем | Health | Volume | Сеть / порт | Credentials |
|
||||||
|
| --- | --- | --- | --- | --- | --- | --- | --- |
|
||||||
|
| Collector | `otel-collector` | otelcol-contrib 0.164.0 | приём OTLP, защита памяти, пакетирование, буфер на время недоступности хранилища, маршрутизация в 3 хранилища. **Не хранит** | метрики `otelcol_*` (scrape) — образ distroless, без shell | нет (очередь в памяти) | telemetry; 4317/4318/8888 не публикуются | нет |
|
||||||
|
| Метрики | `prometheus` | 3.15.1 | числовые ряды: «что, сколько, когда», percentiles из histogram, SLI, alert rules, состояние алертов | `/-/ready` | `promdata` (7 дней, ≤ 1 GB) | telemetry, ops; 127.0.0.1:9090 | нет |
|
||||||
|
| Логи | `loki` | 3.7.9 | события с контекстом: «почему» (error_type, request_id, trace_id) | — (`/ready` проверяют тесты) | `lokidata` (7 дней) | telemetry, ops; 127.0.0.1:3100 | нет |
|
||||||
|
| Трассы | `tempo` | 3.1.1 | причинная цепочка запроса через процессы и время каждого шага: «где» | — (`/ready`) | `tempodata` (7 дней) | telemetry, ops; 127.0.0.1:3200 | нет |
|
||||||
|
| UI | `grafana` | 12.4.14 | запросы к трём хранилищам, dashboard, связки лог ↔ трасса ↔ метрика; сама сигналы не хранит | `/api/health` | `grafanadata` (сессии/настройки UI) | telemetry, ops; 127.0.0.1:3000 | `grafana_admin_password` |
|
||||||
|
|
||||||
|
Почему так: Grafana сама не хранит сигналы, а каждому сигналу нужно своё хранилище с подходящей моделью — TSDB для рядов (Prometheus умеет OTLP напрямую, поэтому без remote-write и без exporter-а в Collector), индекс по меткам + сжатые строки для логов (Loki принимает OTLP и хранит атрибуты записи как structured metadata, поэтому `trace_id`/`request_id` не становятся метками индекса), блоки трасс по trace_id (Tempo 3 single binary без Kafka). Collector отделяет приложение от хранилищ: приложение знает один адрес, а смена/недоступность хранилища его не касается.
|
||||||
|
|
||||||
|
### Сигналы: контракт → реальные имена
|
||||||
|
|
||||||
|
Имена OTel переводит в Prometheus `translation_strategy: UnderscoreEscapingWithSuffixes` (зафиксировано в `prometheus.yml`): точки → `_`, суффикс единицы (`_seconds`, `_bytes`) и `_total` для счётчиков; `service.name` → метка `job`, `service.instance.id` → `instance`, остальные ресурсные атрибуты — в `target_info`.
|
||||||
|
|
||||||
|
| Сигнал контракта (уровень) | Инструмент OTel (единица) | В хранилище | Метки / поля |
|
||||||
|
| --- | --- | --- | --- |
|
||||||
|
| HTTP requests (3) | `http.server.request.duration` histogram, его `_count` | `http_server_request_duration_seconds_count` | `http_request_method`, `http_route` (шаблон), `http_response_status_code`, `http_response_status_class` (`2xx`…) |
|
||||||
|
| HTTP duration (3) | то же, секунды; buckets 5 мс … 10 с | `http_server_request_duration_seconds_bucket` | те же + `le` |
|
||||||
|
| Jobs processed / failed (3) | `messenger.job.duration` histogram, `_count` (одна метрика, без дубля) | `messenger_job_duration_seconds_count` | `job_type="attachment.process.v1"`, `outcome` = ready / retry / failed / stale / lease_lost / db_error |
|
||||||
|
| Job processing duration (3) | `messenger.job.duration` (s), buckets 0,1 … 120 с | `messenger_job_duration_seconds_bucket` | те же |
|
||||||
|
| Доставки/публикации (доп.) | `messenger.messages.consumed`, `messenger.messages.published` counters | `messenger_messages_consumed_total`, `..._published_total` | `messaging_destination_name`, `outcome` (processed/noop/dead_lettered/requeued; ok/error) |
|
||||||
|
| Логи API и worker (3) | Python `logging` → OTel Logs API | Loki: метки `service_name`, `service_instance_id`, `deployment_environment_name`; structured metadata `event`, `severity_text`, `trace_id`, `span_id`, `request_id`, `job_id`, `error_type`… | в stderr та же запись одной JSON-строкой (`ts`, `severity`, `service.name`, `event`, `trace_id`, `span_id`) |
|
||||||
|
| Трассы API и worker (3) | server / producer / consumer / client / internal спаны | Tempo | `http.route`, `http.response.status_code`, `messaging.*`, `messenger.job.*`, `db.statement`, `aws.s3.key`, `error.type` |
|
||||||
|
| Queue depth / oldest job age (4) | `messenger.queue.depth` gauge; `messenger.queue.oldest_age` (s); доп. `messenger.jobs.backlog`, `messenger.queue.consumers`, `messenger.outbox.pending`, `messenger.outbox.oldest_age` (s) | `messenger_queue_depth`, `messenger_queue_oldest_age_seconds`, … | `messaging_destination_name` (очередь/DLQ), `status` |
|
||||||
|
| DB pool used/waiting (4) | `db.client.connection.count` (`state`=used/idle), `.max`, `.pending_requests`, `.timeouts`, `db.client.connection.wait_time` (s, histogram) | `db_client_connection_*` | `db_client_connection_pool_name="messenger-api"`, `db_client_connection_state` |
|
||||||
|
| Worker busy/capacity (4) | `messenger.worker.busy`, `messenger.worker.capacity` | `messenger_worker_busy`, `messenger_worker_capacity` | `messaging_destination_name` |
|
||||||
|
| Process / container CPU, RAM (4) | `process.cpu.time` (s), `process.memory.usage` (By); `container.cpu.time`, `container.cpu.limit`, `container.memory.usage`, `container.memory.limit` (cgroup v2, только в Docker) | `process_cpu_time_seconds_total`, `process_memory_usage_bytes`, `container_*` | `cpu_mode` |
|
||||||
|
| End-to-end time to ready (4) | `messenger.attachment.time_to_ready` (s), buckets 0,5 … 600 с; от COMMIT задания в API до COMMIT ready, по часам PostgreSQL | `messenger_attachment_time_to_ready_seconds_*` | `job_type` |
|
||||||
|
| Сквозная связь upload → publish → consume → S3/DB (4) | W3C `traceparent`/`tracestate`: parent-child, replay — link | Tempo | см. ниже |
|
||||||
|
| Потери телеметрии (4) | SDK: `otel.sdk.processor.span.processed{error.type}`, `otel.sdk.exporter.span.exported{error.type}`; Collector: `otelcol_exporter_{sent,send_failed,enqueue_failed}_*`, `otelcol_exporter_queue_size/capacity`, `otelcol_receiver_refused_*` | Prometheus | `exporter`, `error_type` |
|
||||||
|
|
||||||
|
Ресурс (одинаков для всех трёх сигналов процесса): `service.name` = `messenger-api` / `messenger-worker` / `messenger-dispatcher` / `messenger-janitor` (и `messenger-ops` у `ops.py`) — задаётся в коде процесса, поэтому общий `.env` не делает janitor «api»; `service.version` = `APP_VERSION`; `service.instance.id` = hostname контейнера (тот же, что `X-Instance-Id`); `deployment.environment.name=lab` (`OTEL_RESOURCE_ATTRIBUTES`).
|
||||||
|
|
||||||
|
**Что не попадает в телеметрию.** Метки метрик — только перечисленные выше (метод, шаблон маршрута, класс/код статуса, тип/исход задания, очередь, состояние соединения, режим CPU); `check_dashboard.py` проверяет, что у метрик приложения нет `user_id`, `chat_id`, `message_id`, `attachment_id`, `request_id`, URL, имени файла, текста и значений-UUID. В спанах и логах есть технические ID синтетических данных (`job_id`, `attachment_id`, `request_id`, ключ S3 вида `attachments/<uuid>/v1/crop.jpg`) — без них нельзя пройти от загрузки к результату worker. Не пишутся: пароли, `Authorization`, токен сессии и его хеш (спаны Redis без ключа), текст сообщения, имя файла, байты изображения, значения параметров SQL (`capture_parameters=False`), signed URL (их нет). `test_telemetry.py request` ищет текст сообщения, пароль и токен в Loki и атрибутах спанов — 0 совпадений. В Collector дополнительно `attributes/scrub` удаляет любой атрибут, чьё имя похоже на password/authorization/cookie/secret/token/api key.
|
||||||
|
|
||||||
|
### Перенос контекста через очередь
|
||||||
|
|
||||||
|
```text
|
||||||
|
[api] POST /api/v1/chats/{chat_id}/attachments SERVER
|
||||||
|
├─ PostgreSQL INSERT attachments, S3 PutObject staging/<hex>, UPDATE ... state='queued'
|
||||||
|
└─ create attachments PRODUCER ← строка outbox в ТОЙ ЖЕ транзакции; traceparent этого спана
|
||||||
|
│ пишется в attachment_jobs.traceparent и в payload outbox
|
||||||
|
└─ [dispatcher] publish attachments PRODUCER ← родитель = traceparent из outbox; свой контекст
|
||||||
|
│ кладёт в AMQP header traceparent и в поле сообщения
|
||||||
|
└─ [worker] process attachments.process.v1 CONSUMER ← родитель = header (запасной — поле тела)
|
||||||
|
├─ claim (PostgreSQL), S3 GetObject, imaging, S3 PutObject x4, commit result (PostgreSQL)
|
||||||
|
├─ (ошибка) create attachments PRODUCER → publish → process (попытка 2) — та же трасса
|
||||||
|
└─ cleanup staging (S3 DeleteObject, UPDATE) — после ack, в той же трассе
|
||||||
|
```
|
||||||
|
|
||||||
|
Выбранная семантика: **parent-child** вдоль причинной цепочки (публикация и обработка — продолжение запроса пользователя, а задержка между ними видна как разрыв во времени). **Link** — когда это новое решение, а не продолжение: `ops.py replay` создаёт новую трассу (корень `ops replay`), спан `create` которой ссылается на исходную загрузку. Восстановление истёкшей аренды dispatcher-ом и сверка «queued без сообщения» создают спаны `recover expired lease` / `republish lost message` с родителем из `attachment_jobs.traceparent`. В lab5 `traceparent` был случайным; теперь это контекст реального спана.
|
||||||
|
|
||||||
|
Сэмплирование: `ParentBased(AlwaysOn)`, но клиентский спан без родителя не записывается — это фоновые опросы (dispatcher раз в секунду читает outbox, продление аренды, проверка соединений пула). Всё, что относится к запросу или заданию, сохраняется полностью (100%). Проверки здоровья (`/health/*`, HAProxy раз в секунду) — только в метриках, без трасс.
|
||||||
|
|
||||||
|
### SLI / SLO и окно
|
||||||
|
|
||||||
|
| SLO | SLI (числитель / знаменатель) | Цель | Окно | Recording rule |
|
||||||
|
| --- | --- | --- | --- | --- |
|
||||||
|
| SLO-1 доступность API | бизнес-запросы API без 5xx / все бизнес-запросы API (все маршруты кроме `/health/*`). 4xx — ошибки клиента, не входят в числитель | ≥ 99% | скользящий 1 ч (учебное окно; в эксплуатации то же выражение с 28d) | `messenger:slo_availability:ratio_1h`, бюджет `messenger:slo_availability:error_budget_remaining_1h` |
|
||||||
|
| SLO-2 задержка записи | успешные `POST /api/v1/chats/{chat_id}/messages` быстрее 250 мс (bucket `le="0.25"`) / все успешные такие запросы | ≥ 95% | 1 ч | `messenger:slo_latency_messages:ratio_1h` |
|
||||||
|
| SLO-3 фоновая обработка | вложения ready не позже 30 с от приёма (bucket `le="30"` гистограммы time_to_ready) / все ставшие ready | ≥ 95% | 1 ч | `messenger:slo_time_to_ready:ratio_1h` |
|
||||||
|
|
||||||
|
Проверяемость: все SLI — отношения счётчиков/buckets гистограмм, просуммированных по репликам (`sum(increase(...[1h]))`), поэтому SLO вычисляется одинаково на любом наборе реплик и видно на dashboard (строка «Обзор и SLO»).
|
||||||
|
|
||||||
|
### Dashboard: запросы и единицы
|
||||||
|
|
||||||
|
Dashboard — `infra/grafana/dashboards/messenger.json` (импортируется provisioning-ом, папка Messenger; 34 панели в 6 строках: обзор/SLO и алерты, API RED, фоновая обработка, насыщение, конвейер телеметрии, логи и трассы). Ключевые запросы (`$__rate_interval` подставляет Grafana; `API` = `job="messenger-api", http_route!~"/health/.*"`):
|
||||||
|
|
||||||
|
| Панель | PromQL | Единица |
|
||||||
|
| --- | --- | --- |
|
||||||
|
| RPS по маршрутам / по классу статуса | `sum by (http_request_method, http_route) (rate(http_server_request_duration_seconds_count{API}[$__rate_interval]))` | req/s |
|
||||||
|
| Доля 5xx | `(sum(rate(..._count{API, http_response_status_class="5xx"}[…])) or on() vector(0)) / sum(rate(..._count{API}[…]))` | доля (0–1) |
|
||||||
|
| p50 / p95 / p99 API | `histogram_quantile(0.95, sum by (le) (rate(http_server_request_duration_seconds_bucket{API}[$__rate_interval])))` | с |
|
||||||
|
| p95 по маршрутам | `histogram_quantile(0.95, sum by (le, http_route) (rate(..._bucket{API}[…])))` | с |
|
||||||
|
| Попытки заданий по исходу | `sum by (outcome) (rate(messenger_job_duration_seconds_count[…]))` | 1/с |
|
||||||
|
| Длительность попытки / время до ready p50/p95/p99 | `histogram_quantile(q, sum by (le) (rate(messenger_job_duration_seconds_bucket{outcome="ready"}[…])))`, то же для `messenger_attachment_time_to_ready_seconds_bucket` | с |
|
||||||
|
| Глубина очереди и backlog | `max by (messaging_destination_name) (messenger_queue_depth)`, `max by (status) (messenger_jobs_backlog)` | сообщений / заданий |
|
||||||
|
| Возраст старейшего задания и outbox | `max(messenger_queue_oldest_age_seconds{messaging_destination_name="attachments.process.v1"})`, `max(messenger_outbox_oldest_age_seconds)` | с |
|
||||||
|
| Пул БД | `sum by (instance) (db_client_connection_count{db_client_connection_state="used"})`, `max(db_client_connection_max)`, `sum by (instance) (db_client_connection_pending_requests)`; p95 ожидания `histogram_quantile(0.95, sum by (le, instance) (rate(db_client_connection_wait_time_seconds_bucket[…])))` | соединений / с |
|
||||||
|
| Насыщение worker | `sum(messenger_worker_busy) / sum(messenger_worker_capacity)` | доля |
|
||||||
|
| CPU / RAM | `sum by (job, instance) (rate(process_cpu_time_seconds_total[…]))` (ядер), `rate(container_cpu_time_seconds_total) / container_cpu_limit` (доля лимита), `process_memory_usage_bytes`, `container_memory_usage_bytes` | ядра / доля / байты |
|
||||||
|
| Collector и SDK | `rate(otelcol_receiver_accepted_spans[…])`, `rate(otelcol_exporter_send_failed_spans[…])`, `otelcol_exporter_queue_size / otelcol_exporter_queue_capacity`, `otelcol_process_memory_rss`, `rate(otel_sdk_exporter_span_exported_total{error_type!=""}[…])` | 1/с, доля, байты |
|
||||||
|
| Логи / трассы | Loki `{service_name=~"messenger-.*"} \| detected_level=~"warn\|error"`; Tempo TraceQL `{ status = error }` | — |
|
||||||
|
|
||||||
|
Percentile только из buckets гистограммы, просуммированных по всем репликам **до** `histogram_quantile` (`sum by (le)`); среднее и усреднение p95 реплик не используются. Точность — ширина bucket: например, при запросах по 5,0 с (statement_timeout) интерполяция внутри bucket 5–7,5 с даёт p95 ≈ 7,4 с; это ограничение метода, видно на инциденте 2.
|
||||||
|
|
||||||
|
### Алерты
|
||||||
|
|
||||||
|
`infra/prometheus/rules/alerts.yml`; состояние — Prometheus `/alerts` и `ALERTS{}` (видно и resolved по истории), Grafana показывает их как data source rules. Внешней доставки нет (по заданию).
|
||||||
|
|
||||||
|
| Алерт | Условие | Окно | Порог | for | Runbook |
|
||||||
|
| --- | --- | --- | --- | --- | --- |
|
||||||
|
| MessengerHighErrorRate | доля 5xx бизнес-запросов API и трафик > 0,2 запроса/с | rate 1m | > 5% (в 5 раз выше бюджета SLO-1) | 1m | `docs/runbooks.md#messengerhigherrorrate` |
|
||||||
|
| MessengerQueueLagGrowing | возраст старейшего готового задания | мгновенное значение (обновление 5–10 с) | > 30 с (SLO-3) | 1m | `#messengerqueuelaggrowing` |
|
||||||
|
| MessengerWorkerNoConsumers | consumers очереди = 0 | — | 0 | 1m | `#messengerworkernoconsumers` |
|
||||||
|
| MessengerDbPoolSaturated | запросы ждут соединение из пула | — | > 0 | 1m | `#messengerdbpoolsaturated` |
|
||||||
|
| TelemetryExportFailing | потери в Collector (send/enqueue failed) **или** очередь exporter заполнена > 5% | rate 2m | > 0 / > 5% | 1m | `#telemetryexportfailing` |
|
||||||
|
| TelemetryCollectorDown | `up{job="otel-collector"} == 0` | — | — | 1m | `#telemetrycollectordown` |
|
||||||
|
|
||||||
|
Пороги, окна и `for` проверены unit-тестами `infra/prometheus/tests/alerts_test.yml` (promtool: не срабатывает раньше `for`, не срабатывает при пустом трафике и на ошибках `/health`, resolved после восстановления) и живыми инцидентами ниже.
|
||||||
|
|
||||||
|
### Collector и SDK: память и недоступность хранилища
|
||||||
|
|
||||||
|
- SDK (каждый процесс): `BatchSpanProcessor` и `BatchLogRecordProcessor` с очередью 2048 (`OTEL_BSP_MAX_QUEUE_SIZE`, `OTEL_BLRP_MAX_QUEUE_SIZE`), экспорт в фоновом потоке, таймаут 5 с (`OTEL_EXPORTER_OTLP_TIMEOUT`); метрики — агрегаты фиксированного размера раз в 10 с. Переполнение и непринятые пакеты считаются самим SDK (`OTEL_PYTHON_SDK_INTERNAL_METRICS_ENABLED=true`). Ошибки экспорта пишутся в лог не чаще раза в 30 с и не отправляются тем же экспортом (иначе петля).
|
||||||
|
- Collector: первым в каждом pipeline `memory_limiter` (heap Go: жёсткий предел 200 MiB, мягкий 160 MiB — receiver отвечает клиентам ошибкой, SDK отбрасывает у себя), затем `batch` (2 с / 1024–2048 элементов). У каждого exporter своя `sending_queue` в памяти (Tempo, Loki — 200 пакетов, Prometheus — 100) и `retry_on_failure` (1 → 10 с, `max_elapsed_time` 120 с). Окно повторов считается для пакета с первой попытки; ожидание в очереди в него не входит, поэтому буфер на время недоступности — это очередь. Пакет отбрасывается, когда истекли его 120 с повторов (`otelcol_exporter_send_failed_*`) или очередь полна (`otelcol_exporter_enqueue_failed_*`). Pipeline независимы: недоступный Tempo не мешает логам и метрикам. Очередь не персистентная: рестарт Collector теряет буфер (осознанный компромисс учебного стенда, `file_storage` не нужен).
|
||||||
|
- Контейнер Collector 384 MiB: RSS = heap (≤ 200 MiB) + runtime Go + отображённый в память бинарник contrib (~150 MiB вытесняемых страниц); поэтому тест сравнивает с лимитом именно heap (`otelcol_process_runtime_heap_alloc_bytes`), а RSS/память контейнера приводит для сведения.
|
||||||
|
|
||||||
|
### Соответствие `.env.example`
|
||||||
|
|
||||||
|
| Переменная | Что делает в этом решении |
|
||||||
|
| --- | --- |
|
||||||
|
| всё из lab5 | как в lab5 (см. REPORT lab5) |
|
||||||
|
| OTEL_SERVICE_NAME | `messenger-api` для api; compose задаёт свой каждому сервису; фактически `service.name` берётся из кода процесса |
|
||||||
|
| OTEL_EXPORTER_OTLP_ENDPOINT / _PROTOCOL | `http://otel-collector:4318`, `http/protobuf` (SDK добавляет `/v1/traces`, `/v1/metrics`, `/v1/logs`) |
|
||||||
|
| OTEL_RESOURCE_ATTRIBUTES | `deployment.environment.name=lab` |
|
||||||
|
| OTEL_SDK_DISABLED | `true` — SDK выключен во всех процессах (логи остаются JSON в stderr); для сравнения telemetry off/on |
|
||||||
|
| (только compose) OTEL_EXPORTER_OTLP_TIMEOUT, OTEL_METRIC_EXPORT_INTERVAL, OTEL_BSP_MAX_QUEUE_SIZE, OTEL_BLRP_MAX_QUEUE_SIZE, OTEL_PYTHON_SDK_INTERNAL_METRICS_ENABLED | 5 с, 10000 мс, 2048, 2048, `true` |
|
||||||
|
|
||||||
## Доказательства критериев
|
## Доказательства критериев
|
||||||
|
|
||||||
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
|
`python student-tests/collect_evidence.py …` пишет вывод каждого шага в `evidence/<шаг>.txt` (дата, commit, код выхода), итог — `evidence/summary.md`, JSON инцидентов — `evidence/incident-*.json`, а таблицы замеров — в блок «Замеры (Docker)» ниже. Те же сценарии прогнаны в нативной песочнице (те же версии Collector/Prometheus/Loki/Tempo, процессы без Docker; см. «Где проверено») — `evidence/native-sandbox/`.
|
||||||
| --- | --- | --- | --- |
|
|
||||||
| Публичный smoke | make test | TODO | TODO |
|
|
||||||
| Остальные пункты уровня 3 | TODO | TODO | TODO |
|
|
||||||
| Все пункты уровня 4, если заявлен | TODO | TODO | TODO |
|
|
||||||
| Выбранный трек 5, если заявлен | TODO | TODO | TODO |
|
|
||||||
|
|
||||||
TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены.
|
| Критерий README | Команда/сценарий | Наблюдаемый результат (Docker-стенд, `evidence/`; то же в нативной песочнице — `evidence/native-sandbox/`) | Файл доказательства |
|
||||||
|
| --- | --- | --- | --- |
|
||||||
|
| 3: минимум lab5 работает, `make test` при включённой телеметрии | `tests/smoke.py` | PASS через балансировщик с SDK во всех процессах | smoke.txt |
|
||||||
|
| 3: Collector, 3 хранилища и Grafana в одном воспроизводимом стенде; конфиги/provisioning в Git; persistent volumes | `check_stack.py`, `check_configs.py`, `test_telemetry.py history` | конфиги валидны инструментами тех же версий; unit-тесты алертов SUCCESS; после restart prometheus/loki/tempo прежние трасса, логи и точка метрики находятся | stack.txt, configs.txt, telemetry-history.txt |
|
||||||
|
| 3: API и worker отдают OTLP; RPS/errors/duration API, jobs ok/failed и duration worker; логи и реальные трассы | `check_dashboard.py`, `test_telemetry.py` | все 54 PromQL-выражения dashboard и 16 выражений правил выполняются без ошибок, ключевые сигналы с данными; трассы API, dispatcher, worker | dashboard.txt, telemetry.txt |
|
||||||
|
| 3: импортируемый dashboard; низкая кардинальность; trace/span id в логах; нет содержимого/секретов | `check_dashboard.py`, `check_configs.py` (Grafana API), `test_telemetry.py request` | метки только из белого списка, `http_route` — шаблоны; dashboard provisioned=true; текст, пароль, токен: 0 совпадений в Loki и спанах | dashboard.txt, configs.txt, telemetry.txt |
|
||||||
|
| 3: контролируемый запрос → trace + лог; job → worker trace + лог | `test_telemetry.py request upload-chain` | свой `traceparent` → server-спан с дочерними PostgreSQL/Redis → лог `event=http.request` с тем же trace_id/span_id и request_id = `X-Request-Id`; логи worker `job.claimed`/`job.ready` с trace_id загрузки | telemetry.txt |
|
||||||
|
| 3: назначение каждого хранилища | раздел «Роли и назначение хранилищ» | — | REPORT.md |
|
||||||
|
| 4: W3C через API → broker → worker, producer/consumer, цепочка до S3/DB, включая retry | `test_telemetry.py upload-chain retry-chain replay-link` | одна трасса: server → create (producer) → publish (producer, dispatcher) → process (consumer, worker) → 4×S3 PutObject + commit; retry: попытка 1 ERROR `storage_unavailable` → create → publish → попытка 2 ready; replay — новая трасса с link на исходную | telemetry.txt |
|
||||||
|
| 4: p50/p95/p99 из histogram, доля ошибок, queue depth/oldest age, DB pool, saturation worker; запросы и единицы | dashboard + раздел «Dashboard: запросы и единицы» | все панели выполняются на живых данных | dashboard.txt |
|
||||||
|
| 4: SLI/SLO и окно; alert rules high error rate и queue lag; проверены отказом (firing/resolved) | `incidents.py worker-stop db-slow`, `promtool test rules` | см. таблицы инцидентов: firing и resolved по каждому алерту | incident-worker.txt, incident-db.txt, configs.txt |
|
||||||
|
| 4: memory limiter/batch, ограниченная буферизация; остановка хранилища телеметрии не ломает сообщения и не растит RAM | `incidents.py telemetry-down collector-down` | API 100% успешных ответов, heap Collector ограничен, потери посчитаны | incident-telemetry.txt, incident-collector.txt |
|
||||||
|
| 4: разбор двух инцидентов и шаги диагностики | раздел «Эксперименты и отказы» | — | REPORT.md, docs/runbooks.md |
|
||||||
|
| Выбранный трек 5 | — | не выбран | — |
|
||||||
|
|
||||||
|
### Где проверено
|
||||||
|
|
||||||
|
**Нативная песочница (выполнено).** Среда, где писалось решение, не имеет доступа к Docker Hub, поэтому весь стенд был поднят теми же процессами без контейнеров: PostgreSQL 16, Redis 7, RabbitMQ 3.12, SeaweedFS 3.80, HAProxy 2.8, 2 реплики API за балансировщиком, dispatcher, worker, janitor (Python 3.11, те же `requirements.txt`) и **те же версии** otelcol-contrib 0.164.0, Prometheus 3.15.1, Loki 3.7.9, Tempo 3.1.1 с конфигами из `infra/` (заменены только адреса `tempo:4317` → `127.0.0.1:14317` и т.п.). Тесты запускались без изменений через тонкую подмену команды `docker compose` (stop/start/exec на процессы). Результаты — `evidence/native-sandbox/`, цифры — в разделе ниже. Grafana в песочнице не запускалась (нет бинарника без Docker Hub): её provisioning проверяет `check_configs.py` через HTTP API Grafana на Docker-стенде, а все запросы dashboard выполнены против живых Prometheus/Loki/Tempo (`check_dashboard.py`).
|
||||||
|
|
||||||
|
**Docker-стенд (Windows 10, Docker Desktop 28.1.1, Compose 2.35.1; VM 16 CPU / 15,2 GiB) — выполнено 2026-11-01.** Стенд поднят из чистого состояния (`init_secrets.py`, `docker compose pull`, `up -d --build --wait`) без правок кода и конфигов: `migrate`, `s3-init`, `telemetry-init` — Exited (0), остальные running/healthy; Loki и Tempo пишут в volumes от uid 10001 без ошибок прав. Полный прогон `collect_evidence.py` — все шаги PASS: lab6 (check, stack, smoke, configs, telemetry, dashboard, telemetry-history, 4 инцидента, load) и регрессия lab1–5 (`--from jobs-variants`, 22 шага, включая failover и persistence; restart hook вместе с ожиданием готовности — 60,9 с при лимите 90 с). Вывод — `evidence/*.txt`, сводка — `evidence/summary.md`, таблицы — блок «Замеры (Docker)» ниже. Grafana проверена через её API: 3 источника данных healthy, dashboard provisioned, трасса по `X-Trace-Id` открывается через источник Tempo, лог с тем же `trace_id` — через Loki (связки tracesToLogs и derived field TraceID).
|
||||||
|
|
||||||
## Эксперименты и отказы
|
## Эксперименты и отказы
|
||||||
|
|
||||||
TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения.
|
Гипотезы:
|
||||||
|
1. По сохранённым метрикам, логам и трассам можно найти причину инцидента, не заходя в контейнеры: остановленный worker отличается от медленной БД, а медленная БД — от медленного обработчика.
|
||||||
|
2. Недоступность любого звена телеметрии (хранилище, Collector) не влияет на ответы API, память Collector и процессов приложения ограничена, а потери телеметрии считаются.
|
||||||
|
|
||||||
|
Методика: `student-tests/incidents.py` — нагрузка (closed loop, POST сообщений), контролируемый отказ, опрос `/api/v1/alerts` каждые 5 с до firing, «диагностика» только запросами к Prometheus/Loki/Tempo, устранение отказа, ожидание resolved; хронология в выводе, итог в `evidence/incident-*.json` (без токенов и текстов).
|
||||||
|
|
||||||
|
### Нативная песочница: результаты
|
||||||
|
|
||||||
|
Окружение: Linux x86_64, 2 vCPU, 8 GiB RAM на всё (стенд, хранилища телеметрии и клиент нагрузки); PostgreSQL 16, Redis 7, RabbitMQ 3.12, SeaweedFS 3.80, HAProxy 2.8, 2 реплики API, Python 3.11; otelcol-contrib 0.164.0, Prometheus 3.15.1, Loki 3.7.9, Tempo 3.1.1 (те же версии и конфиги, что в compose.yaml). Дата прогона 2026-11-01. Сводка шагов (все PASS, включая регрессию lab1–5: варианты изображений, outbox, повторы/DLQ/replay, конкуренция worker, вложения, идемпотентность, гонки) — `evidence/native-sandbox/summary.md`.
|
||||||
|
|
||||||
|
Алерты на контролируемых инцидентах (`student-tests/incidents.py`, состояние из Prometheus /api/v1/alerts):
|
||||||
|
|
||||||
|
| Инцидент | Воздействие | Алерт | firing через, с | resolved через, с после устранения |
|
||||||
|
| --- | --- | --- | --- | --- |
|
||||||
|
| 1. worker | `docker compose stop worker` + 3 загрузки | MessengerWorkerNoConsumers | 66.4 (от stop) | 17.0 (от start) |
|
||||||
|
| 1. worker | `docker compose stop worker` + 3 загрузки | MessengerQueueLagGrowing | 111.4 (от stop) | 17.0 (от start) |
|
||||||
|
| 2. PostgreSQL | `LOCK TABLE messages IN ACCESS EXCLUSIVE MODE` на 210 с под нагрузкой 8 потоков | MessengerHighErrorRate | 120.1 (от LOCK) | 30.0 (от COMMIT блокировки) |
|
||||||
|
| 3. телеметрия | `docker compose stop tempo loki` на 150 с | TelemetryExportFailing | 101.4 (от stop) | 155.4 (от start) |
|
||||||
|
|
||||||
|
Инцидент 1, наблюдения в момент firing: consumers очереди = 0.0, сообщений в RabbitMQ = 3.0, queued заданий = 3.0, возраст старейшего = 97.3 с, возраст outbox = 0.0 с (публикация работает), метрики worker не обновлялись 110.0 с, последний лог worker: event=`service.stopped`; трасса загрузки содержит `publish attachments`, но не содержит `process attachments.process.v1`. После start: вложения ready через 2.0 с, backlog 0.0.
|
||||||
|
|
||||||
|
Инцидент 2, наблюдения: доля 5xx (1m) = 1.0, коды POST /messages за 1m = {'201': 0.0, '200': 0.0, '404': 0.0, '400': 0.0, '409': 0.0, '503': 1.6}, p95 POST /messages = 7.375 с, чтения других таблиц (GET /users/me, /chats) во время блокировки {'GET /users/me + GET /chats, avg per request': 4.7, 'max': 5.4} мс, занято соединений пула = 8.0, p95 ожидания пула = 0.001 с, логи API event=db.unavailable по error_type: {'QueryCanceled': 184}. Error-трасса 1c7f8dd06427574a5f155558aa4cd975: server-спан 5006.7 мс, из них `INSERT` 5000.6 мс (доля 0.999, статус ERROR). Контраст «медленный обработчик» (трасса a6da2e3fe7b0c229773f36fa507293ca, POST /users): server 34.0 мс, `argon2id hash` 29.0 мс, все спаны БД 0.6 мс. Нагрузка: до 132.4 успешных записей/с, во время блокировки {'503': 320, '201': 6}, после 131.9/с без ошибок.
|
||||||
|
|
||||||
|
| Отказ телеметрии | API во время отказа | Память | Потери телеметрии |
|
||||||
|
| --- | --- | --- | --- |
|
||||||
|
| `stop tempo loki` на 150 с (окно повторов Collector 120 с) | 11500 запросов, коды {'201': 11500}, доля успеха 1.0 | heap Collector 44.1 -> max 92.7 MiB (limit 200), контейнер max 273.3 MiB | send/enqueue failed в Collector: {'spans': 4096, 'log_records': 269}; пробные трассы дошли 14/15, потеряны отправленные на [142.1] с |
|
||||||
|
| `stop otel-collector` на 90 с | 6263 запросов, коды {'201': 6263}, доля успеха 1.0 | api 280.5 -> max 292.5 MiB (2 реплики) | SDK api потерял спанов: {'processor_queue_full': 0, 'exporter_failed': 24576} (queue_full — очередь 2048 полна, exporter_failed — Collector не принял пакет); после start экспорт возобновился (10524 спанов за минуту) |
|
||||||
|
|
||||||
|
Публичный генератор нагрузки при включённой телеметрии (`tests/load.py`, closed loop, concurrency 4, 30 с, 2 реплики): 291.265 успешных POST/с, ошибок 0, p50 11.557 мс, p95 26.659 мс, p99 56.425 мс (песочница 2 vCPU на всё: стенд + хранилища + клиент; не сравнивать с Docker-машиной).
|
||||||
|
|
||||||
|
### Инцидент 1: остановлен worker («задания не обрабатываются»)
|
||||||
|
|
||||||
|
Воздействие: `docker compose stop worker`, затем 3 загрузки вложений (API отвечает 202 queued — приём от worker не зависит).
|
||||||
|
|
||||||
|
Шаги диагностики (как в `docs/runbooks.md`):
|
||||||
|
1. **Алерт** `MessengerWorkerNoConsumers` (consumers = 0), затем `MessengerQueueLagGrowing` (возраст старейшего задания > 30 с). Dashboard «Возраст старейшего задания» растёт линейно — задания не забирают, а не «их стало много».
|
||||||
|
2. **Публикация или обработка?** `messenger_outbox_oldest_age_seconds` = 0 и `messenger_messages_published_total{outcome="ok"}` растёт → dispatcher и RabbitMQ работают, сообщения в брокере (`messenger_queue_depth` = числу загрузок).
|
||||||
|
3. **Кто не забирает?** `messenger_queue_consumers` = 0, метрики `messenger_worker_*` перестали обновляться (worker не экспортирует), последний лог worker в Loki — `event=service.stopped` (штатная остановка, а не падение — тогда был бы `job.handler_crashed`/обрыв без stopped).
|
||||||
|
4. **Трасса любой из загрузок** (trace_id из лога API `event=http.request` или `X-Trace-Id`): есть `publish attachments`, нет `process attachments.process.v1` — сообщение доставлено в очередь, но не взято.
|
||||||
|
5. Причина: worker остановлен. Действие: `docker compose start worker`. Проверка: вложения ready через 2–5 с (Docker 4,7 с, песочница 2,0 с), backlog → 0, оба алерта resolved (Docker — через 29,7 с после start).
|
||||||
|
|
||||||
|
### Инцидент 2: медленная БД (блокировка таблицы `messages`) vs медленный обработчик
|
||||||
|
|
||||||
|
Воздействие: под нагрузкой 8 потоков `LOCK TABLE messages IN ACCESS EXCLUSIVE MODE` в отдельной транзакции на 210 с (PostgreSQL жив, `/health/ready` 200, но каждая запись ждёт блокировку и через `statement_timeout` 5 с отменяется → API 503).
|
||||||
|
|
||||||
|
Шаги диагностики:
|
||||||
|
1. **Алерт** `MessengerHighErrorRate`. Срабатывает не сразу: окно rate 1m заполнено быстрыми запросами до инцидента (~115–130/с), а во время блокировки closed-loop нагрузка даёт ~1,6 запроса/с (каждый висит 5 с), поэтому доля > 5% появляется только через 50–80 с, плюс `for: 1m` (firing через 140,8 с от LOCK в Docker, 120,1 с в песочнице).
|
||||||
|
2. **Какие маршруты и коды**: только `POST /api/v1/chats/{chat_id}/messages`, код 503; чтения других таблиц (`GET /api/v1/users/me`, `GET /api/v1/chats`) во время блокировки 5–20 мс (Docker 17,2 мс, песочница 4,7 мс) → не «всё медленно», а конкретная таблица/операция.
|
||||||
|
3. **БД или обработчик — по трассе**: TraceQL `{ resource.service.name = "messenger-api" && status = error }` → server-спан ≈ 5005–5015 мс, из них дочерний `INSERT` (PostgreSQL) ≈ 5001 мс со статусом ERROR — 99,8–99,9% времени запрос ждал БД. Для контраста медленный **обработчик**: `POST /api/v1/users` — server 20–35 мс, из них `argon2id hash` 16–29 мс (CPU в процессе API), все спаны БД < 1 мс. По метрикам то же различие: при медленной БД растут `db_client_connection_count{state="used"}` (соединения заняты ожиданием) и p95 конкретного маршрута, при медленном обработчике пул свободен, а растёт CPU процесса API (`process_cpu_time_seconds_total`).
|
||||||
|
4. **Почему — по логам**: `{service_name="messenger-api"} | event="db.unavailable"` с `error_type=QueryCanceled` (statement_timeout), а не `OperationalError` (БД недоступна) или `PoolTimeout` (нет соединений).
|
||||||
|
5. **Подтверждение в PostgreSQL**: `pg_stat_activity` показывает сессии с `wait_event_type=Lock` и держателя блокировки (`LOCK TABLE ...`). Действие: завершить транзакцию-держателя. Проверка: 5xx → 0, алерт resolved через 30–50 с после снятия блокировки (Docker 50,1 с, песочница 30,0 с: окно rate 1m заполняется успешными запросами, затем ближайшая оценка правила).
|
||||||
|
|
||||||
|
### Инцидент 3: недоступна телеметрия (Tempo + Loki; затем сам Collector)
|
||||||
|
|
||||||
|
Ожидаем: API не замечает отказа; Collector копит пакеты в ограниченной очереди, heap не выходит за `memory_limiter`; `TelemetryExportFailing` firing → resolved; после восстановления очередь досылается, потери посчитаны. Без Collector: SDK каждого процесса отбрасывает лишнее (очередь 2048), память API не растёт, потери видны в `otel_sdk_exporter_span_exported_total{error_type!=""}` после восстановления.
|
||||||
|
|
||||||
|
Что показал эксперимент (числа — в таблицах выше):
|
||||||
|
- Tempo + Loki остановлены на 150 с под нагрузкой ~60 записей/с (Docker; в песочнице ~68/с): все ответы API 201 (9555 запросов), p95 не изменился (31 мс до и во время отказа). Очереди exporter Tempo/Loki росли (метрики продолжали идти в Prometheus — pipeline независимы), heap Collector вырос с 35,5 до max 86,9 MiB (песочница 44 → 93 MiB) — далеко от предела 200 MiB, память контейнера Collector не выше 107 MiB; `TelemetryExportFailing` перешёл в firing через 73 с (песочница ~100 с) — по заполнению очереди, ещё до первых потерь. После start очереди досланы за 60 с (песочница ~80 с), алерт resolved через 166 с после start (песочница ~155 с: окно rate 2m по счётчику потерь — последние пакеты с истёкшим окном повторов отброшены около момента восстановления, плюс `for`); все 13 пробных трасс, отправленных во время отказа, нашлись в Tempo (песочница 14 из 15). Потеряно и посчитано Collector-ом 4096 спанов и 235 записей логов (песочница 4096 и 269) — пакеты, у которых истекло окно повторов.
|
||||||
|
- Collector остановлен на 90 с: все ответы API 201 (5852 запроса), память двух реплик API 295 → max 314 MiB (+18 MiB; песочница +12 MiB) и после восстановления не растёт. Экспорт в SDK получает ошибку соединения, пакет отбрасывается (буфера на диске нет). В Docker остановленный контейнер исчезает из DNS сети, и каждая попытка экспорта висит до таймаута 5 с. Экспорт становится медленнее, чем API создаёт спаны, поэтому ограниченная очередь процессора (2048) переполняется и лишнее отбрасывается сразу: за отказ потеряно и посчитано 8252 спана `processor_queue_full` и 10752 `exporter_failed` (`otel_sdk_processor_span_processed_total`, `otel_sdk_exporter_span_exported_total` с `error_type`). В песочнице соединение отклонялось мгновенно: 24,6 тыс. `exporter_failed`, очередь не переполнялась. В обоих случаях память ограничена очередью и ответы API не замедлились (p95 31–32 мс); после start экспорт возобновился без перезапуска приложения (10287 спанов за минуту).
|
||||||
|
- Вывод: отказ любого звена телеметрии не затрагивает пользователей и не растит память без предела; потери ограничены временем отказа и видны в метриках. Короткую недоступность хранилища (до заполнения очереди Collector, здесь минуты) Collector переживает почти без потерь, недоступность самого Collector — нет.
|
||||||
|
|
||||||
|
<!-- measurements:begin -->
|
||||||
|
Машина: Windows 10, AMD64 Family 26 Model 68 Stepping 0, AuthenticAMD; Docker VM: 16 CPU, 15.2 GiB RAM; клиент Python 3.11.1; дата прогона 2026-11-01, commit 470d9d6 + uncommitted changes. Лимиты контейнеров: api 1 CPU / 512 MiB каждая, postgres, s3 (SeaweedFS) и rabbitmq по 1 CPU / 512 MiB, redis и lb по 0,5 CPU / 128 MiB, janitor 0,25 CPU / 192 MiB, dispatcher 0,25 CPU / 128 MiB, worker 2 CPU / 1536 MiB; lab6: otel-collector 0,5 CPU / 384 MiB, prometheus, loki, tempo по 1 CPU / 512 MiB, grafana 0,5 CPU / 384 MiB.
|
||||||
|
|
||||||
|
Алерты на контролируемых инцидентах (`student-tests/incidents.py`, состояние из Prometheus /api/v1/alerts):
|
||||||
|
|
||||||
|
| Инцидент | Воздействие | Алерт | firing через, с | resolved через, с после устранения |
|
||||||
|
| --- | --- | --- | --- | --- |
|
||||||
|
| 1. worker | `docker compose stop worker` + 3 загрузки | MessengerWorkerNoConsumers | 101.7 (от stop) | 29.7 (от start) |
|
||||||
|
| 1. worker | `docker compose stop worker` + 3 загрузки | MessengerQueueLagGrowing | 131.8 (от stop) | 29.7 (от start) |
|
||||||
|
| 2. PostgreSQL | `LOCK TABLE messages IN ACCESS EXCLUSIVE MODE` на 210 с под нагрузкой 8 потоков | MessengerHighErrorRate | 140.8 (от LOCK) | 50.1 (от COMMIT блокировки) |
|
||||||
|
| 3. телеметрия | `docker compose stop tempo loki` на 150 с | TelemetryExportFailing | 73.1 (от stop) | 165.6 (от start) |
|
||||||
|
|
||||||
|
Инцидент 1, наблюдения в момент firing: consumers очереди = 0.0, сообщений в RabbitMQ = 6.0, queued заданий = 3.0, возраст старейшего = 127.3 с, возраст outbox = 0.0 с (публикация работает), метрики worker не обновлялись 131.0 с, последний лог worker: event=`service.stopped`; трасса загрузки содержит `publish attachments`, но не содержит `process attachments.process.v1`. После start: вложения ready через 4.7 с, backlog 0.0.
|
||||||
|
|
||||||
|
Инцидент 2, наблюдения: доля 5xx (1m) = 1.0, коды POST /messages за 1m = {'201': 0.0, '200': 0.0, '404': 0.0, '400': 0.0, '409': 0.0, '503': 1.6}, p95 POST /messages = 7.375 с, чтения других таблиц (GET /users/me, /chats) во время блокировки {'GET /users/me + GET /chats, avg per request': 17.2, 'max': 23.5} мс, занято соединений пула = 8.0, p95 ожидания пула = 0.001 с, логи API event=db.unavailable по error_type: {'QueryCanceled': 216}. Error-трасса 237e2d6b1c7db22adbb7d4752fa47f2: server-спан 5013.3 мс, из них `INSERT` 5001.3 мс (доля 0.998, статус ERROR). Контраст «медленный обработчик» (трасса 4a222e478c2d51006538a7bfeb485eb1, POST /users): server 21.2 мс, `argon2id hash` 16.4 мс, все спаны БД 0.7 мс. Нагрузка: до 115.5 успешных записей/с, во время блокировки {'503': 320, '201': 8}, после 116.8/с без ошибок.
|
||||||
|
|
||||||
|
| Отказ телеметрии | API во время отказа | Память | Потери телеметрии |
|
||||||
|
| --- | --- | --- | --- |
|
||||||
|
| `stop tempo loki` на 150 с (окно повторов Collector 120 с) | 9555 запросов, коды {'201': 9555}, доля успеха 1.0 | heap Collector 35.5 -> max 86.9 MiB (limit 200), контейнер max 107.0 MiB | send/enqueue failed в Collector: {'spans': 4096, 'log_records': 235}; пробные трассы дошли 13/13, потеряны отправленные на [] с |
|
||||||
|
| `stop otel-collector` на 90 с | 5852 запросов, коды {'201': 5852}, доля успеха 1.0 | api 295.2 -> max 313.5 MiB (2 реплики) | SDK api потерял спанов: {'processor_queue_full': 8252, 'exporter_failed': 10752} (queue_full — очередь 2048 полна, exporter_failed — Collector не принял пакет); после start экспорт возобновился (10287 спанов за минуту) |
|
||||||
|
|
||||||
|
Публичный тест отказа (`make test-failover`, 15 с записи во время остановки реплики):
|
||||||
|
|
||||||
|
| Прогон | Остановлена реплика | Реплик до отказа | Успешных записей | Неудачных попыток (повторены с тем же ключом) | Первая запись после hook, с |
|
||||||
|
| --- | --- | --- | --- | --- | --- |
|
||||||
|
| 1 | 171a473b0d28 | 2 | 127 | 0 | 4.75 |
|
||||||
|
| 2 | 1db2fedff78f | 2 | 126 | 0 | 4.578 |
|
||||||
|
|
||||||
|
Одновременные загрузки (`student-tests/upload_memory.py`, перед каждым сценарием реплики перезапускаются, лимит api 512 MiB / 1 CPU). Пик RSS — VmHWM процесса API; cgroup — память контейнера, включая page cache временных файлов:
|
||||||
|
|
||||||
|
| Сценарий | Одновременно | Объём, MiB | Время, с (p50 / max) | Реплика: загрузок | RSS до, MiB | Пик RSS, MiB | Рост, MiB | Пик cgroup, MiB |
|
||||||
|
| --- | --- | --- | --- | --- | --- | --- | --- | --- |
|
||||||
|
| 10 x ~9.5 MiB PNG (noise) | 10 | 94.9 | 1.77 / 1.95 | 171a473b0d28: 5 | 109.6 | 124.5 | 14.9 | 162.8 |
|
||||||
|
| | | | | 1db2fedff78f: 5 | 110.5 | 125.6 | 15.1 | 163.3 |
|
||||||
|
| 10 x 40 Mpx PNG (8000x5000 RGB) | 10 | 1.1 | 0.98 / 1.3 | 171a473b0d28: 5 | 107.6 | 112.4 | 4.8 | 101.5 |
|
||||||
|
| | | | | 1db2fedff78f: 5 | 107.1 | 109.8 | 2.7 | 100.0 |
|
||||||
|
| 20 mixed: large + 40 Mpx + JPEG | 20 | 133.2 | 1.45 / 1.95 | 171a473b0d28: 10 | 109.0 | 136.3 | 27.3 | 191.4 |
|
||||||
|
| | | | | 1db2fedff78f: 10 | 110.6 | 135.0 | 24.3 | 172.8 |
|
||||||
|
|
||||||
|
Один PNG 8000x5000 RGB внутри контейнера api, дополнительный пик RSS: наивно (тело целиком в памяти + полное декодирование Pillow) 161 MiB, потоковая проверка 0 MiB. 10 таких наивных загрузок одновременно не поместились бы в 512 MiB.
|
||||||
|
|
||||||
|
Публичный генератор нагрузки (`tests/load.py`, closed loop, concurrency 4, 30 с, 2 реплики): 239.02 успешных POST/с, ошибок 0 (доля 0.0), задержка успешных p50 16.0 мс, p95 32.0 мс, p99 47.0 мс.
|
||||||
|
|
||||||
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
|
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
|
||||||
| --- | --- | --- | --- |
|
| --- | --- | --- | --- |
|
||||||
| TODO | TODO | TODO | TODO |
|
| публичный `make test-failover` для каждой реплики | запись продолжается не позже 10 с, подтверждённые сообщения на месте, дублей нет | PASS 2 раз(а), неудачных попыток: 0, 0 | первая запись после hook через 4.75, 4.578 с (hook ждёт остановки ~4 с); потерь нет |
|
||||||
|
| перезапуск всего стенда (`restart` hook: postgres, redis, s3, api, janitor, lb) | данные, файл и прежний токен живы | PASS, те же контейнеры с новым временем старта | 60.9 с до готовности; потерь нет |
|
||||||
|
| `docker compose stop s3` (SeaweedFS остановлен) | загрузка и скачивание -> 503, метаданные, сообщения и /health/ready работают | upload 503 за 20234 мс, download 503, остальное 200/201 | после start файлы на месте; uploading-строку убрал janitor; потерь нет |
|
||||||
|
| `docker compose pause s3` (SeaweedFS завис) | то же, но через таймауты S3-клиента | upload 503 за 14703 мс (2 попытки x 5 с + откат 2 с) | после unpause файлы на месте; потерь нет |
|
||||||
|
| падение между PUT в S3 и COMMIT в БД (код API в контейнере, finish_upload падает) | вложение не видно, janitor не трогает его до конца grace period, потом удаляет объект и строку | GET/прикрепление -> 404; janitor: свежая строка сохранена, dry-run только отчёт, затем удалено | мусора не осталось; повторный проход ничего не делает |
|
||||||
|
| обрыв загрузки клиентом (9 МБ заявлено, отправлено 0,2 и 3 МБ) | ничего не сохраняется | строк и объектов не прибавилось, временных файлов 0 | мгновенно |
|
||||||
|
| worker остановлен, upload, рестарт rabbitmq и api с volumes, start worker | 202 queued, сообщение с вложением 201, content 409; после рестартов job в очереди; тот же job_id -> ready | worker started -> same job 6a45774d-0d88-46ec-87b9-70fb4c122bbd ready, thumbnail downloadable; staging object deleted | потерь нет; staging удалён после COMMIT |
|
||||||
|
| rabbitmq остановлен во время upload | 202 и /health/ready 200 без брокера; строка outbox ждёт; после start публикуется | rabbitmq started -> dispatcher published the pending row -> ready | потерь нет |
|
||||||
|
| crash dispatcher между COMMIT задания и publish | строка outbox не отмечена -> публикуется после рестарта процесса | dispatcher killed between COMMIT and publish; restarted by policy, outbox re-published -> job d1eb476a-fbe1-4271-861f-74f3e943b8cc ready, attempt 1, no manual action | без ручного действия |
|
||||||
|
| crash dispatcher после confirm, до отметки outbox | дубль сообщения, worker распознаёт по состоянию job | dispatcher killed after publisher confirm, before marking outbox -> message published twice; worker: ['created', 'claimed', 'ready', 'duplicate_ignored'] -> one result set | один набор вариантов |
|
||||||
|
| crash worker после PUT в S3, до COMMIT и ack | повтор после истечения аренды перезаписывает те же ключи | worker killed after S3 PUT (before COMMIT and ack): redelivery saw a live lease (no-op), lease expired after 15 s, attempt 2 rewrote the same keys -> one set; events ['created', 'claimed', 'duplicate_ignored', 'lease_expired', 'claimed', 'ready'] | один набор, 4 объекта |
|
||||||
|
| crash worker после COMMIT, до ack | redelivery ready-задачи -> no-op ack | worker killed after COMMIT, before ack: broker redelivered, worker acked as no-op (1:duplicate_ignored:status=ready); queue empty (0 ready, 0 unacked) | очередь пуста, дублей нет |
|
||||||
|
| worker завис (не продлевает аренду) | dispatcher возвращает job, другой поток завершает; поздний результат отклонён fencing | frozen attempt 1 stopped renewing; lease expired, recovered by dispatcher, attempt 2 ready after 17 s; frozen attempt woke up -> stale_result_rejected; events ['created', 'claimed', 'lease_expired', 'claimed', 'ready', 'stale_result_rejected'] | ready не откатан |
|
||||||
|
| зависание обработки изображения | процесс убит по JOB_TIMEOUT_SECONDS, временная ошибка, аренда продлевалась | imaging hang -> killed after 60 s (processing_timeout, temporary), lease renewed meanwhile (no lease_expired), attempt 2 ready after 62 s | повтор -> ready |
|
||||||
|
| нехватка памяти в процессе обработки (RLIMIT_AS) | failed/resource_limit, worker жив | memory bomb in the imaging process -> failed/resource_limit after attempt 1; worker itself survived | постоянная ошибка, без повторов |
|
||||||
|
| 2 временные ошибки S3 | backoff с jitter, затем ready | 2 x storage_unavailable -> retries after 1.90 s and 3.20 s (equal jitter: 1..2 s, 2..4 s), attempt 3 ready | - |
|
||||||
|
| 5 временных ошибок подряд | failed/retry_exhausted, реестр failed, content 409 | 5 temporary errors -> failed/retry_exhausted (public error_code), content 409, attach 409, listed by ops.py failed (replayable) | replay оператора |
|
||||||
|
| replay failed и повторный replay ready | тот же job_id -> ready; повтор отклонён | ops replay -> same job_id d8ab860c-eb40-4df2-8d98-67aade24f2f2 attempt 6 ready; second replay refused: "job is ready, replay is only for failed jobs" | дубля нет |
|
||||||
|
| невалидное сообщение и неизвестный job_id | reject -> DLQ, dlq-replay не исполняет | schema-invalid message and unknown job_id -> reject -> DLQ (2 message(s)); dlq-replay kept them: ['invalid message is kept for analysis', 'unknown job'] | сообщения сохранены |
|
||||||
|
| 8 заданий в очереди, старт worker | WORKER_CONCURRENCY=2, prefetch 1 | 2 consumers, prefetch_count 1, manual ack; 8 queued 3000x2000 JPEG drained in 2.4 s after worker start, peak processing 2 of 8 (6 samples) | - |
|
||||||
|
<!-- measurements:end -->
|
||||||
|
|
||||||
## Ограничения и следующие шаги
|
## Ограничения и следующие шаги
|
||||||
|
|
||||||
TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза.
|
- Выдерживается: недоступность любого хранилища телеметрии и самого Collector — API и обработка вложений продолжают работать; память ограничена очередями SDK (2048 на сигнал и процесс) и Collector (memory_limiter, очереди фиксированного размера). Переживают рестарт хранилищ (volumes): метрики, логи, трассы, настройки Grafana.
|
||||||
|
- Теряется (и считается): телеметрия, отправленная, пока Collector недоступен (SDK не буферизует на диск); пакеты, у которых истекло окно повторов Collector или очередь переполнена; буфер Collector при его рестарте. Метрики приложения за время недоступности Collector — пропуск на графиках (cumulative-счётчики после восстановления продолжают с верного итога).
|
||||||
|
- Один экземпляр каждого хранилища и Collector на одном Docker-host — нет HA телеметрии; при отказе Prometheus алерты не вычисляются (нужен внешний «мета-мониторинг» — вне учебного стенда).
|
||||||
|
- Метрики очереди считает один dispatcher: если он упал, значения `messenger_queue_*` устаревают (видно по отсутствию свежих точек); алерт `absent()` на них — следующий шаг.
|
||||||
|
- Учебные упрощения: внутренние соединения OTLP и UI без TLS (lab7), Prometheus/Loki/Tempo опубликованы на 127.0.0.1 для тестов и защиты; сэмплирование трасс 100% (для учебной нагрузки; стоимость и баланс сэмплирования — трек 5.1, не заявлен); burn-rate алерты с несколькими окнами — трек 5.2, не заявлен.
|
||||||
|
- Подтверждено измерением: алерты срабатывают и снимаются на контролируемых отказах; диагностика отличает остановленный worker, медленную БД и медленный обработчик; отказ телеметрии не влияет на API. Гипотеза без измерения: накладные расходы телеметрии на задержку и CPU (сравнение `OTEL_SDK_DISABLED=true/false` при одинаковой нагрузке) — трек 5.1.
|
||||||
|
|
||||||
|
## Ответы на вопросы защиты (кратко)
|
||||||
|
|
||||||
|
- **Лог, метрика и trace об одном инциденте.** Метрика отвечает «что и когда, насколько массово» (доля 5xx выросла в 14:02, только на POST messages) — дёшево, агрегатно, для алертов. Трасса — «где именно» в одном запросе (5 с из 5,004 с — `INSERT` в PostgreSQL), через все процессы. Лог — «почему» с деталями события (`QueryCanceled`, request_id, job_id). Связаны через `trace_id`: из алерта → error-трасса → её логи.
|
||||||
|
- **Почему `chat_id` в labels может исчерпать хранилище.** Каждая уникальная комбинация меток — отдельный временной ряд со своим индексом и чанками в памяти. Чатов — неограниченное и растущее число, умноженное на маршруты, коды и buckets гистограммы → миллионы рядов, рост RAM/диска TSDB и медленные запросы. Поэтому в метрики — только ограниченные множества, а ID — в трассы/логи.
|
||||||
|
- **Как очередь переносит причинную связь между процессами.** Producer записывает W3C `traceparent` (trace_id + id своего спана) в сообщение — здесь в строку outbox, затем dispatcher в AMQP header и поле тела; consumer извлекает его и начинает свой спан с этим родителем (или с link). Так trace_id один для HTTP-запроса и фоновой обработки, хотя между ними секунды и разные процессы.
|
||||||
|
- **Почему среднее latency не заменяет p99 и почему нельзя усреднять p99 реплик.** Среднее прячет хвост: 99 запросов по 10 мс и один по 5 с дают среднее 60 мс, а пользователь видит 5 с. Перцентиль — не аддитивная величина: p99 двух реплик с разной нагрузкой нельзя усреднить (реплика с 10 запросами весила бы как реплика с 10 000). Правильно — сложить buckets гистограмм всех реплик (`sum by (le)`) и уже из суммы вычислить `histogram_quantile`.
|
||||||
|
|||||||
@@ -0,0 +1,80 @@
|
|||||||
|
"""Приём вложения (lab5): staging в S3 + задание в PostgreSQL до ответа 202.
|
||||||
|
|
||||||
|
Распределённой транзакции между S3 и PostgreSQL нет, порядок записи выбран так, чтобы у каждого
|
||||||
|
объекта в bucket была строка-указатель в БД, а у каждого 202 — устойчивый исходник и намерение:
|
||||||
|
|
||||||
|
1. INSERT attachments (state=uploading, source_key) и COMMIT — упало: в S3 ничего нет, клиенту 503.
|
||||||
|
2. PUT исходника в staging/<hex> — упало: удаляем объект (PUT мог пройти)
|
||||||
|
и строку, клиенту 503. Если и это не вышло —
|
||||||
|
uploading-строку уберёт janitor после grace.
|
||||||
|
3. ОДНА транзакция: state=queued + attachment_jobs + outbox + NOTIFY, COMMIT
|
||||||
|
— упало/неизвестно: клиенту 503, строка
|
||||||
|
остаётся uploading -> janitor удалит объект
|
||||||
|
и строку. Если COMMIT всё же прошёл, задание
|
||||||
|
есть и будет обработано (повтор клиента даст
|
||||||
|
второе вложение — в контракте нет ключа).
|
||||||
|
4. 202 Attachment(status=queued) — только после шага 3.
|
||||||
|
|
||||||
|
Брокер в этом пути не участвует: сообщение из outbox публикует dispatcher. Поэтому недоступный
|
||||||
|
брокер или упавший worker не мешают приёму, а API никогда не делает resize синхронно.
|
||||||
|
"""
|
||||||
|
import logging
|
||||||
|
from uuid import UUID, uuid4
|
||||||
|
|
||||||
|
import db
|
||||||
|
import settings
|
||||||
|
import storage
|
||||||
|
from errors import unavailable
|
||||||
|
from uploads import ImageInfo, Upload
|
||||||
|
|
||||||
|
log = logging.getLogger("messenger.attachments")
|
||||||
|
|
||||||
|
|
||||||
|
def store(chat_id: UUID, owner_id: UUID, filename: str, upload: Upload, image: ImageInfo) -> dict:
|
||||||
|
attachment_id = uuid4()
|
||||||
|
key = storage.new_key(uuid4().hex)
|
||||||
|
db.create_upload_intent(attachment_id, chat_id, owner_id, filename, image.content_type,
|
||||||
|
upload.size, upload.sha256, image.width, image.height, key)
|
||||||
|
try:
|
||||||
|
storage.put(key, upload.file, upload.size, image.content_type, upload.sha256)
|
||||||
|
except storage.StorageUnavailable:
|
||||||
|
compensate(attachment_id, key)
|
||||||
|
raise unavailable("File storage is temporarily unavailable") from None
|
||||||
|
row = db.enqueue_upload(attachment_id, "api@" + settings.INSTANCE_ID)
|
||||||
|
if row is None:
|
||||||
|
# janitor уже признал загрузку брошенной и удалил объект (возможно только при зависании дольше grace).
|
||||||
|
log.warning("Upload %s was reclaimed before it was queued", attachment_id)
|
||||||
|
raise unavailable()
|
||||||
|
log.info("Attachment %s accepted into staging and queued", attachment_id)
|
||||||
|
return {**row, "variants": []}
|
||||||
|
|
||||||
|
|
||||||
|
def compensate(attachment_id: UUID, key: str) -> None:
|
||||||
|
"""Откат после неудачного PUT. Любая ошибка здесь не страшна: остаток уберёт janitor."""
|
||||||
|
try:
|
||||||
|
storage.delete(key, quick=True)
|
||||||
|
db.delete_upload_intent(attachment_id)
|
||||||
|
log.info("Upload %s rolled back after S3 failure", attachment_id)
|
||||||
|
except Exception as exc: # noqa: BLE001 — откат best-effort, исходную ошибку отдаём клиенту
|
||||||
|
log.warning("Rollback of upload %s postponed to janitor: %s", attachment_id, type(exc).__name__)
|
||||||
|
|
||||||
|
|
||||||
|
def public(row: dict) -> dict:
|
||||||
|
"""Строка БД -> Attachment из контракта (без ключей S3, размеров в пикселях и служебных полей)."""
|
||||||
|
return {
|
||||||
|
"id": row["id"],
|
||||||
|
"chat_id": row["chat_id"],
|
||||||
|
"owner_id": row["owner_id"],
|
||||||
|
"filename": row["filename"],
|
||||||
|
"content_type": row["content_type"],
|
||||||
|
"size_bytes": row["size_bytes"],
|
||||||
|
"sha256": row["sha256"],
|
||||||
|
"status": row["state"],
|
||||||
|
"variants": [
|
||||||
|
{key: v[key] for key in ("name", "content_type", "size_bytes", "width", "height", "sha256")}
|
||||||
|
for v in row.get("variants", [])
|
||||||
|
],
|
||||||
|
"error_code": row["error_code"] if row["state"] == "failed" else None,
|
||||||
|
"created_at": row["created_at"],
|
||||||
|
"updated_at": row["updated_at"],
|
||||||
|
}
|
||||||
@@ -0,0 +1,150 @@
|
|||||||
|
import base64
|
||||||
|
import binascii
|
||||||
|
import hashlib
|
||||||
|
import logging
|
||||||
|
import secrets
|
||||||
|
from dataclasses import dataclass
|
||||||
|
from datetime import datetime, timedelta, timezone
|
||||||
|
from uuid import UUID
|
||||||
|
|
||||||
|
import redis
|
||||||
|
from argon2 import PasswordHasher
|
||||||
|
from argon2.exceptions import InvalidHashError, VerificationError
|
||||||
|
from redis.backoff import NoBackoff
|
||||||
|
from redis.retry import Retry
|
||||||
|
|
||||||
|
import settings
|
||||||
|
import telemetry
|
||||||
|
from errors import BASIC_CHALLENGE, unauthorized, unavailable
|
||||||
|
|
||||||
|
log = logging.getLogger("messenger.auth")
|
||||||
|
|
||||||
|
hasher = PasswordHasher(time_cost=3, memory_cost=12 * 1024, parallelism=1)
|
||||||
|
|
||||||
|
DUMMY_HASH = hasher.hash("there-is-no-such-user")
|
||||||
|
|
||||||
|
PASSWORD_MIN_LENGTH = 12
|
||||||
|
PASSWORD_MAX_LENGTH = 128
|
||||||
|
|
||||||
|
|
||||||
|
def hash_password(password: str) -> str:
|
||||||
|
# Argon2id нагружает CPU обработчика (а не БД): в трассе это отдельный спан.
|
||||||
|
with telemetry.internal_span("argon2id hash"):
|
||||||
|
return hasher.hash(password)
|
||||||
|
|
||||||
|
|
||||||
|
def password_matches(password_hash: str | None, password: str) -> bool:
|
||||||
|
with telemetry.internal_span("argon2id verify"):
|
||||||
|
try:
|
||||||
|
hasher.verify(password_hash or DUMMY_HASH, password)
|
||||||
|
except (VerificationError, InvalidHashError):
|
||||||
|
return False
|
||||||
|
return password_hash is not None
|
||||||
|
|
||||||
|
def parse_basic(header: str | None) -> tuple[str, str] | None:
|
||||||
|
"""'Basic base64(username:password)' -> (username, password) или None.
|
||||||
|
|
||||||
|
Пароль может содержать двоеточие, поэтому делим по первому двоеточию (RFC 7617).
|
||||||
|
"""
|
||||||
|
scheme, _, value = (header or "").strip().partition(" ")
|
||||||
|
if scheme.lower() != "basic" or not value or len(value) > 1024:
|
||||||
|
return None
|
||||||
|
try:
|
||||||
|
decoded = base64.b64decode(value.strip(), validate=True).decode("utf-8")
|
||||||
|
except (binascii.Error, UnicodeDecodeError):
|
||||||
|
return None
|
||||||
|
username, colon, password = decoded.partition(":")
|
||||||
|
if not colon or not username or not username.isascii():
|
||||||
|
return None
|
||||||
|
return username, password
|
||||||
|
|
||||||
|
|
||||||
|
def parse_bearer(header: str | None) -> str | None:
|
||||||
|
scheme, _, value = (header or "").strip().partition(" ")
|
||||||
|
value = value.strip()
|
||||||
|
if scheme.lower() != "bearer" or not value or len(value) > 512 or " " in value:
|
||||||
|
return None
|
||||||
|
return value
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
KEY_PREFIX = "session:"
|
||||||
|
# Атрибуты спанов Redis: система и операция. Ключ (session:<SHA-256 токена>) и значения не пишутся.
|
||||||
|
REDIS_ATTRS = {"db.system.name": "redis", "server.address": "redis"}
|
||||||
|
|
||||||
|
_redis = redis.Redis(
|
||||||
|
connection_pool=redis.BlockingConnectionPool.from_url(
|
||||||
|
settings.REDIS_URL or "redis://localhost:6379/0",
|
||||||
|
max_connections=settings.REDIS_POOL_MAX,
|
||||||
|
timeout=settings.REDIS_TIMEOUT_MS / 1000,
|
||||||
|
socket_timeout=settings.REDIS_TIMEOUT_MS / 1000,
|
||||||
|
socket_connect_timeout=settings.REDIS_TIMEOUT_MS / 1000,
|
||||||
|
retry=Retry(NoBackoff(), 0),
|
||||||
|
decode_responses=True,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class Session:
|
||||||
|
user_id: UUID
|
||||||
|
key: str
|
||||||
|
|
||||||
|
|
||||||
|
def session_key(token: str) -> str:
|
||||||
|
return KEY_PREFIX + hashlib.sha256(token.encode("utf-8")).hexdigest()
|
||||||
|
|
||||||
|
|
||||||
|
def create_session(user_id: UUID) -> tuple[str, datetime]:
|
||||||
|
"""Новая сессия: случайный токен и абсолютное время истечения."""
|
||||||
|
token = secrets.token_urlsafe(32)
|
||||||
|
expires_at = datetime.now(timezone.utc) + timedelta(seconds=settings.SESSION_TTL_SECONDS)
|
||||||
|
try:
|
||||||
|
with telemetry.client_span("redis SET", REDIS_ATTRS | {"db.operation.name": "SET"}):
|
||||||
|
_redis.set(session_key(token), str(user_id), pxat=int(expires_at.timestamp() * 1000))
|
||||||
|
except redis.RedisError as exc:
|
||||||
|
log.warning("Redis is unavailable on login: %s", type(exc).__name__,
|
||||||
|
extra={"event": "redis.unavailable", "error.type": type(exc).__name__})
|
||||||
|
raise unavailable() from None
|
||||||
|
return token, expires_at
|
||||||
|
|
||||||
|
|
||||||
|
def check_bearer(header: str | None) -> Session:
|
||||||
|
token = parse_bearer(header)
|
||||||
|
if token is None:
|
||||||
|
raise unauthorized("Bearer token is required")
|
||||||
|
key = session_key(token)
|
||||||
|
try:
|
||||||
|
with telemetry.client_span("redis GET", REDIS_ATTRS | {"db.operation.name": "GET"}):
|
||||||
|
user_id = _redis.get(key)
|
||||||
|
except redis.RedisError as exc:
|
||||||
|
log.warning("Redis is unavailable: %s", type(exc).__name__,
|
||||||
|
extra={"event": "redis.unavailable", "error.type": type(exc).__name__})
|
||||||
|
raise unavailable() from None
|
||||||
|
if user_id is None:
|
||||||
|
raise unauthorized("Session is invalid or expired")
|
||||||
|
return Session(UUID(user_id), key)
|
||||||
|
|
||||||
|
|
||||||
|
def delete_session(session: Session) -> None:
|
||||||
|
try:
|
||||||
|
with telemetry.client_span("redis DEL", REDIS_ATTRS | {"db.operation.name": "DEL"}):
|
||||||
|
_redis.delete(session.key)
|
||||||
|
except redis.RedisError:
|
||||||
|
raise unavailable() from None
|
||||||
|
|
||||||
|
|
||||||
|
def ping_redis() -> None:
|
||||||
|
try:
|
||||||
|
_redis.ping()
|
||||||
|
except redis.RedisError:
|
||||||
|
raise unavailable() from None
|
||||||
|
|
||||||
|
|
||||||
|
def login_failed():
|
||||||
|
return unauthorized("Invalid username or password", BASIC_CHALLENGE)
|
||||||
|
|
||||||
|
|
||||||
|
def close() -> None:
|
||||||
|
_redis.close()
|
||||||
|
_redis.connection_pool.disconnect()
|
||||||
@@ -0,0 +1,104 @@
|
|||||||
|
"""RabbitMQ: подключение, проверка топологии, публикация с publisher confirms.
|
||||||
|
|
||||||
|
Топология объявлена декларативно в definitions (secrets/rabbitmq_definitions.json, генерирует
|
||||||
|
scripts/init_secrets.py) и загружается брокером при каждом старте. У сервисов нет права configure:
|
||||||
|
они только проверяют её пассивным declare и не могут создать/удалить очередь.
|
||||||
|
|
||||||
|
exchange attachments (direct, durable)
|
||||||
|
--attachment.process.v1--> queue attachments.process.v1 (quorum, durable)
|
||||||
|
x-delivery-limit=20, dead-letter -> attachments.dlx (стратегия at-least-once),
|
||||||
|
x-max-length=100000 + x-overflow=reject-publish (переполнение -> nack издателю, не потеря)
|
||||||
|
exchange attachments.dlx (direct, durable)
|
||||||
|
--attachment.process.v1.dlq--> queue attachments.process.v1.dlq (classic, durable: просмотр в ops.py
|
||||||
|
через basic_get + nack не расходует delivery-limit)
|
||||||
|
|
||||||
|
Сообщения persistent (delivery_mode=2). Quorum queue пишет их в журнал Raft на диск (fsync) до
|
||||||
|
подтверждения издателю; при рестарте брокера с volume сообщения и неподтверждённые доставки сохраняются.
|
||||||
|
"""
|
||||||
|
import json
|
||||||
|
import logging
|
||||||
|
|
||||||
|
import pika
|
||||||
|
from pika.exceptions import AMQPError
|
||||||
|
|
||||||
|
import pipeline
|
||||||
|
import settings
|
||||||
|
|
||||||
|
log = logging.getLogger("messenger.broker")
|
||||||
|
logging.getLogger("pika").setLevel(logging.WARNING) # pika пишет INFO на каждое соединение/канал
|
||||||
|
|
||||||
|
BrokerError = (AMQPError, OSError)
|
||||||
|
|
||||||
|
|
||||||
|
def parameters(url: str | None = None) -> pika.URLParameters:
|
||||||
|
params = pika.URLParameters(url or settings.BROKER_URL)
|
||||||
|
params.heartbeat = settings.BROKER_HEARTBEAT_SECONDS
|
||||||
|
params.blocked_connection_timeout = 30 # брокер в memory/disk alarm блокирует издателя: не ждать вечно
|
||||||
|
params.socket_timeout = 5
|
||||||
|
params.connection_attempts = 1
|
||||||
|
params.client_properties = {"connection_name": settings.INSTANCE_ID}
|
||||||
|
return params
|
||||||
|
|
||||||
|
|
||||||
|
def connect(url: str | None = None) -> pika.BlockingConnection:
|
||||||
|
return pika.BlockingConnection(parameters(url))
|
||||||
|
|
||||||
|
|
||||||
|
def check_topology(channel) -> dict:
|
||||||
|
"""Пассивная проверка: exchange и очереди существуют. Возвращает число сообщений в очередях."""
|
||||||
|
channel.exchange_declare(pipeline.EXCHANGE, passive=True)
|
||||||
|
main = channel.queue_declare(pipeline.QUEUE, passive=True).method
|
||||||
|
dlq = channel.queue_declare(pipeline.DLQ, passive=True).method
|
||||||
|
return {"queue": main.message_count, "consumers": main.consumer_count, "dlq": dlq.message_count}
|
||||||
|
|
||||||
|
|
||||||
|
def properties(message: dict) -> pika.BasicProperties:
|
||||||
|
# W3C Trace Context в native headers AMQP (основной carrier для worker) и в теле (поле схемы).
|
||||||
|
headers = {key: message[key] for key in ("traceparent", "tracestate") if key in message}
|
||||||
|
return pika.BasicProperties(
|
||||||
|
delivery_mode=pika.DeliveryMode.Persistent,
|
||||||
|
content_type="application/json",
|
||||||
|
message_id=f"{message['job_id']}:{message['attempt']}",
|
||||||
|
type=message["type"],
|
||||||
|
headers=headers or None,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class Publisher:
|
||||||
|
"""Канал в режиме confirm: basic_publish возвращает управление только после ack брокера.
|
||||||
|
|
||||||
|
mandatory=True: если сообщение некуда положить (нет привязки), брокер вернёт его -> UnroutableError.
|
||||||
|
Nack (например, очередь переполнена и x-overflow=reject-publish) -> NackError. В обоих случаях
|
||||||
|
строка outbox остаётся неопубликованной и будет отправлена позже.
|
||||||
|
"""
|
||||||
|
|
||||||
|
def __init__(self):
|
||||||
|
self.connection = None
|
||||||
|
self.channel = None
|
||||||
|
|
||||||
|
def open(self):
|
||||||
|
if self.channel is None or self.channel.is_closed or self.connection.is_closed:
|
||||||
|
self.close()
|
||||||
|
self.connection = connect()
|
||||||
|
self.channel = self.connection.channel()
|
||||||
|
self.channel.confirm_delivery()
|
||||||
|
check_topology(self.channel)
|
||||||
|
log.info("connected to broker, publisher confirms enabled", extra={"event": "broker.connected"})
|
||||||
|
return self.channel
|
||||||
|
|
||||||
|
def publish(self, message: dict, exchange: str = pipeline.EXCHANGE, routing_key: str = pipeline.ROUTING_KEY):
|
||||||
|
body = json.dumps(message, separators=(",", ":")).encode("utf-8")
|
||||||
|
self.open().basic_publish(exchange, routing_key, body, properties(message), mandatory=True)
|
||||||
|
|
||||||
|
def pump(self):
|
||||||
|
if self.connection is not None and self.connection.is_open:
|
||||||
|
self.connection.process_data_events(time_limit=0) # heartbeat в паузах
|
||||||
|
|
||||||
|
def close(self):
|
||||||
|
try:
|
||||||
|
if self.connection is not None and self.connection.is_open:
|
||||||
|
self.connection.close()
|
||||||
|
except BrokerError:
|
||||||
|
pass
|
||||||
|
self.connection = None
|
||||||
|
self.channel = None
|
||||||
+494
@@ -0,0 +1,494 @@
|
|||||||
|
name: messenger-lab6
|
||||||
|
|
||||||
|
x-app-build: &app-build
|
||||||
|
context: .
|
||||||
|
args:
|
||||||
|
APP_VERSION: ${APP_VERSION:-dev}
|
||||||
|
|
||||||
|
# Общие настройки фоновой обработки (worker и dispatcher); значения по умолчанию — из settings.py.
|
||||||
|
x-pipeline-env: &pipeline-env
|
||||||
|
PIPELINE_VERSION: ${PIPELINE_VERSION:-1}
|
||||||
|
WORKER_CONCURRENCY: ${WORKER_CONCURRENCY:-2}
|
||||||
|
WORKER_PREFETCH: ${WORKER_PREFETCH:-1}
|
||||||
|
JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5}
|
||||||
|
JOB_TIMEOUT_SECONDS: ${JOB_TIMEOUT_SECONDS:-60}
|
||||||
|
JOB_LEASE_SECONDS: ${JOB_LEASE_SECONDS:-15}
|
||||||
|
JOB_BACKOFF_BASE_SECONDS: ${JOB_BACKOFF_BASE_SECONDS:-2}
|
||||||
|
JOB_BACKOFF_MAX_SECONDS: ${JOB_BACKOFF_MAX_SECONDS:-60}
|
||||||
|
STAGING_RETENTION_SECONDS: ${STAGING_RETENTION_SECONDS:-86400}
|
||||||
|
# Учебный стенд: точки внедрения сбоев включены (срабатывают только по файлу в volume faults,
|
||||||
|
# доступном лишь worker/dispatcher). В эксплуатации FAULT_INJECTION=0.
|
||||||
|
FAULT_INJECTION: ${FAULT_INJECTION:-1}
|
||||||
|
FAULTS_DIR: /faults
|
||||||
|
|
||||||
|
# lab6: OpenTelemetry SDK всех процессов приложения -> OTel Collector (OTLP/HTTP, внутренняя сеть telemetry).
|
||||||
|
# Имена переменных — стандартные OTEL_* (их читает SDK); service.name задаётся в коде процесса и дублируется
|
||||||
|
# в OTEL_SERVICE_NAME у каждого сервиса. OTEL_SDK_DISABLED=true в .env выключает телеметрию целиком.
|
||||||
|
x-otel-env: &otel-env
|
||||||
|
OTEL_EXPORTER_OTLP_ENDPOINT: http://otel-collector:4318
|
||||||
|
OTEL_EXPORTER_OTLP_PROTOCOL: http/protobuf
|
||||||
|
OTEL_EXPORTER_OTLP_TIMEOUT: "5" # секунды (Python SDK): экспорт не держит память дольше
|
||||||
|
OTEL_RESOURCE_ATTRIBUTES: deployment.environment.name=lab
|
||||||
|
OTEL_METRIC_EXPORT_INTERVAL: "10000" # мс
|
||||||
|
OTEL_BSP_MAX_QUEUE_SIZE: "2048" # спанов в памяти процесса, лишние отбрасываются и считаются
|
||||||
|
OTEL_BLRP_MAX_QUEUE_SIZE: "2048" # записей логов, так же
|
||||||
|
OTEL_PYTHON_SDK_INTERNAL_METRICS_ENABLED: "true" # otel.sdk.processor.*: очереди и потери SDK
|
||||||
|
OTEL_SDK_DISABLED: ${OTEL_SDK_DISABLED:-false}
|
||||||
|
|
||||||
|
x-otel-depends: &otel-depends
|
||||||
|
# Порядок старта, но не жёсткая зависимость: без Collector приложение работает, телеметрия теряется.
|
||||||
|
condition: service_started
|
||||||
|
required: false
|
||||||
|
|
||||||
|
x-alive-check: &alive-check
|
||||||
|
# Фоновые процессы без HTTP: жив, если обновлял файл не позже 30 с назад.
|
||||||
|
test: ["CMD", "python", "-c", "import os, sys, time; sys.exit(time.time() - os.stat('/tmp/alive').st_mtime > 30)"]
|
||||||
|
interval: 10s
|
||||||
|
timeout: 3s
|
||||||
|
retries: 3
|
||||||
|
start_period: 20s
|
||||||
|
start_interval: 1s
|
||||||
|
|
||||||
|
x-s3-api-secrets: &s3-api-secrets
|
||||||
|
S3_ENDPOINT: ${S3_ENDPOINT:-http://s3:8333}
|
||||||
|
S3_REGION: ${S3_REGION:-us-east-1}
|
||||||
|
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
|
||||||
|
S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id
|
||||||
|
S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key
|
||||||
|
|
||||||
|
services:
|
||||||
|
migrate:
|
||||||
|
build: *app-build
|
||||||
|
command: ["python", "migrate.py"]
|
||||||
|
environment:
|
||||||
|
DATABASE_URL_FILE: /run/secrets/database_url
|
||||||
|
secrets: [database_url]
|
||||||
|
networks: [data]
|
||||||
|
depends_on:
|
||||||
|
postgres: {condition: service_healthy}
|
||||||
|
healthcheck:
|
||||||
|
disable: true
|
||||||
|
restart: "no"
|
||||||
|
deploy:
|
||||||
|
resources:
|
||||||
|
limits: {cpus: "0.50", memory: 128M}
|
||||||
|
|
||||||
|
# S3-совместимое хранилище SeaweedFS (master + volume + filer + S3 в одном процессе): байты вложений.
|
||||||
|
# Только во внутренней сети data, порты не публикуются. Ключи и их права — в секрете s3_config
|
||||||
|
# (генерирует scripts/init_secrets.py): admin-ключ для bootstrap и ключ API с правами только на свой bucket.
|
||||||
|
# Анонимного доступа нет: identity "anonymous" в конфиге не объявлена.
|
||||||
|
s3:
|
||||||
|
image: chrislusf/seaweedfs:3.80
|
||||||
|
entrypoint: ["/usr/bin/weed"]
|
||||||
|
command: ["server", "-dir=/data", "-ip=127.0.0.1", "-ip.bind=0.0.0.0",
|
||||||
|
"-master.volumeSizeLimitMB=1024", "-volume.max=0",
|
||||||
|
"-s3", "-s3.port=8333", "-s3.config=/run/secrets/s3_config"]
|
||||||
|
secrets: [s3_config]
|
||||||
|
volumes: ["s3data:/data"]
|
||||||
|
networks: [data]
|
||||||
|
healthcheck:
|
||||||
|
# S3-порт отвечает по HTTP (анонимный запрос получает 403 — это и есть «жив и закрыт»)
|
||||||
|
test: ["CMD-SHELL", "wget -q -S -O /dev/null http://127.0.0.1:8333/ 2>&1 | grep -q 'HTTP/'"]
|
||||||
|
interval: 5s
|
||||||
|
timeout: 3s
|
||||||
|
retries: 24
|
||||||
|
start_period: 15s
|
||||||
|
start_interval: 1s
|
||||||
|
restart: unless-stopped
|
||||||
|
stop_grace_period: 30s
|
||||||
|
deploy:
|
||||||
|
resources:
|
||||||
|
limits: {cpus: "1.00", memory: 512M}
|
||||||
|
|
||||||
|
# Одноразовый идемпотентный bootstrap (s3_init.py, образ приложения): приватный bucket + проверки.
|
||||||
|
s3-init:
|
||||||
|
build: *app-build
|
||||||
|
command: ["python", "s3_init.py"]
|
||||||
|
environment:
|
||||||
|
S3_ENDPOINT: ${S3_ENDPOINT:-http://s3:8333}
|
||||||
|
S3_REGION: ${S3_REGION:-us-east-1}
|
||||||
|
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
|
||||||
|
S3_ADMIN_ACCESS_KEY_ID_FILE: /run/secrets/s3_root_user
|
||||||
|
S3_ADMIN_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_root_password
|
||||||
|
S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id
|
||||||
|
S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key
|
||||||
|
secrets: [s3_root_user, s3_root_password, s3_access_key_id, s3_secret_access_key]
|
||||||
|
networks: [data]
|
||||||
|
depends_on:
|
||||||
|
s3: {condition: service_healthy}
|
||||||
|
healthcheck:
|
||||||
|
disable: true
|
||||||
|
restart: "no"
|
||||||
|
deploy:
|
||||||
|
resources:
|
||||||
|
limits: {cpus: "0.25", memory: 128M}
|
||||||
|
|
||||||
|
api:
|
||||||
|
build: *app-build
|
||||||
|
env_file: .env
|
||||||
|
environment:
|
||||||
|
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600}
|
||||||
|
BROKER_URL_FILE: "" # у API нет доступа к брокеру: задание уходит через outbox в PostgreSQL
|
||||||
|
OTEL_SERVICE_NAME: messenger-api
|
||||||
|
<<: [*s3-api-secrets, *otel-env]
|
||||||
|
secrets: [database_url, cache_url, s3_access_key_id, s3_secret_access_key]
|
||||||
|
networks: [edge, data, telemetry]
|
||||||
|
depends_on:
|
||||||
|
migrate: {condition: service_completed_successfully}
|
||||||
|
s3-init: {condition: service_completed_successfully}
|
||||||
|
redis: {condition: service_healthy}
|
||||||
|
otel-collector: *otel-depends
|
||||||
|
healthcheck:
|
||||||
|
test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=2)"]
|
||||||
|
interval: 5s
|
||||||
|
timeout: 3s
|
||||||
|
retries: 3
|
||||||
|
start_period: 30s
|
||||||
|
start_interval: 1s
|
||||||
|
restart: unless-stopped
|
||||||
|
stop_grace_period: 15s
|
||||||
|
deploy:
|
||||||
|
replicas: ${API_REPLICAS:-2}
|
||||||
|
resources:
|
||||||
|
limits: {cpus: "1.00", memory: 512M}
|
||||||
|
|
||||||
|
# Очистка брошенных загрузок и сверка БД <-> S3 (janitor.py). Тот же образ и тот же ограниченный S3-ключ.
|
||||||
|
janitor:
|
||||||
|
build: *app-build
|
||||||
|
command: ["python", "janitor.py"]
|
||||||
|
env_file: .env
|
||||||
|
environment:
|
||||||
|
CACHE_URL_FILE: "" # janitor не ходит в Redis
|
||||||
|
BROKER_URL_FILE: "" # и в брокер
|
||||||
|
OTEL_SERVICE_NAME: messenger-janitor
|
||||||
|
<<: [*s3-api-secrets, *otel-env]
|
||||||
|
secrets: [database_url, s3_access_key_id, s3_secret_access_key]
|
||||||
|
networks: [data, telemetry]
|
||||||
|
depends_on:
|
||||||
|
migrate: {condition: service_completed_successfully}
|
||||||
|
s3-init: {condition: service_completed_successfully}
|
||||||
|
otel-collector: *otel-depends
|
||||||
|
healthcheck: *alive-check
|
||||||
|
restart: unless-stopped
|
||||||
|
stop_grace_period: 10s
|
||||||
|
deploy:
|
||||||
|
resources:
|
||||||
|
limits: {cpus: "0.25", memory: 192M}
|
||||||
|
|
||||||
|
# Брокер RabbitMQ: durable quorum-очередь с журналом на volume. Топология/пользователи/права —
|
||||||
|
# в секрете rabbitmq_definitions (scripts/init_secrets.py), импорт при каждом старте.
|
||||||
|
# hostname фиксирован: имя узла rabbit@rabbitmq определяет каталог данных в volume.
|
||||||
|
rabbitmq:
|
||||||
|
image: rabbitmq:4.1.4-alpine
|
||||||
|
hostname: rabbitmq
|
||||||
|
volumes:
|
||||||
|
- ./infra/rabbitmq.conf:/etc/rabbitmq/conf.d/90-messenger.conf:ro
|
||||||
|
- rabbitmqdata:/var/lib/rabbitmq
|
||||||
|
secrets: [rabbitmq_definitions]
|
||||||
|
networks: [data]
|
||||||
|
healthcheck:
|
||||||
|
# Узел запущен, и vhost с очередями загружен (а не только «порт открыт»).
|
||||||
|
test: ["CMD-SHELL", "rabbitmq-diagnostics -q check_running && rabbitmqctl -q list_queues -p messenger name | grep -q '^attachments.process.v1$$'"]
|
||||||
|
interval: 5s
|
||||||
|
timeout: 10s
|
||||||
|
retries: 24
|
||||||
|
start_period: 20s
|
||||||
|
start_interval: 2s
|
||||||
|
restart: unless-stopped
|
||||||
|
stop_grace_period: 30s
|
||||||
|
deploy:
|
||||||
|
resources:
|
||||||
|
limits: {cpus: "1.00", memory: 512M}
|
||||||
|
|
||||||
|
# Dispatcher: публикует transactional outbox в RabbitMQ (publisher confirms), восстанавливает
|
||||||
|
# просроченные аренды processing и сверяет queued без сообщения. У API доступа к брокеру нет.
|
||||||
|
dispatcher:
|
||||||
|
build: *app-build
|
||||||
|
command: ["python", "dispatcher.py"]
|
||||||
|
environment:
|
||||||
|
DATABASE_URL_FILE: /run/secrets/database_url
|
||||||
|
BROKER_URL_FILE: /run/secrets/broker_url_dispatcher
|
||||||
|
OTEL_SERVICE_NAME: messenger-dispatcher
|
||||||
|
<<: [*pipeline-env, *otel-env]
|
||||||
|
secrets: [database_url, broker_url_dispatcher]
|
||||||
|
volumes: ["faults:/faults"]
|
||||||
|
networks: [data, telemetry]
|
||||||
|
depends_on:
|
||||||
|
migrate: {condition: service_completed_successfully}
|
||||||
|
rabbitmq: {condition: service_healthy}
|
||||||
|
otel-collector: *otel-depends
|
||||||
|
healthcheck: *alive-check
|
||||||
|
restart: unless-stopped
|
||||||
|
stop_grace_period: 10s
|
||||||
|
deploy:
|
||||||
|
resources:
|
||||||
|
limits: {cpus: "0.25", memory: 128M}
|
||||||
|
|
||||||
|
# Worker: потребитель очереди, отдельный процесс и сервис. Обработка изображения — в дочернем
|
||||||
|
# процессе imaging.py с RLIMIT_AS=WORKER_MEMORY_LIMIT_MB, RLIMIT_CPU и таймаутом JOB_TIMEOUT_SECONDS.
|
||||||
|
# Лимит контейнера >= WORKER_CONCURRENCY x WORKER_MEMORY_LIMIT_MB + сам worker.
|
||||||
|
worker:
|
||||||
|
build: *app-build
|
||||||
|
command: ["python", "worker.py"]
|
||||||
|
environment:
|
||||||
|
DATABASE_URL_FILE: /run/secrets/database_url
|
||||||
|
BROKER_URL_FILE: /run/secrets/broker_url_worker
|
||||||
|
MAX_UPLOAD_BYTES: ${MAX_UPLOAD_BYTES:-10485760}
|
||||||
|
MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS:-40000000}
|
||||||
|
WORKER_MEMORY_LIMIT_MB: ${WORKER_MEMORY_LIMIT_MB:-512}
|
||||||
|
OTEL_SERVICE_NAME: messenger-worker
|
||||||
|
<<: [*pipeline-env, *s3-api-secrets, *otel-env]
|
||||||
|
secrets: [database_url, broker_url_worker, s3_access_key_id, s3_secret_access_key]
|
||||||
|
volumes: ["faults:/faults"]
|
||||||
|
tmpfs:
|
||||||
|
- /tmp:size=256m,mode=1777
|
||||||
|
networks: [data, telemetry]
|
||||||
|
depends_on:
|
||||||
|
migrate: {condition: service_completed_successfully}
|
||||||
|
s3-init: {condition: service_completed_successfully}
|
||||||
|
rabbitmq: {condition: service_healthy}
|
||||||
|
otel-collector: *otel-depends
|
||||||
|
healthcheck: *alive-check
|
||||||
|
restart: unless-stopped
|
||||||
|
# SIGTERM: потоки доделывают текущее задание (ограничено JOB_TIMEOUT_SECONDS) и выходят;
|
||||||
|
# если не успели — SIGKILL, ack не отправлен, брокер доставит задание снова.
|
||||||
|
stop_grace_period: 70s
|
||||||
|
deploy:
|
||||||
|
resources:
|
||||||
|
limits: {cpus: "2.00", memory: 1536M}
|
||||||
|
|
||||||
|
lb:
|
||||||
|
image: haproxy:3.2.25-alpine
|
||||||
|
ports:
|
||||||
|
- "127.0.0.1:8080:8080"
|
||||||
|
- "127.0.0.1:8404:8404"
|
||||||
|
volumes:
|
||||||
|
- ./infra/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro
|
||||||
|
networks: [edge]
|
||||||
|
depends_on:
|
||||||
|
api: {condition: service_healthy}
|
||||||
|
healthcheck:
|
||||||
|
test: ["CMD", "wget", "-q", "-T", "2", "-O", "/dev/null", "http://127.0.0.1:8404/lb-health"]
|
||||||
|
interval: 5s
|
||||||
|
timeout: 3s
|
||||||
|
retries: 3
|
||||||
|
start_period: 10s
|
||||||
|
start_interval: 1s
|
||||||
|
restart: unless-stopped
|
||||||
|
stop_grace_period: 15s
|
||||||
|
deploy:
|
||||||
|
resources:
|
||||||
|
limits: {cpus: "0.50", memory: 128M}
|
||||||
|
|
||||||
|
postgres:
|
||||||
|
image: postgres:17.11-alpine
|
||||||
|
environment:
|
||||||
|
POSTGRES_USER: messenger
|
||||||
|
POSTGRES_DB: messenger
|
||||||
|
POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password
|
||||||
|
command: ["postgres", "-c", "fsync=on", "-c", "synchronous_commit=on"]
|
||||||
|
secrets: [postgres_password]
|
||||||
|
volumes: ["pgdata:/var/lib/postgresql/data"]
|
||||||
|
networks: [data]
|
||||||
|
healthcheck:
|
||||||
|
test: ["CMD-SHELL", "pg_isready -h 127.0.0.1 -U messenger -d messenger"]
|
||||||
|
interval: 5s
|
||||||
|
timeout: 3s
|
||||||
|
retries: 12
|
||||||
|
start_period: 10s
|
||||||
|
restart: unless-stopped
|
||||||
|
stop_grace_period: 30s
|
||||||
|
deploy:
|
||||||
|
resources:
|
||||||
|
limits: {cpus: "1.00", memory: 512M}
|
||||||
|
|
||||||
|
redis:
|
||||||
|
image: redis:7.4.11-alpine
|
||||||
|
command: ["redis-server", "/usr/local/etc/redis/redis.conf"]
|
||||||
|
volumes:
|
||||||
|
- ./infra/redis.conf:/usr/local/etc/redis/redis.conf:ro
|
||||||
|
- redisdata:/data
|
||||||
|
secrets: [redis_auth]
|
||||||
|
networks: [data]
|
||||||
|
healthcheck:
|
||||||
|
test: ["CMD-SHELL", "REDISCLI_AUTH=\"$$(sed -n 's/^requirepass //p' /run/secrets/redis_auth)\" redis-cli ping | grep -q PONG"]
|
||||||
|
interval: 5s
|
||||||
|
timeout: 3s
|
||||||
|
retries: 12
|
||||||
|
restart: unless-stopped
|
||||||
|
deploy:
|
||||||
|
resources:
|
||||||
|
limits: {cpus: "0.50", memory: 128M}
|
||||||
|
|
||||||
|
# ---------------- lab6: наблюдаемость ----------------
|
||||||
|
# OTel Collector: приём OTLP от приложения, memory_limiter + batch, ограниченные очереди exporter,
|
||||||
|
# отправка в Tempo / Loki / Prometheus. Не хранилище. Порты не публикуются (только сеть telemetry).
|
||||||
|
# Образ distroless (без shell), поэтому без healthcheck: здоровье видно по otelcol_* в Prometheus.
|
||||||
|
otel-collector:
|
||||||
|
image: otel/opentelemetry-collector-contrib:0.164.0
|
||||||
|
command: ["--config=/etc/otelcol-contrib/config.yaml"]
|
||||||
|
volumes:
|
||||||
|
- ./infra/otel/collector.yaml:/etc/otelcol-contrib/config.yaml:ro
|
||||||
|
networks: [telemetry]
|
||||||
|
restart: unless-stopped
|
||||||
|
stop_grace_period: 10s
|
||||||
|
deploy:
|
||||||
|
resources:
|
||||||
|
# heap ограничивает memory_limiter (200 MiB); сверху — runtime Go и отображённый бинарник contrib.
|
||||||
|
limits: {cpus: "0.50", memory: 384M}
|
||||||
|
|
||||||
|
# Одноразовый bootstrap прав на volumes Loki и Tempo (оба процесса работают как uid 10001, не root).
|
||||||
|
telemetry-init:
|
||||||
|
image: busybox:1.37.0
|
||||||
|
user: root
|
||||||
|
command: ["sh", "-c", "chown -R 10001:10001 /loki /tempo"]
|
||||||
|
volumes:
|
||||||
|
- lokidata:/loki
|
||||||
|
- tempodata:/tempo
|
||||||
|
network_mode: none
|
||||||
|
restart: "no"
|
||||||
|
deploy:
|
||||||
|
resources:
|
||||||
|
limits: {cpus: "0.25", memory: 32M}
|
||||||
|
|
||||||
|
# Prometheus: метрики (OTLP receiver для Collector + pull служебных метрик), recording/alert rules.
|
||||||
|
# 127.0.0.1:9090 — только для evidence-скриптов и /alerts на защите.
|
||||||
|
prometheus:
|
||||||
|
image: prom/prometheus:v3.15.1
|
||||||
|
command:
|
||||||
|
- --config.file=/etc/prometheus/prometheus.yml
|
||||||
|
- --storage.tsdb.path=/prometheus
|
||||||
|
- --storage.tsdb.retention.time=7d
|
||||||
|
- --storage.tsdb.retention.size=1GB
|
||||||
|
- --web.enable-otlp-receiver
|
||||||
|
volumes:
|
||||||
|
- ./infra/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro
|
||||||
|
- ./infra/prometheus/rules:/etc/prometheus/rules:ro
|
||||||
|
- ./infra/prometheus/tests:/etc/prometheus/tests:ro # promtool test rules /etc/prometheus/tests/alerts_test.yml
|
||||||
|
- promdata:/prometheus
|
||||||
|
ports:
|
||||||
|
- "127.0.0.1:9090:9090"
|
||||||
|
networks: [telemetry, ops]
|
||||||
|
healthcheck:
|
||||||
|
test: ["CMD", "wget", "-q", "-T", "2", "-O", "/dev/null", "http://127.0.0.1:9090/-/ready"]
|
||||||
|
interval: 5s
|
||||||
|
timeout: 3s
|
||||||
|
retries: 24
|
||||||
|
start_period: 10s
|
||||||
|
restart: unless-stopped
|
||||||
|
stop_grace_period: 30s
|
||||||
|
deploy:
|
||||||
|
resources:
|
||||||
|
limits: {cpus: "1.00", memory: 512M}
|
||||||
|
|
||||||
|
# Loki: логи (OTLP от Collector), файловое хранилище в volume lokidata. 127.0.0.1:3100 — для тестов.
|
||||||
|
loki:
|
||||||
|
image: grafana/loki:3.7.9
|
||||||
|
command: ["-config.file=/etc/loki/loki.yaml"]
|
||||||
|
volumes:
|
||||||
|
- ./infra/loki/loki.yaml:/etc/loki/loki.yaml:ro
|
||||||
|
- lokidata:/loki
|
||||||
|
ports:
|
||||||
|
- "127.0.0.1:3100:3100"
|
||||||
|
networks: [telemetry, ops]
|
||||||
|
depends_on:
|
||||||
|
telemetry-init: {condition: service_completed_successfully}
|
||||||
|
restart: unless-stopped
|
||||||
|
stop_grace_period: 30s
|
||||||
|
deploy:
|
||||||
|
resources:
|
||||||
|
limits: {cpus: "1.00", memory: 512M}
|
||||||
|
|
||||||
|
# Tempo: трассы (OTLP gRPC от Collector), блоки в volume tempodata. 127.0.0.1:3200 — для тестов.
|
||||||
|
tempo:
|
||||||
|
image: grafana/tempo:3.1.1
|
||||||
|
command: ["-config.file=/etc/tempo/tempo.yaml"]
|
||||||
|
volumes:
|
||||||
|
- ./infra/tempo/tempo.yaml:/etc/tempo/tempo.yaml:ro
|
||||||
|
- tempodata:/var/tempo
|
||||||
|
ports:
|
||||||
|
- "127.0.0.1:3200:3200"
|
||||||
|
networks: [telemetry, ops]
|
||||||
|
depends_on:
|
||||||
|
telemetry-init: {condition: service_completed_successfully}
|
||||||
|
restart: unless-stopped
|
||||||
|
stop_grace_period: 30s
|
||||||
|
deploy:
|
||||||
|
resources:
|
||||||
|
limits: {cpus: "1.00", memory: 512M}
|
||||||
|
|
||||||
|
# Grafana: интерфейс и запросы к трём хранилищам. Источники данных и dashboard — provisioning из Git
|
||||||
|
# (infra/grafana), ручного набора нет. Вход: admin / пароль из secrets/grafana_admin_password.
|
||||||
|
grafana:
|
||||||
|
image: grafana/grafana:12.4.14
|
||||||
|
environment:
|
||||||
|
GF_SECURITY_ADMIN_USER: admin
|
||||||
|
GF_SECURITY_ADMIN_PASSWORD__FILE: /run/secrets/grafana_admin_password
|
||||||
|
GF_USERS_ALLOW_SIGN_UP: "false"
|
||||||
|
GF_AUTH_ANONYMOUS_ENABLED: "false"
|
||||||
|
GF_ANALYTICS_REPORTING_ENABLED: "false"
|
||||||
|
GF_ANALYTICS_CHECK_FOR_UPDATES: "false"
|
||||||
|
GF_ANALYTICS_CHECK_FOR_PLUGIN_UPDATES: "false"
|
||||||
|
GF_NEWS_NEWS_FEED_ENABLED: "false"
|
||||||
|
GF_PLUGINS_PREINSTALL_DISABLED: "true" # стенд не скачивает плагины из интернета при старте
|
||||||
|
GF_DASHBOARDS_DEFAULT_HOME_DASHBOARD_PATH: /etc/grafana/dashboards/messenger.json
|
||||||
|
secrets: [grafana_admin_password]
|
||||||
|
volumes:
|
||||||
|
- ./infra/grafana/provisioning:/etc/grafana/provisioning:ro
|
||||||
|
- ./infra/grafana/dashboards:/etc/grafana/dashboards:ro
|
||||||
|
- grafanadata:/var/lib/grafana
|
||||||
|
ports:
|
||||||
|
- "127.0.0.1:3000:3000"
|
||||||
|
networks: [telemetry, ops]
|
||||||
|
depends_on:
|
||||||
|
prometheus: {condition: service_healthy}
|
||||||
|
loki: {condition: service_started}
|
||||||
|
tempo: {condition: service_started}
|
||||||
|
healthcheck:
|
||||||
|
# wget есть в образе на Alpine, curl — в варианте на Ubuntu: проверка работает в обоих.
|
||||||
|
test: ["CMD-SHELL", "wget -q -T 2 -O /dev/null http://127.0.0.1:3000/api/health || curl -fsS -m 2 -o /dev/null http://127.0.0.1:3000/api/health"]
|
||||||
|
interval: 5s
|
||||||
|
timeout: 3s
|
||||||
|
retries: 24
|
||||||
|
start_period: 20s
|
||||||
|
restart: unless-stopped
|
||||||
|
stop_grace_period: 15s
|
||||||
|
deploy:
|
||||||
|
resources:
|
||||||
|
limits: {cpus: "0.50", memory: 384M}
|
||||||
|
|
||||||
|
networks:
|
||||||
|
edge: {}
|
||||||
|
data:
|
||||||
|
internal: true
|
||||||
|
# lab6: приложение -> Collector -> хранилища -> Grafana. Внутренняя: наружу ничего не публикуется.
|
||||||
|
telemetry:
|
||||||
|
internal: true
|
||||||
|
# Только для публикации UI/API хранилищ на 127.0.0.1 (internal-сеть не может публиковать порты).
|
||||||
|
ops: {}
|
||||||
|
|
||||||
|
volumes:
|
||||||
|
pgdata: {}
|
||||||
|
redisdata: {}
|
||||||
|
s3data: {}
|
||||||
|
rabbitmqdata: {}
|
||||||
|
faults: {}
|
||||||
|
promdata: {}
|
||||||
|
lokidata: {}
|
||||||
|
tempodata: {}
|
||||||
|
grafanadata: {}
|
||||||
|
|
||||||
|
secrets:
|
||||||
|
postgres_password: {file: ./secrets/postgres_password}
|
||||||
|
database_url: {file: ./secrets/database_url}
|
||||||
|
cache_url: {file: ./secrets/cache_url}
|
||||||
|
redis_auth: {file: ./secrets/redis_auth}
|
||||||
|
s3_root_user: {file: ./secrets/s3_root_user}
|
||||||
|
s3_root_password: {file: ./secrets/s3_root_password}
|
||||||
|
s3_access_key_id: {file: ./secrets/s3_access_key_id}
|
||||||
|
s3_secret_access_key: {file: ./secrets/s3_secret_access_key}
|
||||||
|
s3_config: {file: ./secrets/s3_config.json}
|
||||||
|
broker_url_dispatcher: {file: ./secrets/broker_url_dispatcher}
|
||||||
|
broker_url_worker: {file: ./secrets/broker_url_worker}
|
||||||
|
rabbitmq_definitions: {file: ./secrets/rabbitmq_definitions.json}
|
||||||
|
grafana_admin_password: {file: ./secrets/grafana_admin_password}
|
||||||
@@ -0,0 +1,452 @@
|
|||||||
|
import logging
|
||||||
|
import time
|
||||||
|
from contextlib import contextmanager
|
||||||
|
from datetime import datetime
|
||||||
|
from uuid import UUID, uuid4
|
||||||
|
|
||||||
|
import psycopg
|
||||||
|
from psycopg import errors as pg_errors
|
||||||
|
from psycopg.rows import dict_row
|
||||||
|
from psycopg_pool import ConnectionPool, PoolTimeout
|
||||||
|
|
||||||
|
import pipeline
|
||||||
|
import settings
|
||||||
|
import telemetry
|
||||||
|
from errors import bad_request, conflict, not_found, unavailable
|
||||||
|
|
||||||
|
log = logging.getLogger("messenger.db")
|
||||||
|
|
||||||
|
|
||||||
|
def check_connection(conn) -> None:
|
||||||
|
"""Проверка соединения при выдаче из пула (пустой запрос) — без спана: это не работа запроса."""
|
||||||
|
from opentelemetry.instrumentation.utils import suppress_instrumentation
|
||||||
|
with suppress_instrumentation():
|
||||||
|
ConnectionPool.check_connection(conn)
|
||||||
|
|
||||||
|
|
||||||
|
pool = ConnectionPool(
|
||||||
|
conninfo=settings.DATABASE_URL,
|
||||||
|
min_size=settings.DB_POOL_MIN,
|
||||||
|
max_size=settings.DB_POOL_MAX,
|
||||||
|
timeout=settings.DB_POOL_WAIT_SECONDS,
|
||||||
|
kwargs={
|
||||||
|
"connect_timeout": settings.DB_CONNECT_TIMEOUT_SECONDS,
|
||||||
|
"row_factory": dict_row,
|
||||||
|
"options": f"-c statement_timeout={settings.DB_STATEMENT_TIMEOUT_MS}"
|
||||||
|
" -c idle_in_transaction_session_timeout=10000 -c TimeZone=UTC",
|
||||||
|
},
|
||||||
|
check=check_connection,
|
||||||
|
open=False,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
# lab6: насыщение пула соединений API (contracts/observability.md, «DB pool used/waiting»).
|
||||||
|
POOL_NAME = {"db.client.connection.pool.name": "messenger-api"}
|
||||||
|
POOL_WAIT = telemetry.meter.create_histogram(
|
||||||
|
"db.client.connection.wait_time", unit="s", description="Time to obtain a connection from the pool")
|
||||||
|
|
||||||
|
|
||||||
|
def _pool_stats():
|
||||||
|
try:
|
||||||
|
return pool.get_stats()
|
||||||
|
except Exception: # noqa: BLE001 — пул ещё не открыт/закрыт: метрик нет
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def _observe_connections(_options):
|
||||||
|
from opentelemetry.metrics import Observation
|
||||||
|
stats = _pool_stats()
|
||||||
|
if not stats:
|
||||||
|
return []
|
||||||
|
size, available = stats.get("pool_size", 0), stats.get("pool_available", 0)
|
||||||
|
return [Observation(size - available, POOL_NAME | {"db.client.connection.state": "used"}),
|
||||||
|
Observation(available, POOL_NAME | {"db.client.connection.state": "idle"})]
|
||||||
|
|
||||||
|
|
||||||
|
def _observe_stat(key):
|
||||||
|
def callback(_options):
|
||||||
|
from opentelemetry.metrics import Observation
|
||||||
|
stats = _pool_stats()
|
||||||
|
return [Observation(stats.get(key, 0), POOL_NAME)] if stats else []
|
||||||
|
return callback
|
||||||
|
|
||||||
|
|
||||||
|
telemetry.meter.create_observable_up_down_counter(
|
||||||
|
"db.client.connection.count", [_observe_connections], unit="{connection}",
|
||||||
|
description="Connections in the pool by state (used/idle)")
|
||||||
|
telemetry.meter.create_observable_up_down_counter(
|
||||||
|
"db.client.connection.max", [_observe_stat("pool_max")], unit="{connection}",
|
||||||
|
description="Maximum pool size (DB_POOL_MAX)")
|
||||||
|
telemetry.meter.create_observable_up_down_counter(
|
||||||
|
"db.client.connection.pending_requests", [_observe_stat("requests_waiting")], unit="{request}",
|
||||||
|
description="Requests waiting for a free connection now")
|
||||||
|
telemetry.meter.create_observable_counter(
|
||||||
|
"db.client.connection.timeouts", [_observe_stat("requests_errors")], unit="{timeout}",
|
||||||
|
description="Requests that did not get a connection within DB_POOL_WAIT_SECONDS")
|
||||||
|
|
||||||
|
|
||||||
|
@contextmanager
|
||||||
|
def connection(wait_seconds: float | None = None):
|
||||||
|
try:
|
||||||
|
started = time.perf_counter()
|
||||||
|
try:
|
||||||
|
with pool.connection(timeout=wait_seconds) as conn:
|
||||||
|
POOL_WAIT.record(time.perf_counter() - started, POOL_NAME)
|
||||||
|
yield conn
|
||||||
|
except PoolTimeout:
|
||||||
|
POOL_WAIT.record(time.perf_counter() - started, POOL_NAME)
|
||||||
|
raise
|
||||||
|
except (psycopg.OperationalError, PoolTimeout) as exc:
|
||||||
|
log.warning("PostgreSQL is unavailable: %s", type(exc).__name__,
|
||||||
|
extra={"event": "db.unavailable", "error.type": type(exc).__name__})
|
||||||
|
raise unavailable() from None
|
||||||
|
except psycopg.DataError as exc:
|
||||||
|
log.warning("PostgreSQL rejected a value: %s", type(exc).__name__,
|
||||||
|
extra={"event": "db.rejected_value", "error.type": type(exc).__name__})
|
||||||
|
raise bad_request("Invalid request data") from None
|
||||||
|
|
||||||
|
|
||||||
|
TEXT_FIELDS = ("display_name", "title", "text", "filename")
|
||||||
|
|
||||||
|
|
||||||
|
def to_db(value: str) -> bytes:
|
||||||
|
return value.encode("utf-8")
|
||||||
|
|
||||||
|
|
||||||
|
def from_db(row: dict | None) -> dict | None:
|
||||||
|
"""Строка из БД -> dict для API: байты текстовых полей обратно в str."""
|
||||||
|
if row is None:
|
||||||
|
return None
|
||||||
|
return {key: bytes(value).decode("utf-8") if key in TEXT_FIELDS else value for key, value in row.items()}
|
||||||
|
|
||||||
|
|
||||||
|
def ping() -> None:
|
||||||
|
with connection(wait_seconds=1) as conn:
|
||||||
|
conn.execute("SELECT 1")
|
||||||
|
|
||||||
|
|
||||||
|
MIGRATIONS_LOCK = 20260202
|
||||||
|
|
||||||
|
|
||||||
|
def migration_files() -> list:
|
||||||
|
return sorted(settings.MIGRATIONS_DIR.glob("*.sql"))
|
||||||
|
|
||||||
|
|
||||||
|
def run_migrations() -> list[str]:
|
||||||
|
files = migration_files()
|
||||||
|
applied = []
|
||||||
|
with psycopg.connect(settings.DATABASE_URL, autocommit=True,
|
||||||
|
connect_timeout=settings.DB_CONNECT_TIMEOUT_SECONDS) as conn:
|
||||||
|
conn.execute("SELECT pg_advisory_lock(%s)", (MIGRATIONS_LOCK,))
|
||||||
|
try:
|
||||||
|
conn.execute("""CREATE TABLE IF NOT EXISTS schema_migrations (
|
||||||
|
version text PRIMARY KEY,
|
||||||
|
applied_at timestamptz NOT NULL DEFAULT now())""")
|
||||||
|
done = {row[0] for row in conn.execute("SELECT version FROM schema_migrations")}
|
||||||
|
for path in files:
|
||||||
|
if path.name in done:
|
||||||
|
continue
|
||||||
|
with conn.transaction():
|
||||||
|
conn.execute(path.read_text(encoding="utf-8"))
|
||||||
|
conn.execute("INSERT INTO schema_migrations (version) VALUES (%s)", (path.name,))
|
||||||
|
applied.append(path.name)
|
||||||
|
finally:
|
||||||
|
conn.execute("SELECT pg_advisory_unlock(%s)", (MIGRATIONS_LOCK,))
|
||||||
|
return applied
|
||||||
|
|
||||||
|
|
||||||
|
def wait_and_migrate(max_wait_seconds: int = 60) -> None:
|
||||||
|
deadline = time.monotonic() + max_wait_seconds
|
||||||
|
while True:
|
||||||
|
try:
|
||||||
|
applied = run_migrations()
|
||||||
|
log.info("Migrations applied: %s", ", ".join(applied) if applied else "none (schema is up to date)")
|
||||||
|
return
|
||||||
|
except psycopg.OperationalError:
|
||||||
|
if time.monotonic() > deadline:
|
||||||
|
raise
|
||||||
|
log.warning("PostgreSQL is not ready yet, retry in 1 s")
|
||||||
|
time.sleep(1)
|
||||||
|
|
||||||
|
|
||||||
|
def missing_migrations() -> list[str]:
|
||||||
|
expected = [path.name for path in migration_files()]
|
||||||
|
with psycopg.connect(settings.DATABASE_URL, autocommit=True,
|
||||||
|
connect_timeout=settings.DB_CONNECT_TIMEOUT_SECONDS) as conn:
|
||||||
|
try:
|
||||||
|
done = {row[0] for row in conn.execute("SELECT version FROM schema_migrations")}
|
||||||
|
except pg_errors.UndefinedTable:
|
||||||
|
return expected
|
||||||
|
return [name for name in expected if name not in done]
|
||||||
|
|
||||||
|
|
||||||
|
def wait_for_schema(max_wait_seconds: int) -> None:
|
||||||
|
deadline = time.monotonic() + max_wait_seconds
|
||||||
|
while True:
|
||||||
|
try:
|
||||||
|
missing = missing_migrations()
|
||||||
|
except psycopg.OperationalError:
|
||||||
|
missing = None
|
||||||
|
if missing == []:
|
||||||
|
log.info("Schema is up to date: %s", ", ".join(path.name for path in migration_files()))
|
||||||
|
return
|
||||||
|
if time.monotonic() > deadline:
|
||||||
|
raise RuntimeError(f"Database schema is not ready, missing migrations: {missing}")
|
||||||
|
log.warning("Waiting for the migrate service: %s", "PostgreSQL is not ready" if missing is None
|
||||||
|
else "missing " + ", ".join(missing))
|
||||||
|
time.sleep(1)
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
USER_COLUMNS = "id, username, display_name, created_at"
|
||||||
|
|
||||||
|
|
||||||
|
def create_user(username: str, display_name: str, password_hash: str) -> dict:
|
||||||
|
try:
|
||||||
|
with connection() as conn:
|
||||||
|
return from_db(conn.execute(
|
||||||
|
f"""INSERT INTO users (id, username, display_name, password_hash)
|
||||||
|
VALUES (%s, %s, %s, %s) RETURNING {USER_COLUMNS}""",
|
||||||
|
(uuid4(), username, to_db(display_name), password_hash),
|
||||||
|
).fetchone())
|
||||||
|
except pg_errors.UniqueViolation:
|
||||||
|
raise conflict("Username already exists") from None
|
||||||
|
|
||||||
|
|
||||||
|
def get_user(user_id: UUID) -> dict | None:
|
||||||
|
with connection() as conn:
|
||||||
|
return from_db(conn.execute(f"SELECT {USER_COLUMNS} FROM users WHERE id = %s", (user_id,)).fetchone())
|
||||||
|
|
||||||
|
|
||||||
|
def get_user_with_password(username: str) -> dict | None:
|
||||||
|
with connection() as conn:
|
||||||
|
return from_db(conn.execute(
|
||||||
|
f"SELECT {USER_COLUMNS}, password_hash FROM users WHERE username = %s", (username,)
|
||||||
|
).fetchone())
|
||||||
|
|
||||||
|
|
||||||
|
CHAT_COLUMNS = """c.id, c.title, c.created_at,
|
||||||
|
ARRAY(SELECT m.user_id FROM chat_members m WHERE m.chat_id = c.id ORDER BY m.user_id) AS member_ids"""
|
||||||
|
|
||||||
|
|
||||||
|
def create_chat(title: str, member_ids: list[UUID]) -> dict:
|
||||||
|
chat_id = uuid4()
|
||||||
|
members = sorted(member_ids)
|
||||||
|
try:
|
||||||
|
with connection() as conn:
|
||||||
|
with conn.transaction():
|
||||||
|
row = conn.execute(
|
||||||
|
"INSERT INTO chats (id, title) VALUES (%s, %s) RETURNING id, title, created_at",
|
||||||
|
(chat_id, to_db(title)),
|
||||||
|
).fetchone()
|
||||||
|
with conn.cursor() as cur:
|
||||||
|
cur.executemany(
|
||||||
|
"INSERT INTO chat_members (chat_id, user_id) VALUES (%s, %s)",
|
||||||
|
[(chat_id, member_id) for member_id in members],
|
||||||
|
)
|
||||||
|
except pg_errors.ForeignKeyViolation:
|
||||||
|
raise bad_request("Unknown member") from None
|
||||||
|
return {**from_db(row), "member_ids": members}
|
||||||
|
|
||||||
|
|
||||||
|
def get_chat(chat_id: UUID, user_id: UUID) -> dict:
|
||||||
|
with connection() as conn:
|
||||||
|
chat = conn.execute(
|
||||||
|
f"""SELECT {CHAT_COLUMNS} FROM chats c
|
||||||
|
JOIN chat_members me ON me.chat_id = c.id AND me.user_id = %s
|
||||||
|
WHERE c.id = %s""",
|
||||||
|
(user_id, chat_id),
|
||||||
|
).fetchone()
|
||||||
|
if chat is None:
|
||||||
|
raise not_found("Chat not found")
|
||||||
|
return from_db(chat)
|
||||||
|
|
||||||
|
|
||||||
|
def list_chats(user_id: UUID, after: tuple[datetime, UUID] | None, limit: int) -> list[dict]:
|
||||||
|
"""Чаты пользователя после курсора. Возвращает до limit+1 строк: лишняя значит «есть ещё»."""
|
||||||
|
where, params = "", [user_id]
|
||||||
|
if after is not None:
|
||||||
|
where = "WHERE (c.created_at, c.id) > (%s, %s)"
|
||||||
|
params += list(after)
|
||||||
|
with connection() as conn:
|
||||||
|
rows = conn.execute(
|
||||||
|
f"""SELECT {CHAT_COLUMNS} FROM chats c
|
||||||
|
JOIN chat_members me ON me.chat_id = c.id AND me.user_id = %s
|
||||||
|
{where}
|
||||||
|
ORDER BY c.created_at, c.id
|
||||||
|
LIMIT %s""",
|
||||||
|
(*params, limit + 1),
|
||||||
|
).fetchall()
|
||||||
|
return [from_db(row) for row in rows]
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
MESSAGE_COLUMNS = """m.id, m.chat_id, m.sender_id, m.text, m.created_at,
|
||||||
|
ARRAY(SELECT a.attachment_id FROM message_attachments a
|
||||||
|
WHERE a.message_id = m.id ORDER BY a.position) AS attachment_ids"""
|
||||||
|
|
||||||
|
|
||||||
|
def is_member(conn, chat_id: UUID, user_id: UUID) -> bool:
|
||||||
|
return conn.execute(
|
||||||
|
"SELECT 1 FROM chat_members WHERE chat_id = %s AND user_id = %s", (chat_id, user_id)
|
||||||
|
).fetchone() is not None
|
||||||
|
|
||||||
|
|
||||||
|
def check_member(chat_id: UUID, user_id: UUID) -> None:
|
||||||
|
with connection() as conn:
|
||||||
|
if not is_member(conn, chat_id, user_id):
|
||||||
|
raise not_found("Chat not found")
|
||||||
|
|
||||||
|
|
||||||
|
def add_message(chat_id: UUID, sender_id: UUID, text: str, attachment_ids: list[UUID],
|
||||||
|
idempotency_key: str, request_hash: bytes) -> dict:
|
||||||
|
try:
|
||||||
|
with connection() as conn:
|
||||||
|
with conn.transaction():
|
||||||
|
if not is_member(conn, chat_id, sender_id): # проверяется и при повторе: чужой чат — 404
|
||||||
|
raise not_found("Chat not found")
|
||||||
|
if attachment_ids:
|
||||||
|
# Только принятые вложения ЭТОГО чата (lab5: queued/processing/ready можно, failed — 409).
|
||||||
|
# FOR KEY SHARE: строку не удалят до конца транзакции, а worker может менять её состояние.
|
||||||
|
found = conn.execute(
|
||||||
|
"""SELECT id, state FROM attachments
|
||||||
|
WHERE id = ANY(%s) AND chat_id = %s AND state <> 'uploading' FOR KEY SHARE""",
|
||||||
|
(attachment_ids, chat_id),
|
||||||
|
).fetchall()
|
||||||
|
if len(found) != len(attachment_ids):
|
||||||
|
raise not_found("Attachment not found")
|
||||||
|
failed = any(row["state"] == "failed" for row in found)
|
||||||
|
else:
|
||||||
|
failed = False
|
||||||
|
saved_query = f"""SELECT {MESSAGE_COLUMNS}, m.request_hash FROM messages m
|
||||||
|
WHERE m.chat_id = %s AND m.sender_id = %s AND m.idempotency_key = %s"""
|
||||||
|
if failed:
|
||||||
|
# Повтор уже созданного сообщения возвращает прежний 201, даже если вложение
|
||||||
|
# с тех пор стало failed. Новое сообщение с failed-вложением — 409.
|
||||||
|
saved = conn.execute(saved_query, (chat_id, sender_id, idempotency_key)).fetchone()
|
||||||
|
if saved is None:
|
||||||
|
raise conflict("Failed attachment cannot be attached")
|
||||||
|
created = None
|
||||||
|
else:
|
||||||
|
created = conn.execute(
|
||||||
|
"""INSERT INTO messages (id, chat_id, sender_id, text, idempotency_key, request_hash)
|
||||||
|
VALUES (%s, %s, %s, %s, %s, %s)
|
||||||
|
ON CONFLICT (chat_id, sender_id, idempotency_key) DO NOTHING
|
||||||
|
RETURNING id, chat_id, sender_id, text, created_at""",
|
||||||
|
(uuid4(), chat_id, sender_id, to_db(text), idempotency_key, request_hash),
|
||||||
|
).fetchone()
|
||||||
|
if created is None:
|
||||||
|
saved = conn.execute(saved_query, (chat_id, sender_id, idempotency_key)).fetchone()
|
||||||
|
if created is not None:
|
||||||
|
if attachment_ids:
|
||||||
|
with conn.cursor() as cur:
|
||||||
|
cur.executemany(
|
||||||
|
"""INSERT INTO message_attachments (message_id, chat_id, attachment_id, position)
|
||||||
|
VALUES (%s, %s, %s, %s)""",
|
||||||
|
[(created["id"], chat_id, aid, n) for n, aid in enumerate(attachment_ids)],
|
||||||
|
)
|
||||||
|
return {**from_db(created), "attachment_ids": list(attachment_ids)}
|
||||||
|
except pg_errors.ForeignKeyViolation:
|
||||||
|
raise not_found("Chat not found") from None
|
||||||
|
if saved is None: # не должно случиться: конфликт был, значит строка есть; повтор безопасен
|
||||||
|
raise unavailable()
|
||||||
|
if bytes(saved.pop("request_hash")) != request_hash:
|
||||||
|
raise conflict("Idempotency-Key was already used with a different request")
|
||||||
|
return from_db(saved)
|
||||||
|
|
||||||
|
|
||||||
|
def list_messages(chat_id: UUID, user_id: UUID, after: tuple[datetime, UUID] | None, limit: int) -> list[dict]:
|
||||||
|
"""История чата после курсора (keyset-пагинация), до limit+1 строк."""
|
||||||
|
where, params = "m.chat_id = %s", [chat_id]
|
||||||
|
if after is not None:
|
||||||
|
where += " AND (m.created_at, m.id) > (%s, %s)"
|
||||||
|
params += list(after)
|
||||||
|
with connection() as conn:
|
||||||
|
if not is_member(conn, chat_id, user_id):
|
||||||
|
raise not_found("Chat not found")
|
||||||
|
rows = conn.execute(
|
||||||
|
f"""SELECT {MESSAGE_COLUMNS} FROM messages m
|
||||||
|
WHERE {where}
|
||||||
|
ORDER BY m.created_at, m.id
|
||||||
|
LIMIT %s""",
|
||||||
|
(*params, limit + 1),
|
||||||
|
).fetchall()
|
||||||
|
return [from_db(row) for row in rows]
|
||||||
|
|
||||||
|
|
||||||
|
# ---------- вложения ----------
|
||||||
|
|
||||||
|
ATTACHMENT_COLUMNS = ("id, chat_id, owner_id, filename, content_type, size_bytes, sha256, state, error_code, "
|
||||||
|
"pipeline_version, created_at, updated_at, object_key")
|
||||||
|
VISIBLE_STATES = ("queued", "processing", "ready", "failed")
|
||||||
|
|
||||||
|
|
||||||
|
def create_upload_intent(attachment_id: UUID, chat_id: UUID, owner_id: UUID, filename: str, content_type: str,
|
||||||
|
size_bytes: int, sha256: str, width: int, height: int, source_key: str) -> None:
|
||||||
|
"""Шаг 1 из 3: строка-намерение (state=uploading) фиксируется ДО записи в staging.
|
||||||
|
|
||||||
|
Поэтому у любого объекта, который API кладёт в bucket, уже есть строка в БД: janitor найдёт
|
||||||
|
и удалит его, если загрузка не дошла до queued (обрыв, падение реплики, отказ S3 или БД).
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
with connection() as conn:
|
||||||
|
conn.execute(
|
||||||
|
"""INSERT INTO attachments (id, chat_id, owner_id, filename, content_type, size_bytes, sha256,
|
||||||
|
width, height, source_key, state)
|
||||||
|
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, 'uploading')""",
|
||||||
|
(attachment_id, chat_id, owner_id, to_db(filename), content_type, size_bytes, sha256,
|
||||||
|
width, height, source_key),
|
||||||
|
)
|
||||||
|
except pg_errors.ForeignKeyViolation:
|
||||||
|
raise not_found("Chat not found") from None
|
||||||
|
|
||||||
|
|
||||||
|
def enqueue_upload(attachment_id: UUID, actor: str) -> dict | None:
|
||||||
|
"""Шаг 3 из 3: исходник уже в staging -> queued + задание + outbox ОДНОЙ транзакцией.
|
||||||
|
|
||||||
|
После этого COMMIT принятая работа не может потеряться: исходник в S3, намерение в PostgreSQL,
|
||||||
|
сообщение для брокера в outbox. None, если строку успел убрать janitor (зависание дольше grace).
|
||||||
|
"""
|
||||||
|
with connection() as conn:
|
||||||
|
with conn.transaction():
|
||||||
|
row = conn.execute(
|
||||||
|
f"""UPDATE attachments SET state = 'queued', updated_at = now()
|
||||||
|
WHERE id = %s AND state = 'uploading'
|
||||||
|
RETURNING {ATTACHMENT_COLUMNS}""",
|
||||||
|
(attachment_id,),
|
||||||
|
).fetchone()
|
||||||
|
if row is None:
|
||||||
|
return None
|
||||||
|
pipeline.enqueue_new(conn, attachment_id, actor)
|
||||||
|
return from_db(row)
|
||||||
|
|
||||||
|
|
||||||
|
def delete_upload_intent(attachment_id: UUID) -> None:
|
||||||
|
with connection(wait_seconds=1) as conn:
|
||||||
|
conn.execute("DELETE FROM attachments WHERE id = %s AND state = 'uploading'", (attachment_id,))
|
||||||
|
|
||||||
|
|
||||||
|
def get_attachment(attachment_id: UUID, user_id: UUID) -> dict:
|
||||||
|
"""Вложение (queued/processing/ready/failed), если user_id — участник его чата. Иначе 404.
|
||||||
|
|
||||||
|
Возвращает строку и варианты текущей версии pipeline (они есть только у ready).
|
||||||
|
"""
|
||||||
|
with connection() as conn:
|
||||||
|
row = conn.execute(
|
||||||
|
f"""SELECT {", ".join("a." + c.strip() for c in ATTACHMENT_COLUMNS.split(","))}
|
||||||
|
FROM attachments a
|
||||||
|
JOIN chat_members me ON me.chat_id = a.chat_id AND me.user_id = %s
|
||||||
|
WHERE a.id = %s AND a.state = ANY(%s)""",
|
||||||
|
(user_id, attachment_id, list(VISIBLE_STATES)),
|
||||||
|
).fetchone()
|
||||||
|
if row is None:
|
||||||
|
raise not_found("Attachment not found")
|
||||||
|
variants = []
|
||||||
|
if row["state"] == "ready":
|
||||||
|
variants = conn.execute(
|
||||||
|
"""SELECT name, content_type, size_bytes, width, height, sha256, object_key
|
||||||
|
FROM attachment_variants WHERE attachment_id = %s AND pipeline_version = %s
|
||||||
|
ORDER BY array_position(ARRAY['thumbnail', 'crop', 'optimized'], name)""",
|
||||||
|
(attachment_id, row["pipeline_version"]),
|
||||||
|
).fetchall()
|
||||||
|
return {**from_db(row), "variants": variants}
|
||||||
+260
@@ -0,0 +1,260 @@
|
|||||||
|
"""Dispatcher: публикует outbox в RabbitMQ и восстанавливает зависшие задания.
|
||||||
|
|
||||||
|
Почему отдельный процесс, а не publish прямо в API: API фиксирует задание и строку outbox одной
|
||||||
|
транзакцией PostgreSQL и сразу отвечает 202. Между COMMIT и публикацией нет окна, в котором
|
||||||
|
намерение может потеряться: если брокер недоступен, dispatcher упал или процесс убит, строка outbox
|
||||||
|
остаётся неопубликованной и будет отправлена после восстановления. Поэтому у API нет ни адреса,
|
||||||
|
ни ключей брокера, и недоступность брокера не мешает приёму файлов.
|
||||||
|
|
||||||
|
Цикл:
|
||||||
|
1. publish_due: строки outbox с published_at IS NULL и available_at <= now() (под FOR UPDATE SKIP
|
||||||
|
LOCKED) -> basic_publish с publisher confirm -> published_at = now() -> COMMIT.
|
||||||
|
Падение после confirm, но до COMMIT -> строка будет опубликована ещё раз: дубль сообщения,
|
||||||
|
который worker распознаёт по состоянию job (at-least-once, а не exactly-once).
|
||||||
|
2. Раз в DISPATCHER_POLL_SECONDS: processing с истёкшей арендой -> queued (+ outbox) или failed.
|
||||||
|
3. Раз в 30 с: сверка «queued без сообщения в полёте» (pipeline.reconcile_lost_messages).
|
||||||
|
Просыпается по NOTIFY outbox (сразу после COMMIT в API) или по таймеру.
|
||||||
|
"""
|
||||||
|
import logging
|
||||||
|
import signal
|
||||||
|
import threading
|
||||||
|
import time
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
import psycopg
|
||||||
|
from opentelemetry.trace import SpanKind
|
||||||
|
from psycopg.rows import dict_row
|
||||||
|
|
||||||
|
import settings
|
||||||
|
import telemetry
|
||||||
|
|
||||||
|
telemetry.setup("messenger-dispatcher")
|
||||||
|
|
||||||
|
import broker # noqa: E402
|
||||||
|
import faults # noqa: E402
|
||||||
|
import pipeline # noqa: E402
|
||||||
|
|
||||||
|
log = logging.getLogger("messenger.dispatcher")
|
||||||
|
|
||||||
|
# lab6: глубина очереди и возраст старейшего задания. Источник истины — PostgreSQL (attachment_jobs,
|
||||||
|
# outbox), глубина в брокере — пассивный declare очереди. Считает ОДИН процесс (dispatcher), поэтому
|
||||||
|
# реплики API не дублируют значения. Опрос раз в STATS_SECONDS в основном цикле, callback метрик
|
||||||
|
# только читает последний снимок (не ходит в БД из потока экспорта).
|
||||||
|
STATS_SECONDS = 5
|
||||||
|
STATS = {"at": 0.0, "db": None, "broker": None}
|
||||||
|
PUBLISHED = telemetry.meter.create_counter(
|
||||||
|
"messenger.messages.published", unit="{message}",
|
||||||
|
description="Outbox rows published to the broker (outcome: ok/error)")
|
||||||
|
QUEUE_ATTRS = {"messaging.destination.name": pipeline.QUEUE}
|
||||||
|
|
||||||
|
|
||||||
|
def _fresh(kind):
|
||||||
|
return STATS[kind] if STATS[kind] is not None and time.monotonic() - STATS["at"] < 60 else None
|
||||||
|
|
||||||
|
|
||||||
|
def _observe(kind, build):
|
||||||
|
def callback(_options):
|
||||||
|
from opentelemetry.metrics import Observation
|
||||||
|
data = _fresh(kind)
|
||||||
|
return [Observation(value, attrs) for value, attrs in build(data)] if data else []
|
||||||
|
return callback
|
||||||
|
|
||||||
|
|
||||||
|
telemetry.meter.create_observable_gauge(
|
||||||
|
"messenger.queue.depth", [_observe("broker", lambda d: [
|
||||||
|
(d["queue"], QUEUE_ATTRS), (d["dlq"], {"messaging.destination.name": pipeline.DLQ})])],
|
||||||
|
unit="{message}", description="Messages ready in the broker queue (RabbitMQ passive declare)")
|
||||||
|
telemetry.meter.create_observable_gauge(
|
||||||
|
"messenger.queue.consumers", [_observe("broker", lambda d: [(d["consumers"], QUEUE_ATTRS)])],
|
||||||
|
unit="{consumer}", description="Consumers subscribed to the queue")
|
||||||
|
telemetry.meter.create_observable_gauge(
|
||||||
|
"messenger.jobs.backlog", [_observe("db", lambda d: [
|
||||||
|
(d["queued_due"], {"status": "queued"}), (d["queued_delayed"], {"status": "delayed"}),
|
||||||
|
(d["processing"], {"status": "processing"})])],
|
||||||
|
unit="{job}", description="Jobs not finished yet: queued (due), delayed (backoff), processing")
|
||||||
|
telemetry.meter.create_observable_gauge(
|
||||||
|
"messenger.queue.oldest_age", [_observe("db", lambda d: [(d["oldest_due_age"], QUEUE_ATTRS)])],
|
||||||
|
unit="s", description="Age of the oldest due queued job (queue lag), 0 when nothing waits")
|
||||||
|
telemetry.meter.create_observable_gauge(
|
||||||
|
"messenger.outbox.pending", [_observe("db", lambda d: [(d["outbox_due"], {})])],
|
||||||
|
unit="{message}", description="Outbox rows due but not yet published")
|
||||||
|
telemetry.meter.create_observable_gauge(
|
||||||
|
"messenger.outbox.oldest_age", [_observe("db", lambda d: [(d["outbox_oldest_age"], {})])],
|
||||||
|
unit="s", description="Age of the oldest due unpublished outbox row (publish lag)")
|
||||||
|
|
||||||
|
|
||||||
|
def collect_stats(conn, publisher) -> None:
|
||||||
|
"""Снимок для метрик очереди. Ошибка брокера не мешает метрикам БД и наоборот."""
|
||||||
|
if time.monotonic() - STATS["at"] < STATS_SECONDS:
|
||||||
|
return
|
||||||
|
db_stats = conn.execute(
|
||||||
|
"""SELECT count(*) FILTER (WHERE status = 'queued' AND available_at <= now()) AS queued_due,
|
||||||
|
count(*) FILTER (WHERE status = 'queued' AND available_at > now()) AS queued_delayed,
|
||||||
|
count(*) FILTER (WHERE status = 'processing') AS processing,
|
||||||
|
coalesce(extract(epoch FROM now() - min(available_at)
|
||||||
|
FILTER (WHERE status = 'queued' AND available_at <= now())), 0)::float8 AS oldest_due_age
|
||||||
|
FROM attachment_jobs WHERE status IN ('queued', 'processing')""").fetchone()
|
||||||
|
outbox = conn.execute(
|
||||||
|
"""SELECT count(*) FILTER (WHERE available_at <= now()) AS outbox_due,
|
||||||
|
coalesce(extract(epoch FROM now() - min(available_at)
|
||||||
|
FILTER (WHERE available_at <= now())), 0)::float8 AS outbox_oldest_age
|
||||||
|
FROM outbox WHERE published_at IS NULL""").fetchone()
|
||||||
|
STATS["db"] = {**db_stats, **outbox}
|
||||||
|
try:
|
||||||
|
STATS["broker"] = broker.check_topology(publisher.open())
|
||||||
|
except broker.BrokerError:
|
||||||
|
STATS["broker"] = None
|
||||||
|
publisher.close()
|
||||||
|
STATS["at"] = time.monotonic()
|
||||||
|
|
||||||
|
STOP = threading.Event()
|
||||||
|
ACTOR = "dispatcher@" + settings.INSTANCE_ID
|
||||||
|
|
||||||
|
|
||||||
|
def connect_db():
|
||||||
|
return psycopg.connect(settings.DATABASE_URL, autocommit=True, row_factory=dict_row,
|
||||||
|
connect_timeout=settings.DB_CONNECT_TIMEOUT_SECONDS,
|
||||||
|
options=f"-c statement_timeout={settings.DB_STATEMENT_TIMEOUT_MS} -c TimeZone=UTC")
|
||||||
|
|
||||||
|
|
||||||
|
def publish_due(conn, publisher: broker.Publisher) -> int:
|
||||||
|
published = 0
|
||||||
|
while not STOP.is_set():
|
||||||
|
with conn.transaction():
|
||||||
|
rows = conn.execute(
|
||||||
|
"""SELECT id, job_id, attempt, payload FROM outbox
|
||||||
|
WHERE published_at IS NULL AND available_at <= now()
|
||||||
|
ORDER BY available_at, id LIMIT %s FOR UPDATE SKIP LOCKED""",
|
||||||
|
(settings.DISPATCHER_BATCH,),
|
||||||
|
).fetchall()
|
||||||
|
if not rows:
|
||||||
|
return published
|
||||||
|
for row in rows:
|
||||||
|
publish_one(conn, publisher, row)
|
||||||
|
published += 1
|
||||||
|
log.info("published %s message(s)", len(rows), extra={"event": "outbox.published", "count": len(rows)})
|
||||||
|
if len(rows) < settings.DISPATCHER_BATCH:
|
||||||
|
return published
|
||||||
|
return published
|
||||||
|
|
||||||
|
|
||||||
|
def publish_one(conn, publisher: broker.Publisher, row: dict) -> None:
|
||||||
|
"""Спан PRODUCER «publish attachments»: ребёнок спана create (API или попытка worker), его контекст
|
||||||
|
уходит в native headers AMQP и в поле traceparent сообщения — worker продолжит ту же трассу."""
|
||||||
|
message = dict(row["payload"])
|
||||||
|
job_id, attempt = message.get("job_id"), message.get("attempt")
|
||||||
|
attributes = {"messaging.system": "rabbitmq", "messaging.operation.type": "send",
|
||||||
|
"messaging.destination.name": pipeline.EXCHANGE,
|
||||||
|
"messaging.rabbitmq.destination.routing_key": pipeline.ROUTING_KEY,
|
||||||
|
"messaging.message.id": f"{job_id}:{attempt}", "messenger.job.id": str(job_id),
|
||||||
|
"messenger.job.attempt": attempt, "messenger.outbox.id": row["id"]}
|
||||||
|
parent = telemetry.context_from(message.get("traceparent"), message.get("tracestate"))
|
||||||
|
with telemetry.tracer.start_as_current_span(f"publish {pipeline.EXCHANGE}", context=parent,
|
||||||
|
kind=SpanKind.PRODUCER, attributes=attributes,
|
||||||
|
record_exception=False, set_status_on_exception=False) as span:
|
||||||
|
carrier = telemetry.inject()
|
||||||
|
if carrier.get("traceparent"):
|
||||||
|
message["traceparent"] = carrier["traceparent"]
|
||||||
|
message.pop("tracestate", None)
|
||||||
|
if carrier.get("tracestate"):
|
||||||
|
message["tracestate"] = carrier["tracestate"]
|
||||||
|
faults.crash("dispatcher_crash_before_publish")
|
||||||
|
try:
|
||||||
|
publisher.publish(message)
|
||||||
|
except broker.BrokerError as exc:
|
||||||
|
telemetry.mark_error(span, type(exc).__name__)
|
||||||
|
PUBLISHED.add(1, {"messaging.destination.name": pipeline.EXCHANGE, "outcome": "error"})
|
||||||
|
log.warning("publish of outbox %s failed: %s", row["id"], type(exc).__name__,
|
||||||
|
extra={"event": "outbox.publish_failed", "job.id": str(job_id),
|
||||||
|
"error.type": type(exc).__name__})
|
||||||
|
publisher.close()
|
||||||
|
raise
|
||||||
|
faults.crash("dispatcher_crash_after_confirm")
|
||||||
|
conn.execute("UPDATE outbox SET published_at = now() WHERE id = %s", (row["id"],))
|
||||||
|
PUBLISHED.add(1, {"messaging.destination.name": pipeline.EXCHANGE, "outcome": "ok"})
|
||||||
|
|
||||||
|
|
||||||
|
def note_failure(conn) -> None:
|
||||||
|
"""Счётчик неудачных публикаций для наблюдения (ops.py status)."""
|
||||||
|
try:
|
||||||
|
conn.execute("""UPDATE outbox SET publish_failures = publish_failures + 1
|
||||||
|
WHERE published_at IS NULL AND available_at <= now()""")
|
||||||
|
except psycopg.Error:
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
def touch_alive() -> None:
|
||||||
|
try:
|
||||||
|
Path(settings.ALIVE_FILE).touch()
|
||||||
|
except OSError:
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
def run() -> None:
|
||||||
|
publisher = broker.Publisher()
|
||||||
|
conn = None
|
||||||
|
last_leases = last_reconcile = 0.0
|
||||||
|
pause = 1.0
|
||||||
|
while not STOP.is_set():
|
||||||
|
try:
|
||||||
|
if conn is None or conn.closed:
|
||||||
|
conn = connect_db()
|
||||||
|
conn.execute("LISTEN outbox")
|
||||||
|
log.info("connected to PostgreSQL, listening for outbox notifications",
|
||||||
|
extra={"event": "db.connected"})
|
||||||
|
touch_alive()
|
||||||
|
publish_due(conn, publisher)
|
||||||
|
collect_stats(conn, publisher)
|
||||||
|
now = time.monotonic()
|
||||||
|
if now - last_leases >= settings.DISPATCHER_POLL_SECONDS:
|
||||||
|
last_leases = now
|
||||||
|
if recovered := pipeline.recover_expired_leases(conn, ACTOR):
|
||||||
|
log.warning("recovered %s job(s) with expired lease", recovered,
|
||||||
|
extra={"event": "lease.recovered", "count": recovered})
|
||||||
|
if now - last_reconcile >= 30:
|
||||||
|
last_reconcile = now
|
||||||
|
if lost := pipeline.reconcile_lost_messages(conn, ACTOR, settings.JOB_RECONCILE_SECONDS):
|
||||||
|
log.warning("re-published %s queued job(s) without a message in flight", lost,
|
||||||
|
extra={"event": "outbox.republished", "count": lost})
|
||||||
|
pause = 1.0
|
||||||
|
publisher.pump()
|
||||||
|
# Ждём NOTIFY (COMMIT нового задания) не дольше интервала опроса.
|
||||||
|
for _ in conn.notifies(timeout=settings.DISPATCHER_POLL_SECONDS, stop_after=1):
|
||||||
|
pass
|
||||||
|
except broker.BrokerError:
|
||||||
|
if conn is not None and not conn.closed:
|
||||||
|
note_failure(conn)
|
||||||
|
log.warning("broker is unavailable, outbox is kept, retry in %.0f s", pause,
|
||||||
|
extra={"event": "broker.unavailable"})
|
||||||
|
STOP.wait(pause)
|
||||||
|
pause = min(pause * 2, 15)
|
||||||
|
except psycopg.Error as exc:
|
||||||
|
log.warning("PostgreSQL error %s, reconnect in %.0f s", type(exc).__name__, pause,
|
||||||
|
extra={"event": "db.unavailable", "error.type": type(exc).__name__})
|
||||||
|
try:
|
||||||
|
if conn is not None:
|
||||||
|
conn.close()
|
||||||
|
except psycopg.Error:
|
||||||
|
pass
|
||||||
|
conn = None
|
||||||
|
STOP.wait(pause)
|
||||||
|
pause = min(pause * 2, 15)
|
||||||
|
publisher.close()
|
||||||
|
if conn is not None:
|
||||||
|
conn.close()
|
||||||
|
log.info("stopped", extra={"event": "service.stopped"})
|
||||||
|
telemetry.shutdown()
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> None:
|
||||||
|
if not settings.DATABASE_URL or not settings.BROKER_URL:
|
||||||
|
raise SystemExit("DATABASE_URL_FILE and BROKER_URL_FILE are required")
|
||||||
|
for sig in (signal.SIGTERM, signal.SIGINT):
|
||||||
|
signal.signal(sig, lambda *_: STOP.set())
|
||||||
|
log.info("dispatcher %s started, fault injection %s", settings.INSTANCE_ID,
|
||||||
|
"ON" if settings.FAULT_INJECTION else "off", extra={"event": "service.started"})
|
||||||
|
run()
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
@@ -0,0 +1,123 @@
|
|||||||
|
# Runbooks алертов (lab6)
|
||||||
|
|
||||||
|
Правила — `infra/prometheus/rules/alerts.yml`, unit-тесты правил — `infra/prometheus/tests/alerts_test.yml`.
|
||||||
|
Состояние алертов: http://127.0.0.1:9090/alerts или Grafana → Alerting → Alert rules (data source rules).
|
||||||
|
Dashboard: Grafana http://127.0.0.1:3000 → Messenger → «Messenger — lab6 observability».
|
||||||
|
Общий порядок: **метрика говорит «что и когда», трасса — «где именно», лог — «почему»**.
|
||||||
|
|
||||||
|
Ссылки на трассу из лога: Grafana → Explore → Loki → запись → поле `trace_id` → «Открыть трассу».
|
||||||
|
Из спана в логи: Tempo → спан → «Logs for this span» (запрос по `trace_id`).
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## MessengerHighErrorRate
|
||||||
|
|
||||||
|
**Что значит.** Больше 5% бизнес-запросов API (без `/health/*`) за минуту закончились 5xx, дольше минуты,
|
||||||
|
при трафике > 0,2 запроса/с. 4xx (неверный пароль, чужой чат) сюда не входят.
|
||||||
|
|
||||||
|
**Влияние.** Пользователи получают 503/500, SLO-1 (99% без 5xx за час) расходует бюджет в 5+ раз быстрее нормы.
|
||||||
|
|
||||||
|
**Найти причину.**
|
||||||
|
1. Какие маршруты и коды: dashboard «RPS по классу статуса» или
|
||||||
|
`sum by (http_route, http_response_status_code) (rate(http_server_request_duration_seconds_count{job="messenger-api", http_response_status_class="5xx"}[1m]))`.
|
||||||
|
503 — недоступна зависимость, 500 — ошибка кода.
|
||||||
|
2. Медленная БД или медленный обработчик:
|
||||||
|
- `histogram_quantile(0.95, sum by (le) (rate(db_client_connection_wait_time_seconds_bucket[1m])))` и
|
||||||
|
`db_client_connection_pending_requests` — запросы ждут соединение из пула → соединения заняты долгими запросами;
|
||||||
|
- Tempo, TraceQL `{ resource.service.name = "messenger-api" && status = error }` → открыть трассу:
|
||||||
|
если почти всё время server-спана занимает дочерний спан `INSERT`/`SELECT` (PostgreSQL) с ошибкой —
|
||||||
|
проблема в БД; если время в `argon2id hash`/`inspect image` или «пустое» время самого server-спана — в обработчике;
|
||||||
|
`redis GET` с ошибкой — Redis; `S3 PutObject` — SeaweedFS.
|
||||||
|
3. Логи API: `{service_name="messenger-api"} | event="db.unavailable"` (поле `error_type`: `QueryCanceled` —
|
||||||
|
statement_timeout 5 с, `PoolTimeout` — нет свободного соединения 3 с, `OperationalError` — БД недоступна);
|
||||||
|
`| event="redis.unavailable"`, `| event="s3.unavailable"`.
|
||||||
|
4. В PostgreSQL (если трассы указывают на БД): `docker compose exec postgres psql -U messenger -d messenger -c
|
||||||
|
"SELECT pid, state, wait_event_type, wait_event, now()-query_start AS age, left(query,60) FROM pg_stat_activity
|
||||||
|
WHERE datname='messenger' ORDER BY age DESC"` — блокировки (`wait_event_type = Lock`) и долгие запросы.
|
||||||
|
|
||||||
|
**Исправить.** Снять блокировку/долгий запрос (`pg_terminate_backend(pid)`), восстановить зависимость
|
||||||
|
(`docker compose start postgres|redis|s3`). Не перезапускать API «на всякий случай»: причина не в нём.
|
||||||
|
|
||||||
|
**Проверить восстановление.** Доля 5xx на dashboard → 0, алерт становится inactive через ~1–2 минуты
|
||||||
|
(окно rate 1m). Пример разбора: REPORT.md, инцидент 2.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## MessengerQueueLagGrowing
|
||||||
|
|
||||||
|
**Что значит.** Старейшее задание обработки вложения, которое уже можно брать (статус `queued`,
|
||||||
|
`available_at` прошёл), ждёт больше 30 с дольше минуты. Пока задания не забирают, возраст растёт линейно.
|
||||||
|
|
||||||
|
**Влияние.** Вложения остаются `queued`, клиенты не получают превью; SLO-3 (95% ready ≤ 30 с) нарушается.
|
||||||
|
|
||||||
|
**Найти причину.**
|
||||||
|
1. `messenger_queue_consumers{messaging_destination_name="attachments.process.v1"}` = 0 → worker не подключён
|
||||||
|
(см. MessengerWorkerNoConsumers). > 0 → worker есть, но не успевает или падает на заданиях.
|
||||||
|
2. Насыщение worker: `sum(messenger_worker_busy) / sum(messenger_worker_capacity)` ≈ 1 и растёт
|
||||||
|
`messenger_queue_depth` → нужно больше потоков/реплик worker (`WORKER_CONCURRENCY`, `--scale worker=2`).
|
||||||
|
3. Ошибки обработки: `sum by (outcome) (rate(messenger_job_duration_seconds_count[5m]))` — много `retry`/`failed`;
|
||||||
|
логи `{service_name="messenger-worker"} | event=~"job.retry|job.failed"` (поле `error_type`).
|
||||||
|
4. Отличить от проблемы публикации: `messenger_outbox_oldest_age_seconds` большой → сообщения не доходят до
|
||||||
|
брокера (dispatcher/RabbitMQ), а не worker; лог `{service_name="messenger-dispatcher"} | event="broker.unavailable"`.
|
||||||
|
5. Трасса любой зависшей загрузки (по `trace_id` из лога API `event="http.request"` для POST attachments):
|
||||||
|
есть `publish attachments`, но нет `process attachments.process.v1` → сообщение в брокере, его никто не взял.
|
||||||
|
|
||||||
|
**Исправить.** `docker compose start worker` (или `scripts/worker.sh start`); при насыщении — увеличить
|
||||||
|
параллелизм; при массовых ошибках — устранить их причину, затем `ops.py replay` для failed.
|
||||||
|
|
||||||
|
**Проверить восстановление.** `messenger_jobs_backlog{status="queued"}` → 0, возраст → 0, вложения `ready`,
|
||||||
|
алерт inactive через ~15–30 с. Пример: REPORT.md, инцидент 1.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## MessengerWorkerNoConsumers
|
||||||
|
|
||||||
|
**Что значит.** RabbitMQ дольше минуты не видит ни одного подписчика очереди `attachments.process.v1`.
|
||||||
|
|
||||||
|
**Найти причину.** `docker compose ps worker`; последние записи `{service_name="messenger-worker"}` в Loki
|
||||||
|
(`event="service.stopped"` — остановлен штатно; `event="broker.unavailable"` — не может подключиться к брокеру;
|
||||||
|
отсутствие записей и метрик `messenger_worker_capacity` — процесс не запущен).
|
||||||
|
|
||||||
|
**Исправить.** Запустить worker / восстановить RabbitMQ. Принятые задания не теряются (outbox + durable очередь).
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## MessengerDbPoolSaturated
|
||||||
|
|
||||||
|
**Что значит.** На реплике API запросы дольше минуты ждут свободное соединение из пула (`DB_POOL_MAX`=10).
|
||||||
|
|
||||||
|
**Найти причину.** Почти всегда это следствие медленных запросов или блокировок в PostgreSQL, а не нехватки
|
||||||
|
соединений: те же шаги, что в MessengerHighErrorRate (трассы с долгими DB-спанами, `pg_stat_activity`).
|
||||||
|
Увеличивать пул имеет смысл, только если DB-спаны быстрые, а ожидание пула долгое при высоком RPS.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## TelemetryExportFailing
|
||||||
|
|
||||||
|
**Что значит.** Collector не может отправить данные в Tempo / Loki / Prometheus (метка `exporter`): очередь
|
||||||
|
exporter дольше минуты заполнена больше чем на 5% (хранилище не принимает), либо пакеты уже теряются —
|
||||||
|
у пакета истекло окно повторов 120 с (`send_failed`) или очередь переполнена (`enqueue_failed`). Окно повторов
|
||||||
|
считается для пакета с первой попытки, время ожидания в очереди в него не входит: очередь (200 пакетов) —
|
||||||
|
это и есть буфер на время недоступности.
|
||||||
|
|
||||||
|
**Влияние.** На пользователей не влияет. Телеметрия этого периода частично потеряна (сколько — счётчики
|
||||||
|
`otelcol_exporter_send_failed_*`, `otelcol_exporter_enqueue_failed_*`).
|
||||||
|
|
||||||
|
**Найти причину.** `docker compose ps tempo loki prometheus`; `otelcol_exporter_queue_size / otelcol_exporter_queue_capacity`;
|
||||||
|
логи Collector: `docker compose logs otel-collector --tail 50` (через Loki их не смотреть — это тот же конвейер).
|
||||||
|
|
||||||
|
**Исправить.** `docker compose start tempo loki prometheus`. Очередь Collector досылается сама (в пределах окна
|
||||||
|
повторов). Проверить: `otelcol_exporter_queue_size` → 0, свежая трасса находится. Пример: REPORT.md, инцидент 3.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## TelemetryCollectorDown
|
||||||
|
|
||||||
|
**Что значит.** Prometheus не может забрать метрики Collector (`up{job="otel-collector"} == 0`).
|
||||||
|
|
||||||
|
**Влияние.** Приложение работает; SDK каждого процесса держит не больше 2048 спанов и 2048 записей логов.
|
||||||
|
Потери считаются в самом процессе: пакет, который Collector не принял, — `otel_sdk_exporter_span_exported_total{error_type!=""}`
|
||||||
|
(обычно `ConnectionError`), переполнение очереди — `otel_sdk_processor_span_processed_total{error_type="queue_full"}`. Метрик приложения за
|
||||||
|
этот период нет (пропуск на графиках), cumulative-счётчики после восстановления продолжат расти без потери итогов.
|
||||||
|
|
||||||
|
**Исправить.** `docker compose start otel-collector`; проверить, что трассы и метрики снова приходят.
|
||||||
@@ -0,0 +1,40 @@
|
|||||||
|
# Что сервер пишет в WWW-Authenticate при 401: какую схему входа он ждёт.
|
||||||
|
BASIC_CHALLENGE = 'Basic realm="messenger", charset="UTF-8"' # только POST /auth/sessions
|
||||||
|
BEARER_CHALLENGE = 'Bearer realm="messenger"' # все остальные защищённые запросы
|
||||||
|
|
||||||
|
|
||||||
|
class ApiError(Exception):
|
||||||
|
def __init__(self, status_code: int, code: str, message: str, headers: dict | None = None):
|
||||||
|
super().__init__(message)
|
||||||
|
self.status_code = status_code
|
||||||
|
self.code = code
|
||||||
|
self.message = message
|
||||||
|
self.headers = headers or {}
|
||||||
|
|
||||||
|
|
||||||
|
def bad_request(message: str) -> ApiError:
|
||||||
|
return ApiError(400, "invalid_request", message)
|
||||||
|
|
||||||
|
|
||||||
|
def unauthorized(message: str = "Unauthorized", challenge: str = BEARER_CHALLENGE) -> ApiError:
|
||||||
|
return ApiError(401, "unauthorized", message, {"WWW-Authenticate": challenge})
|
||||||
|
|
||||||
|
|
||||||
|
def not_found(message: str) -> ApiError:
|
||||||
|
return ApiError(404, "not_found", message)
|
||||||
|
|
||||||
|
|
||||||
|
def conflict(message: str) -> ApiError:
|
||||||
|
return ApiError(409, "conflict", message)
|
||||||
|
|
||||||
|
|
||||||
|
def unavailable(message: str = "Service temporarily unavailable") -> ApiError:
|
||||||
|
return ApiError(503, "unavailable", message)
|
||||||
|
|
||||||
|
|
||||||
|
def payload_too_large(message: str) -> ApiError:
|
||||||
|
return ApiError(413, "payload_too_large", message)
|
||||||
|
|
||||||
|
|
||||||
|
def unsupported_media_type(message: str) -> ApiError:
|
||||||
|
return ApiError(415, "unsupported_media_type", message)
|
||||||
@@ -0,0 +1,96 @@
|
|||||||
|
# attachments: lab4: files, limits, damaged images, MIME, filenames, access, attachment_ids
|
||||||
|
# date: 2026-11-01 20:38:49, commit: 470d9d6 + uncommitted changes
|
||||||
|
# command: python.exe student-tests/test_attachments.py
|
||||||
|
# exit code: 0, 10.3 s
|
||||||
|
|
||||||
|
sample.png: 111817 bytes, sha256 99ea7533c441..., downloaded by the other member, same bytes
|
||||||
|
sample.jpg: 1492 bytes, sha256 c310b9a1c208..., downloaded by the other member, same bytes
|
||||||
|
progressive.jpg: 1081 bytes, sha256 913df01aec84..., downloaded by the other member, same bytes
|
||||||
|
interlaced.png: 79 bytes, sha256 38149222f78f..., downloaded by the other member, same bytes
|
||||||
|
1x1.png: 67 bytes, sha256 eaa4a94ea300..., downloaded by the other member, same bytes
|
||||||
|
PASS: PNG, JPEG, progressive JPEG, interlaced PNG and 1x1 PNG are stored and returned byte for byte
|
||||||
|
exactly 10485760 bytes (10 MiB) -> 202, then ready
|
||||||
|
10485761 bytes (10 MiB + 1) with Content-Length -> 413 before reading the body
|
||||||
|
PASS: the 10 MiB limit is exact
|
||||||
|
12 MiB without Content-Length (chunked) -> 413 as soon as the 10 MiB limit is passed
|
||||||
|
PASS: oversized streamed body is cut off, the API keeps working
|
||||||
|
8000x5000 = 40 000 000 pixels (38957 bytes compressed) -> 202, ready 23200a7c
|
||||||
|
8001x5000 = 40 005 000 pixels (38962 bytes) -> 413
|
||||||
|
header claims 100000x100000 in 65 bytes -> 413 from the header, nothing is decoded
|
||||||
|
PASS: pixel limit is checked before decompression
|
||||||
|
PNG cut in half (55908 bytes, image/png) -> 415
|
||||||
|
PNG without IEND (111805 bytes, image/png) -> 415
|
||||||
|
PNG with a flipped byte in IDAT (CRC mismatch) (111817 bytes, image/png) -> 415
|
||||||
|
PNG with garbage after IEND (111824 bytes, image/png) -> 415
|
||||||
|
PNG signature only (8 bytes, image/png) -> 415
|
||||||
|
PNG with less pixel data than the header says (68 bytes, image/png) -> 415
|
||||||
|
JPEG cut in half (746 bytes, image/jpeg) -> 415
|
||||||
|
JPEG header only (20 bytes, image/jpeg) -> 415
|
||||||
|
random bytes (1600 bytes, image/png) -> 415
|
||||||
|
plain text (28 bytes, image/jpeg) -> 415
|
||||||
|
empty body -> 400
|
||||||
|
PASS: damaged and foreign files are rejected with 415, never 500
|
||||||
|
PLTE 771 bytes = 257 entries (review case) (850 bytes, valid CRCs) -> 415
|
||||||
|
PLTE length not a multiple of 3 (83 bytes, valid CRCs) -> 415
|
||||||
|
empty PLTE (79 bytes, valid CRCs) -> 415
|
||||||
|
palette larger than bit depth allows (depth 1, 3 entries) (88 bytes, valid CRCs) -> 415
|
||||||
|
palette image without PLTE (67 bytes, valid CRCs) -> 415
|
||||||
|
PLTE in a grayscale image (85 bytes, valid CRCs) -> 415
|
||||||
|
PLTE after IDAT (85 bytes, valid CRCs) -> 415
|
||||||
|
two PLTE chunks (103 bytes, valid CRCs) -> 415
|
||||||
|
tRNS longer than the palette (100 bytes, valid CRCs) -> 415
|
||||||
|
tRNS before PLTE (98 bytes, valid CRCs) -> 415
|
||||||
|
tRNS of wrong size for RGB (4 bytes instead of 6) (85 bytes, valid CRCs) -> 415
|
||||||
|
tRNS in an image with alpha channel (82 bytes, valid CRCs) -> 415
|
||||||
|
IEND with data (70 bytes, valid CRCs) -> 415
|
||||||
|
unknown critical chunk (81 bytes, valid CRCs) -> 415
|
||||||
|
IDAT chunks not consecutive (95 bytes, valid CRCs) -> 415
|
||||||
|
gAMA after IDAT (85 bytes, valid CRCs) -> 415
|
||||||
|
gAMA of 2 bytes instead of 4 (83 bytes, valid CRCs) -> 415
|
||||||
|
3 MiB metadata chunk before IDAT (3145809 bytes, valid CRCs) -> 415
|
||||||
|
private chunk after IDAT (181 bytes, valid CRCs) -> 415
|
||||||
|
animated PNG frame (fdAT) (91 bytes, valid CRCs) -> 415
|
||||||
|
tIME of 2 bytes instead of 7 (83 bytes, valid CRCs) -> 415
|
||||||
|
tEXt without keyword separator (86 bytes, valid CRCs) -> 415
|
||||||
|
zTXt with broken zlib after IDAT (91 bytes, valid CRCs) -> 415
|
||||||
|
zTXt text bomb: 50 MB of text in ~50 KB after IDAT (48694 bytes, valid CRCs) -> 415
|
||||||
|
palette 256 entries + tRNS + tEXt and tIME after IDAT -> 201, downloaded byte for byte
|
||||||
|
palette depth 2 with 4 entries -> 201, downloaded byte for byte
|
||||||
|
RGB with 6-byte tRNS and gAMA -> 201, downloaded byte for byte
|
||||||
|
compressed zTXt (100 KB of text) and iTXt after IDAT -> 201, downloaded byte for byte
|
||||||
|
PASS: chunk contents are checked, not only CRCs; valid PNGs with metadata are still accepted
|
||||||
|
PNG bytes sent as image/jpeg -> 415
|
||||||
|
JPEG bytes sent as image/png -> 415
|
||||||
|
GIF bytes sent as image/png -> 415
|
||||||
|
PNG sent as image/gif -> 415
|
||||||
|
PNG sent as text/html -> 415
|
||||||
|
PNG sent as application/octet-stream -> 415
|
||||||
|
PNG sent as multipart/form-data -> 415
|
||||||
|
PNG sent as 'IMAGE/PNG; charset=binary' -> 201, stored as image/png
|
||||||
|
PASS: the type is taken from the content; a declared type that does not match is refused
|
||||||
|
13 dangerous or invalid names (path, slashes, NUL, control characters, empty, 256 chars) -> 400
|
||||||
|
no filename -> 400
|
||||||
|
Unicode, 255 chars, leading dots, %20 and ? are kept as display names only
|
||||||
|
PASS: filename never becomes a path; dangerous names are refused
|
||||||
|
outsider -> 404 for metadata and bytes (same as a non-existent id); no token -> 401; bad UUID -> 400
|
||||||
|
upload into someone else's / unknown chat -> 404, without token -> 401
|
||||||
|
variant=preview -> 400 (not in the contract); original/thumbnail/crop/optimized -> 200 (lab5)
|
||||||
|
PASS: only chat members can see an attachment
|
||||||
|
text='' with 2 attachments -> 201, order kept; any member may attach a file uploaded by another member
|
||||||
|
history returns attachment_ids for every message, [] for a message without files; reuse is allowed
|
||||||
|
attachment of another chat (author is a member of both) -> 404
|
||||||
|
own + foreign attachment -> 404
|
||||||
|
unknown attachment_id -> 404
|
||||||
|
outsider referencing an attachment of the chat -> 404
|
||||||
|
duplicate ids -> 400
|
||||||
|
11 ids -> 400
|
||||||
|
bad UUID -> 400
|
||||||
|
empty text without attachments -> 400
|
||||||
|
whitespace text without attachments -> 400
|
||||||
|
PASS: only attachments of the same chat can be linked; rejected requests change nothing
|
||||||
|
same key + same attachments -> same 201; other attachments -> 409; missing field == []
|
||||||
|
PASS: Idempotency-Key covers attachment_ids
|
||||||
|
sent 1.1 MB of a declared 5 MB body and closed the connection
|
||||||
|
the API keeps serving uploads; nothing of the aborted request is visible (DB/S3 side is checked by test_consistency.py)
|
||||||
|
PASS: an interrupted upload is dropped
|
||||||
|
12 passed, 0 failed
|
||||||
@@ -0,0 +1,18 @@
|
|||||||
|
# balancing: balancing, shared sessions/data/files, X-Instance-Id
|
||||||
|
# date: 2026-11-01 20:42:35, commit: 470d9d6 + uncommitted changes
|
||||||
|
# command: python.exe student-tests/test_balancing.py
|
||||||
|
# exit code: 0, 2.1 s
|
||||||
|
|
||||||
|
20 x GET /users/me with one token -> replicas: {'1db2fedff78f': 10, '171a473b0d28': 10}
|
||||||
|
PASS: one session works on every replica, no second login (sessions are shared in Redis)
|
||||||
|
message written by 171a473b0d28; read back by ['171a473b0d28', '1db2fedff78f']
|
||||||
|
PASS: a message written via one replica is read right away via another (shared PostgreSQL)
|
||||||
|
logout on 1db2fedff78f; old token then rejected by ['171a473b0d28', '1db2fedff78f']
|
||||||
|
PASS: logout on one replica revokes the session on all of them
|
||||||
|
first POST on 1db2fedff78f -> 201 7d2bb742-1417-43a4-aa8b-256930bf08ae; same key on 171a473b0d28 -> 201 7d2bb742-1417-43a4-aa8b-256930bf08ae
|
||||||
|
PASS: Idempotency-Key is shared: a retry through another replica returns the same message
|
||||||
|
file uploaded via 1db2fedff78f; downloaded byte for byte via ['171a473b0d28', '1db2fedff78f']
|
||||||
|
PASS: a file uploaded through one replica is downloaded through another (shared S3, nothing local)
|
||||||
|
statuses [200, 200, 201, 400, 401, 404, 409]: every answer has X-Instance-Id of a running container ['171a473b0d28', '1db2fedff78f']; api containers: ['171a473b0d28', '1db2fedff78f']
|
||||||
|
PASS: X-Instance-Id is in every response and points to a real container
|
||||||
|
6 passed, 0 failed
|
||||||
@@ -0,0 +1,6 @@
|
|||||||
|
# check: offline contract check (make check)
|
||||||
|
# date: 2026-11-01 20:07:43, commit: 470d9d6 + uncommitted changes
|
||||||
|
# command: python.exe scripts/check_contract.py
|
||||||
|
# exit code: 0, 0.1 s
|
||||||
|
|
||||||
|
PASS: offline skeleton structure, local refs and Python syntax; application NOT tested.
|
||||||
@@ -0,0 +1,32 @@
|
|||||||
|
# configs: lab6: collector/prometheus/loki/tempo configs by their own tools, alert unit tests, Grafana provisioning
|
||||||
|
# date: 2026-11-01 20:07:53, commit: 470d9d6 + uncommitted changes
|
||||||
|
# command: python.exe student-tests/check_configs.py
|
||||||
|
# exit code: 0, 5.5 s
|
||||||
|
|
||||||
|
--- otelcol-contrib validate (infra/otel/collector.yaml): exit 0
|
||||||
|
|
||||||
|
--- promtool check config (infra/prometheus/prometheus.yml): exit 0
|
||||||
|
|
||||||
|
Checking /etc/prometheus/rules/alerts.yml
|
||||||
|
SUCCESS: 6 rules found
|
||||||
|
|
||||||
|
Checking /etc/prometheus/rules/recording.yml
|
||||||
|
SUCCESS: 10 rules found
|
||||||
|
--- promtool check rules (recording + alerts): exit 0
|
||||||
|
Checking /etc/prometheus/rules/alerts.yml
|
||||||
|
SUCCESS: 6 rules found
|
||||||
|
|
||||||
|
Checking /etc/prometheus/rules/recording.yml
|
||||||
|
SUCCESS: 10 rules found
|
||||||
|
--- promtool test rules (unit tests of alert thresholds/windows/for): exit 0
|
||||||
|
SUCCESS
|
||||||
|
--- loki -verify-config (infra/loki/loki.yaml): exit 0
|
||||||
|
|
||||||
|
--- tempo -config.verify (infra/tempo/tempo.yaml): exit 0
|
||||||
|
|
||||||
|
--- Grafana 12.4.14: database ok
|
||||||
|
datasource prometheus (prometheus): OK Successfully queried the Prometheus API.
|
||||||
|
datasource loki (loki): OK Data source successfully connected.
|
||||||
|
datasource tempo (tempo): OK Data source is working
|
||||||
|
dashboard 'Messenger — lab6 observability' in folder 'Messenger': 34 panels, provisioned=True, datasources ['loki', 'prometheus', 'tempo']
|
||||||
|
PASS: collector, prometheus (config, rules, unit tests), loki and tempo configs are valid; Grafana provisioned 3 healthy datasources and the dashboard without manual steps
|
||||||
@@ -0,0 +1,47 @@
|
|||||||
|
# consistency: lab4: aborted upload, crash between S3 and DB, janitor, orphans, S3 outage
|
||||||
|
# date: 2026-11-01 20:41:39, commit: 470d9d6 + uncommitted changes
|
||||||
|
# command: python.exe student-tests/test_consistency.py
|
||||||
|
# exit code: 0, 159.9 s
|
||||||
|
|
||||||
|
declared 9 000 000 bytes, sent 211817, closed the connection
|
||||||
|
declared 9 000 000 bytes, sent 3111817, closed the connection
|
||||||
|
rows in attachments: 63 -> 63; objects in bucket: 240 -> 240
|
||||||
|
temporary upload files in api containers: {'171a473b0d28': 0, '1db2fedff78f': 0} (removed when the request ends)
|
||||||
|
api containers have no volumes: the original is never stored locally
|
||||||
|
PASS: an interrupted upload leaves no row, no object and no local file
|
||||||
|
crash after S3 PUT: row b1065ae1 state=uploading, object staging/b5763f64729a4b8a8c899bcd7009e599 exists in S3
|
||||||
|
the half-written attachment is invisible: GET -> 404, attach to a message -> 404
|
||||||
|
janitor with default grace (3600 s) right away: {"dry_run": false, "grace_seconds": 3600, "stale_uploads": 0, "stale_uploads_deleted": 0, "staging_ready_deleted": 0, "staging_failed_expired": 0, "staging_kept_for_replay": 0, "outbox_rows_deleted": 0, "objects_scanned": 245, "orphan_objects": 0, "orphan_objects_deleted": 0, "missing_objects": 0, "seconds": 0.133} -> fresh row and object are kept (the upload might still be running)
|
||||||
|
row made 2 h old; janitor --dry-run: {"dry_run": true, "grace_seconds": 3600, "stale_uploads": 1, "stale_uploads_deleted": 0, "staging_ready_deleted": 0, "staging_failed_expired": 0, "staging_kept_for_replay": 0, "outbox_rows_deleted": 0, "objects_scanned": 245, "orphan_objects": 0, "orphan_objects_deleted": 0, "missing_objects": 0, "seconds": 0.132} -> nothing deleted
|
||||||
|
janitor: {"dry_run": false, "grace_seconds": 3600, "stale_uploads": 1, "stale_uploads_deleted": 1, "staging_ready_deleted": 0, "staging_failed_expired": 0, "staging_kept_for_replay": 0, "outbox_rows_deleted": 0, "objects_scanned": 244, "orphan_objects": 0, "orphan_objects_deleted": 0, "missing_objects": 0, "seconds": 0.101} -> row and object deleted
|
||||||
|
janitor again: {"dry_run": false, "grace_seconds": 3600, "stale_uploads": 0, "stale_uploads_deleted": 0, "staging_ready_deleted": 0, "staging_failed_expired": 0, "staging_kept_for_replay": 0, "outbox_rows_deleted": 0, "objects_scanned": 244, "orphan_objects": 0, "orphan_objects_deleted": 0, "missing_objects": 0, "seconds": 0.134} -> idempotent
|
||||||
|
the ready attachment of the same chat is untouched
|
||||||
|
PASS: S3 written but DB not committed -> invisible, reclaimed by janitor only after the grace period
|
||||||
|
object staging/23223961df3644dbbaa0fa4f40d57b82 put into the bucket without a row in PostgreSQL
|
||||||
|
janitor --grace-seconds 3 immediately: {"dry_run": false, "grace_seconds": 3, "stale_uploads": 0, "stale_uploads_deleted": 0, "staging_ready_deleted": 0, "staging_failed_expired": 0, "staging_kept_for_replay": 0, "outbox_rows_deleted": 0, "objects_scanned": 249, "orphan_objects": 0, "orphan_objects_deleted": 0, "missing_objects": 0, "seconds": 0.103} -> fresh object kept
|
||||||
|
after 4 s, --dry-run: {"dry_run": true, "grace_seconds": 3, "stale_uploads": 0, "stale_uploads_deleted": 0, "staging_ready_deleted": 0, "staging_failed_expired": 0, "staging_kept_for_replay": 0, "outbox_rows_deleted": 0, "objects_scanned": 249, "orphan_objects": 1, "orphan_objects_deleted": 0, "missing_objects": 0, "seconds": 0.123} -> reported, kept
|
||||||
|
janitor: {"dry_run": false, "grace_seconds": 3, "stale_uploads": 0, "stale_uploads_deleted": 0, "staging_ready_deleted": 0, "staging_failed_expired": 0, "staging_kept_for_replay": 0, "outbox_rows_deleted": 0, "objects_scanned": 249, "orphan_objects": 1, "orphan_objects_deleted": 1, "missing_objects": 0, "seconds": 0.134} -> orphan deleted
|
||||||
|
ready attachments older than the grace period are untouched (they have rows)
|
||||||
|
object of a ready attachment deleted behind the API's back: janitor reports missing_objects=1 and keeps the row; download -> 503 (not 500)
|
||||||
|
PASS: orphans are found and deleted only after the grace period; missing objects are reported
|
||||||
|
--- S3 stopped (connection refused / name not resolved): docker compose stop s3
|
||||||
|
upload -> 503 unavailable in 20234 ms, no storage details in the body
|
||||||
|
download of an existing file -> 503
|
||||||
|
metadata, messages (also with an already stored attachment) and /health/ready keep working: S3 is needed only for file bytes
|
||||||
|
uploading rows of this chat: before 0, after the failed upload 1 (kept as a pointer for janitor if the rollback could not reach S3); not visible through the API
|
||||||
|
docker compose start s3
|
||||||
|
after S3 recovery the old file downloads byte for byte
|
||||||
|
new upload -> 201
|
||||||
|
janitor after grace period: {"dry_run": false, "grace_seconds": 3600, "stale_uploads": 1, "stale_uploads_deleted": 1, "staging_ready_deleted": 0, "staging_failed_expired": 0, "staging_kept_for_replay": 0, "outbox_rows_deleted": 0, "objects_scanned": 259, "orphan_objects": 0, "orphan_objects_deleted": 0, "missing_objects": 1, "seconds": 0.142}
|
||||||
|
PASS: S3 stopped (connection refused / name not resolved) -> 503 for file operations only, no visible half-uploads, everything cleaned up
|
||||||
|
--- S3 frozen (no answer, timeouts): docker compose pause s3
|
||||||
|
upload -> 503 unavailable in 14703 ms, no storage details in the body
|
||||||
|
download of an existing file -> 503
|
||||||
|
metadata, messages (also with an already stored attachment) and /health/ready keep working: S3 is needed only for file bytes
|
||||||
|
uploading rows of this chat: before 0, after the failed upload 1 (kept as a pointer for janitor if the rollback could not reach S3); not visible through the API
|
||||||
|
docker compose unpause s3
|
||||||
|
after S3 recovery the old file downloads byte for byte
|
||||||
|
new upload -> 201
|
||||||
|
janitor after grace period: {"dry_run": false, "grace_seconds": 3600, "stale_uploads": 1, "stale_uploads_deleted": 1, "staging_ready_deleted": 0, "staging_failed_expired": 0, "staging_kept_for_replay": 0, "outbox_rows_deleted": 0, "objects_scanned": 267, "orphan_objects": 0, "orphan_objects_deleted": 0, "missing_objects": 1, "seconds": 0.192}
|
||||||
|
PASS: S3 frozen (no answer, timeouts) -> 503 for file operations only, no visible half-uploads, everything cleaned up
|
||||||
|
5 passed, 0 failed
|
||||||
@@ -0,0 +1,36 @@
|
|||||||
|
# dashboard: lab6: every dashboard/rule query runs on live data, key signals present, no forbidden labels
|
||||||
|
# date: 2026-11-01 20:09:12, commit: 470d9d6 + uncommitted changes
|
||||||
|
# command: python.exe student-tests/check_dashboard.py
|
||||||
|
# exit code: 0, 1.0 s
|
||||||
|
|
||||||
|
PromQL: 54 dashboard expressions + 16 rule expressions executed, 0 errors, 15 empty now (no traffic of that kind / Docker-only / no incident)
|
||||||
|
empty now: CPU контейнеров / лимит
|
||||||
|
empty now: Collector: потери и отказы
|
||||||
|
empty now: SDK приложений: экспорт и потери
|
||||||
|
empty now: alerts.yml: MessengerDbPoolSaturated
|
||||||
|
empty now: alerts.yml: MessengerHighErrorRate
|
||||||
|
empty now: alerts.yml: MessengerQueueLagGrowing
|
||||||
|
empty now: alerts.yml: MessengerWorkerNoConsumers
|
||||||
|
empty now: alerts.yml: TelemetryCollectorDown
|
||||||
|
empty now: alerts.yml: TelemetryExportFailing
|
||||||
|
empty now: Алерты Prometheus (pending/firing)
|
||||||
|
empty now: Память: RSS процесса и контейнер
|
||||||
|
LogQL ok: Предупреждения и ошибки (Loki): {service_name=~"messenger-.*"} | detected_level=~"warn|error"
|
||||||
|
TraceQL ok: Трассы с ошибками (Tempo, TraceQL): { status = error } -> 4 trace(s)
|
||||||
|
|
||||||
|
data: http_server_request_duration_seconds_count{job="messenger-api"} 48 series <- HTTP requests/duration (3)
|
||||||
|
data: messenger_job_duration_seconds_count 3 series <- jobs processed/failed + duration (3)
|
||||||
|
data: messenger_attachment_time_to_ready_seconds_count 1 series <- end-to-end time to ready (4)
|
||||||
|
data: messenger_queue_depth 2 series <- queue depth (4)
|
||||||
|
data: messenger_queue_oldest_age_seconds 1 series <- oldest job age (4)
|
||||||
|
data: db_client_connection_count 4 series <- DB pool used/idle (4)
|
||||||
|
data: db_client_connection_pending_requests 2 series <- DB pool waiting (4)
|
||||||
|
data: messenger_worker_busy 1 series <- worker busy (4)
|
||||||
|
data: messenger_worker_capacity 1 series <- worker capacity (4)
|
||||||
|
data: process_cpu_time_seconds_total 12 series <- process CPU (4)
|
||||||
|
data: process_memory_usage_bytes 6 series <- process RAM (4)
|
||||||
|
data: otelcol_exporter_queue_size 3 series <- collector queue (4)
|
||||||
|
|
||||||
|
labels used by application metrics: __name__, cpu_mode, db_client_connection_pool_name, db_client_connection_state, deployment_environment_name, http_request_method, http_response_status_class, http_response_status_code, http_route, instance, job, job_type, le, messaging_destination_name, otel_component_name, otel_component_type, otel_span_parent_origin, otel_span_sampling_result, outcome, server_address, server_port, service_version, status, telemetry_sdk_language, telemetry_sdk_name, telemetry_sdk_version
|
||||||
|
http_route values (templates, not URLs): ['/api/v1/attachments/{attachment_id}', '/api/v1/attachments/{attachment_id}/content', '/api/v1/auth/sessions', '/api/v1/auth/sessions/current', '/api/v1/chats', '/api/v1/chats/{chat_id}', '/api/v1/chats/{chat_id}/attachments', '/api/v1/chats/{chat_id}/messages', '/api/v1/users', '/api/v1/users/me', '/health/live', '/health/ready']
|
||||||
|
PASS: every dashboard/rule query runs, key signals have data, no forbidden labels
|
||||||
@@ -0,0 +1,21 @@
|
|||||||
|
# db: PostgreSQL: rollback, FK, Argon2id, unique keys, migrations
|
||||||
|
# date: 2026-11-01 20:42:53, commit: 470d9d6 + uncommitted changes
|
||||||
|
# command: python.exe student-tests/test_db.py
|
||||||
|
# exit code: 0, 11.6 s
|
||||||
|
|
||||||
|
chat with unknown member -> 400; rows with this title: 0; chats without members: 0; chats insert counter: 71 -> 72
|
||||||
|
PASS: chat row was inserted and then rolled back together with members
|
||||||
|
direct INSERT of a message from a non-member -> exit code 1: ERROR: insert or update on table "messages" violates foreign key constraint "messages_chat_id_sender_id_fkey"
|
||||||
|
PASS: the database itself rejects a message from a non-member
|
||||||
|
stored password: $argon2id$v=19$m=12288,t=3,p=1$... (length 97)
|
||||||
|
PASS: password is stored as Argon2id hash
|
||||||
|
unique constraint on users: UNIQUE (username)
|
||||||
|
PASS: username is unique in the database
|
||||||
|
applied migrations: 001_init.sql,002_text_as_bytes.sql,003_idempotency.sql,004_attachments.sql,005_jobs.sql; running the migrate service once more
|
||||||
|
after the second run: 001_init.sql,002_text_as_bytes.sql,003_idempotency.sql,004_attachments.sql,005_jobs.sql; log: Migrations applied: none (schema is up to date)
|
||||||
|
PASS: migrations are applied once, a repeated run does nothing
|
||||||
|
api replicas that checked the schema at start: 10; api log lines 'Migrations applied': 0; migrate service: Exited (0) 4 minutes ago
|
||||||
|
PASS: only the migrate service applies migrations, replicas wait for the schema
|
||||||
|
unique constraint for Idempotency-Key: UNIQUE (chat_id, sender_id, idempotency_key)
|
||||||
|
PASS: one message per (chat, sender, Idempotency-Key) is enforced by the database
|
||||||
|
7 passed, 0 failed
|
||||||
@@ -0,0 +1,109 @@
|
|||||||
|
=== Data volume
|
||||||
|
messages_total | messages_in_big_chat
|
||||||
|
----------------+----------------------
|
||||||
|
68448 | 15000
|
||||||
|
|
||||||
|
|
||||||
|
=== History index
|
||||||
|
indexdef
|
||||||
|
----------------------------------------------------------------------------------------
|
||||||
|
CREATE INDEX messages_history ON public.messages USING btree (chat_id, created_at, id)
|
||||||
|
|
||||||
|
|
||||||
|
=== 1. First page: limit=50, the app asks for 51 rows to know if there is a next page
|
||||||
|
QUERY PLAN
|
||||||
|
------------------------------------------------------------------------------------------------
|
||||||
|
Limit (actual time=0.007..0.019 rows=51 loops=1)
|
||||||
|
Buffers: shared hit=54
|
||||||
|
-> Index Scan using messages_history on messages (actual time=0.007..0.016 rows=51 loops=1)
|
||||||
|
Index Cond: (chat_id = '2ce6ad8d-9425-42cb-aa5f-94f2857a7f39'::uuid)
|
||||||
|
Buffers: shared hit=54
|
||||||
|
Planning:
|
||||||
|
Buffers: shared hit=8
|
||||||
|
Planning Time: 0.067 ms
|
||||||
|
Execution Time: 0.026 ms
|
||||||
|
|
||||||
|
|
||||||
|
=== 2. Page after the 10000th message by keyset cursor (this is what the app does)
|
||||||
|
QUERY PLAN
|
||||||
|
------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
|
||||||
|
Limit (actual time=0.007..0.012 rows=51 loops=1)
|
||||||
|
Buffers: shared hit=5
|
||||||
|
-> Index Scan using messages_history on messages (actual time=0.006..0.009 rows=51 loops=1)
|
||||||
|
Index Cond: ((chat_id = '2ce6ad8d-9425-42cb-aa5f-94f2857a7f39'::uuid) AND (ROW(created_at, id) > ROW('2026-09-01 02:46:40+00'::timestamp with time zone, '09b0c717-4cc7-469c-afb2-fc088519bea2'::uuid)))
|
||||||
|
Buffers: shared hit=5
|
||||||
|
Planning Time: 0.083 ms
|
||||||
|
Execution Time: 0.047 ms
|
||||||
|
|
||||||
|
|
||||||
|
=== 2b. The same page exactly as the app sends it since lab4: with attachment_ids of every message
|
||||||
|
QUERY PLAN
|
||||||
|
------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
|
||||||
|
Limit (actual time=0.024..0.176 rows=51 loops=1)
|
||||||
|
Buffers: shared hit=59
|
||||||
|
-> Index Scan using messages_history on messages m (actual time=0.024..0.173 rows=51 loops=1)
|
||||||
|
Index Cond: ((chat_id = '2ce6ad8d-9425-42cb-aa5f-94f2857a7f39'::uuid) AND (ROW(created_at, id) > ROW('2026-09-01 02:46:40+00'::timestamp with time zone, '09b0c717-4cc7-469c-afb2-fc088519bea2'::uuid)))
|
||||||
|
Buffers: shared hit=59
|
||||||
|
SubPlan 1
|
||||||
|
-> Sort (actual time=0.002..0.002 rows=0 loops=51)
|
||||||
|
Sort Key: a."position"
|
||||||
|
Sort Method: quicksort Memory: 25kB
|
||||||
|
Buffers: shared hit=54
|
||||||
|
-> Bitmap Heap Scan on message_attachments a (actual time=0.000..0.000 rows=0 loops=51)
|
||||||
|
Recheck Cond: (message_id = m.id)
|
||||||
|
Buffers: shared hit=51
|
||||||
|
-> Bitmap Index Scan on message_attachments_message_id_attachment_id_key (actual time=0.000..0.000 rows=0 loops=51)
|
||||||
|
Index Cond: (message_id = m.id)
|
||||||
|
Buffers: shared hit=51
|
||||||
|
Planning:
|
||||||
|
Buffers: shared hit=63
|
||||||
|
Planning Time: 0.210 ms
|
||||||
|
Execution Time: 0.193 ms
|
||||||
|
|
||||||
|
|
||||||
|
=== 3. The same page with OFFSET 10000 (the app does NOT do this)
|
||||||
|
QUERY PLAN
|
||||||
|
--------------------------------------------------------------------------------
|
||||||
|
Limit (actual time=4.087..4.090 rows=51 loops=1)
|
||||||
|
Buffers: shared hit=1255
|
||||||
|
-> Sort (actual time=3.670..3.904 rows=10051 loops=1)
|
||||||
|
Sort Key: created_at, id
|
||||||
|
Sort Method: quicksort Memory: 2377kB
|
||||||
|
Buffers: shared hit=1255
|
||||||
|
-> Seq Scan on messages (actual time=0.006..2.281 rows=15000 loops=1)
|
||||||
|
Filter: (chat_id = '2ce6ad8d-9425-42cb-aa5f-94f2857a7f39'::uuid)
|
||||||
|
Rows Removed by Filter: 53448
|
||||||
|
Buffers: shared hit=1255
|
||||||
|
Planning Time: 0.041 ms
|
||||||
|
Execution Time: 4.136 ms
|
||||||
|
|
||||||
|
|
||||||
|
=== 4. Query 2 without the index (index is dropped only inside this transaction)
|
||||||
|
QUERY PLAN
|
||||||
|
-----------------------------------------------------------------------------------------------------------------------------------------------------
|
||||||
|
Limit (actual time=1.375..1.378 rows=51 loops=1)
|
||||||
|
Buffers: shared hit=1006
|
||||||
|
-> Sort (actual time=1.374..1.376 rows=51 loops=1)
|
||||||
|
Sort Key: created_at, id
|
||||||
|
Sort Method: top-N heapsort Memory: 31kB
|
||||||
|
Buffers: shared hit=1006
|
||||||
|
-> Bitmap Heap Scan on messages (actual time=0.726..1.001 rows=5000 loops=1)
|
||||||
|
Recheck Cond: (chat_id = '2ce6ad8d-9425-42cb-aa5f-94f2857a7f39'::uuid)
|
||||||
|
Filter: (ROW(created_at, id) > ROW('2026-09-01 02:46:40+00'::timestamp with time zone, '09b0c717-4cc7-469c-afb2-fc088519bea2'::uuid))
|
||||||
|
Rows Removed by Filter: 10000
|
||||||
|
Heap Blocks: exact=991
|
||||||
|
Buffers: shared hit=1006
|
||||||
|
-> Bitmap Index Scan on messages_idempotency_key (actual time=0.217..0.217 rows=15000 loops=1)
|
||||||
|
Index Cond: (chat_id = '2ce6ad8d-9425-42cb-aa5f-94f2857a7f39'::uuid)
|
||||||
|
Buffers: shared hit=15
|
||||||
|
Planning:
|
||||||
|
Buffers: shared hit=4
|
||||||
|
Planning Time: 0.057 ms
|
||||||
|
Execution Time: 1.395 ms
|
||||||
|
|
||||||
|
|
||||||
|
=== ROLLBACK: test data removed, index is back
|
||||||
|
explain_users_left
|
||||||
|
--------------------
|
||||||
|
0
|
||||||
|
|
||||||
@@ -0,0 +1,46 @@
|
|||||||
|
# explain: EXPLAIN of the history page (full plans: explain-history.txt)
|
||||||
|
# date: 2026-11-01 20:42:57, commit: 470d9d6 + uncommitted changes
|
||||||
|
# command: python.exe student-tests/explain.py
|
||||||
|
# exit code: 0, 1.2 s
|
||||||
|
|
||||||
|
=== Data volume
|
||||||
|
=== History index
|
||||||
|
=== 1. First page: limit=50, the app asks for 51 rows to know if there is a next page
|
||||||
|
Buffers: shared hit=54
|
||||||
|
-> Index Scan using messages_history on messages (actual time=0.007..0.016 rows=51 loops=1)
|
||||||
|
Buffers: shared hit=54
|
||||||
|
Buffers: shared hit=8
|
||||||
|
Execution Time: 0.026 ms
|
||||||
|
=== 2. Page after the 10000th message by keyset cursor (this is what the app does)
|
||||||
|
Buffers: shared hit=5
|
||||||
|
-> Index Scan using messages_history on messages (actual time=0.006..0.009 rows=51 loops=1)
|
||||||
|
Buffers: shared hit=5
|
||||||
|
Execution Time: 0.047 ms
|
||||||
|
=== 2b. The same page exactly as the app sends it since lab4: with attachment_ids of every message
|
||||||
|
Buffers: shared hit=59
|
||||||
|
-> Index Scan using messages_history on messages m (actual time=0.024..0.173 rows=51 loops=1)
|
||||||
|
Buffers: shared hit=59
|
||||||
|
Buffers: shared hit=54
|
||||||
|
-> Bitmap Heap Scan on message_attachments a (actual time=0.000..0.000 rows=0 loops=51)
|
||||||
|
Buffers: shared hit=51
|
||||||
|
-> Bitmap Index Scan on message_attachments_message_id_attachment_id_key (actual time=0.000..0.000 rows=0 loops=51)
|
||||||
|
Buffers: shared hit=51
|
||||||
|
Buffers: shared hit=63
|
||||||
|
Execution Time: 0.193 ms
|
||||||
|
=== 3. The same page with OFFSET 10000 (the app does NOT do this)
|
||||||
|
Buffers: shared hit=1255
|
||||||
|
Buffers: shared hit=1255
|
||||||
|
-> Seq Scan on messages (actual time=0.006..2.281 rows=15000 loops=1)
|
||||||
|
Buffers: shared hit=1255
|
||||||
|
Execution Time: 4.136 ms
|
||||||
|
=== 4. Query 2 without the index (index is dropped only inside this transaction)
|
||||||
|
Buffers: shared hit=1006
|
||||||
|
Buffers: shared hit=1006
|
||||||
|
-> Bitmap Heap Scan on messages (actual time=0.726..1.001 rows=5000 loops=1)
|
||||||
|
Buffers: shared hit=1006
|
||||||
|
-> Bitmap Index Scan on messages_idempotency_key (actual time=0.217..0.217 rows=15000 loops=1)
|
||||||
|
Buffers: shared hit=15
|
||||||
|
Buffers: shared hit=4
|
||||||
|
Execution Time: 1.395 ms
|
||||||
|
=== ROLLBACK: test data removed, index is back
|
||||||
|
PASS: full plans saved to evidence\explain-history.txt
|
||||||
@@ -0,0 +1,33 @@
|
|||||||
|
# failover: public failover test for every replica, file downloaded via the other one (make test-failover)
|
||||||
|
# date: 2026-11-01 20:43:39, commit: 470d9d6 + uncommitted changes
|
||||||
|
# command: python.exe student-tests/run_failover.py
|
||||||
|
# exit code: 0, 42.0 s
|
||||||
|
|
||||||
|
hook: scripts\stop-one.cmd; api replicas: ['171a473b0d28', '1db2fedff78f']
|
||||||
|
=== run 1: api containers before
|
||||||
|
SERVICE CONTAINER ID STATUS
|
||||||
|
api 171a473b0d28 Up 41 seconds (healthy)
|
||||||
|
api 1db2fedff78f Up 41 seconds (healthy)
|
||||||
|
Executing explicit failover hook; hook output suppressed to avoid leaking secrets.␍
|
||||||
|
{"observed_instances_before": 2, "successful_writes": 127, "failed_attempts": 0, "first_recovery_seconds": 4.75}␍
|
||||||
|
PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.
|
||||||
|
api containers after
|
||||||
|
SERVICE CONTAINER ID STATUS
|
||||||
|
api 171a473b0d28 Exited (0) 11 seconds ago
|
||||||
|
api 1db2fedff78f Up 58 seconds (healthy)
|
||||||
|
the hook stopped 171a473b0d28 (X-Instance-Id of that replica = container ID); other api containers kept running, postgres/redis/lb were not touched
|
||||||
|
restored: docker start 171a473b0d28 -> healthy and back in HAProxy
|
||||||
|
=== run 2: api containers before
|
||||||
|
SERVICE CONTAINER ID STATUS
|
||||||
|
api 171a473b0d28 Up 2 seconds (healthy)
|
||||||
|
api 1db2fedff78f Up About a minute (healthy)
|
||||||
|
Executing explicit failover hook; hook output suppressed to avoid leaking secrets.␍
|
||||||
|
{"observed_instances_before": 2, "successful_writes": 126, "failed_attempts": 0, "first_recovery_seconds": 4.578}␍
|
||||||
|
PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.
|
||||||
|
api containers after
|
||||||
|
SERVICE CONTAINER ID STATUS
|
||||||
|
api 171a473b0d28 Up 19 seconds (healthy)
|
||||||
|
api 1db2fedff78f Exited (0) 11 seconds ago
|
||||||
|
the hook stopped 1db2fedff78f (X-Instance-Id of that replica = container ID); other api containers kept running, postgres/redis/lb were not touched
|
||||||
|
restored: docker start 1db2fedff78f -> healthy and back in HAProxy
|
||||||
|
PASS: public failover test passed 2 times; every api replica was stopped once and restored
|
||||||
@@ -0,0 +1,29 @@
|
|||||||
|
# failures: Redis and PostgreSQL outages via the balancer
|
||||||
|
# date: 2026-11-01 20:45:06, commit: 470d9d6 + uncommitted changes
|
||||||
|
# command: python.exe student-tests/test_failures.py
|
||||||
|
# exit code: 0, 55.6 s
|
||||||
|
|
||||||
|
stopping redis
|
||||||
|
/health/live -> 200, /health/ready -> 503 (8016 ms)
|
||||||
|
valid token: GET /users/me -> 503 unavailable (8000 ms), session cannot be checked
|
||||||
|
unknown token -> 503, not 200: nobody is trusted without a check
|
||||||
|
no token -> 401
|
||||||
|
login -> 503: there is nowhere to store the session
|
||||||
|
registration -> 201: it needs only PostgreSQL
|
||||||
|
starting redis
|
||||||
|
after start: the old token works, confirmed messages are in place
|
||||||
|
PASS: Redis outage gives 503 and does not let anyone in without a session check
|
||||||
|
killing redis with SIGKILL (docker compose kill redis): no graceful shutdown
|
||||||
|
right after the crash: GET /users/me -> 503
|
||||||
|
starting redis
|
||||||
|
after start: the session issued before the crash still works (AOF, appendfsync always)
|
||||||
|
PASS: sessions survive a Redis crash, not only a clean restart
|
||||||
|
stopping postgres
|
||||||
|
/health/live -> 200, /health/ready -> 503 (1016 ms)
|
||||||
|
valid token: users/me, chats, history, new message -> 503 unavailable, no details
|
||||||
|
no token -> 401, unknown token -> 401: anonymous users are not let in
|
||||||
|
login -> 503, registration -> 503
|
||||||
|
starting postgres
|
||||||
|
after start: the old token works, confirmed messages are in place
|
||||||
|
PASS: PostgreSQL outage gives limited 503 errors and keeps anonymous users out
|
||||||
|
3 passed, 0 failed
|
||||||
@@ -0,0 +1,22 @@
|
|||||||
|
# idempotency: Idempotency-Key, parallel same key via replicas
|
||||||
|
# date: 2026-11-01 20:42:37, commit: 470d9d6 + uncommitted changes
|
||||||
|
# command: python.exe student-tests/test_idempotency.py
|
||||||
|
# exit code: 0, 2.2 s
|
||||||
|
|
||||||
|
first -> 201; same JSON written differently -> 201; plain retry -> 201
|
||||||
|
PASS: a retry with the same key returns the original 201 and message, no duplicate
|
||||||
|
20 parallel POSTs with one key: statuses {201: 20}, replicas {'171a473b0d28': 10, '1db2fedff78f': 10}, different bodies in answers: 1, messages in history: 1
|
||||||
|
PASS: parallel requests with one key through different replicas create exactly one message
|
||||||
|
20 parallel POSTs, one key, two different bodies: {('body A', 409): 10, ('body B', 201): 10}; saved: ['body B']
|
||||||
|
PASS: the first body wins, the same body gets 201, the other body gets 409
|
||||||
|
one key: alice in chat 1, bob in chat 1, alice in chat 2 -> 3 messages
|
||||||
|
PASS: the key is scoped to the current user and chat; another body with a used key -> 409
|
||||||
|
missing, empty, 129 chars, space, '/', non-ASCII, '"', '+' -> 400; 128 chars of A-Z a-z 0-9 . _ : - -> 201
|
||||||
|
PASS: Idempotency-Key is required and checked: [A-Za-z0-9._:-]{1,128}
|
||||||
|
key used by a 400 and by a 404 first, then valid request -> 201
|
||||||
|
PASS: rejected requests (400, 404) do not store a result for the key
|
||||||
|
same key and body by an outsider -> 404; by alice after logout -> 401
|
||||||
|
PASS: a retry checks the session and chat access again, the saved answer is not leaked
|
||||||
|
POST sent, connection closed without reading the answer; retry -> 201; messages: 1
|
||||||
|
PASS: an unconfirmed POST can be safely retried with the same key
|
||||||
|
8 passed, 0 failed
|
||||||
@@ -0,0 +1,39 @@
|
|||||||
|
{
|
||||||
|
"scenario": "collector-down",
|
||||||
|
"date": "2026-11-01 20:25:18",
|
||||||
|
"outage_s": 90,
|
||||||
|
"baseline": {
|
||||||
|
"requests": 870,
|
||||||
|
"codes": {
|
||||||
|
"201": 870
|
||||||
|
},
|
||||||
|
"success_ratio": 1.0,
|
||||||
|
"success_rps": 58.0,
|
||||||
|
"p95_ms_successful": 32.0
|
||||||
|
},
|
||||||
|
"during": {
|
||||||
|
"requests": 5852,
|
||||||
|
"codes": {
|
||||||
|
"201": 5852
|
||||||
|
},
|
||||||
|
"success_ratio": 1.0,
|
||||||
|
"success_rps": 57.9,
|
||||||
|
"p95_ms_successful": 31.0
|
||||||
|
},
|
||||||
|
"api_mib_before": 295.2,
|
||||||
|
"api_mib_max": 313.5,
|
||||||
|
"sdk_lost_spans_api": {
|
||||||
|
"processor_queue_full": 8252,
|
||||||
|
"exporter_failed": 10752
|
||||||
|
},
|
||||||
|
"sdk_exported_ok_after_restart": 10287,
|
||||||
|
"after": {
|
||||||
|
"requests": 1403,
|
||||||
|
"codes": {
|
||||||
|
"201": 1403
|
||||||
|
},
|
||||||
|
"success_ratio": 1.0,
|
||||||
|
"success_rps": 56.1,
|
||||||
|
"p95_ms_successful": 32.0
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,20 @@
|
|||||||
|
# incident-collector: lab6: Collector down -> API unaffected, SDK queue bounded, drops counted
|
||||||
|
# date: 2026-11-01 20:27:53, commit: 470d9d6 + uncommitted changes
|
||||||
|
# command: python.exe student-tests/incidents.py collector-down
|
||||||
|
# exit code: 0, 155.8 s
|
||||||
|
|
||||||
|
=== collector-down
|
||||||
|
[ 24.6s] baseline: {'requests': 870, 'codes': {'201': 870}, 'success_ratio': 1.0, 'success_rps': 58.0, 'p95_ms_successful': 32.0}, api containers 295.2 MiB
|
||||||
|
[ 25.3s] docker compose stop otel-collector
|
||||||
|
[ 29.3s] {'t': 0.7, 'api_mib': 292.8}
|
||||||
|
[ 43.6s] {'t': 14.7, 'api_mib': 298.9}
|
||||||
|
[ 58.0s] {'t': 29.0, 'api_mib': 306.5}
|
||||||
|
[ 72.6s] {'t': 43.4, 'api_mib': 307.8}
|
||||||
|
[ 87.0s] {'t': 58.0, 'api_mib': 308.2}
|
||||||
|
[ 101.0s] {'t': 72.4, 'api_mib': 313.5}
|
||||||
|
[ 115.0s] {'t': 86.4, 'api_mib': 313.3}
|
||||||
|
[ 125.6s] docker compose start otel-collector
|
||||||
|
[ 155.6s] API during collector outage: {'requests': 5852, 'codes': {'201': 5852}, 'success_ratio': 1.0, 'success_rps': 57.9, 'p95_ms_successful': 31.0}; api memory 295.2 -> max 313.5 MiB; SDK lost spans (api): {'processor_queue_full': 8252, 'exporter_failed': 10752}; exported OK in the last minute after restart: 10287
|
||||||
|
[ 155.6s] saved C:\Users\Gigabyte\Desktop\lab6\mtusi2026lab6\evidence\incident-collector-down.json
|
||||||
|
PASS: Collector down -> API unaffected, SDK memory bounded, lost spans counted, export resumes after start
|
||||||
|
1 passed, 0 failed
|
||||||
@@ -0,0 +1,82 @@
|
|||||||
|
{
|
||||||
|
"scenario": "db-slow",
|
||||||
|
"date": "2026-11-01 20:13:54",
|
||||||
|
"lock_seconds": 210,
|
||||||
|
"baseline": {
|
||||||
|
"requests": 2310,
|
||||||
|
"codes": {
|
||||||
|
"201": 2310
|
||||||
|
},
|
||||||
|
"success_ratio": 1.0,
|
||||||
|
"success_rps": 115.5,
|
||||||
|
"p95_ms_successful": 32.0,
|
||||||
|
"server_p95_s": 0.019
|
||||||
|
},
|
||||||
|
"firing_after_lock_s": 140.8,
|
||||||
|
"diagnosis": {
|
||||||
|
"http_5xx_ratio_1m": 1.0,
|
||||||
|
"http_p95_messages_s": 7.375,
|
||||||
|
"pool_used": 8.0,
|
||||||
|
"pool_wait_p95_s": 0.001,
|
||||||
|
"status_codes_1m": {
|
||||||
|
"201": 0.0,
|
||||||
|
"200": 0.0,
|
||||||
|
"404": 0.0,
|
||||||
|
"400": 0.0,
|
||||||
|
"409": 0.0,
|
||||||
|
"503": 1.6
|
||||||
|
},
|
||||||
|
"reads_during_lock_ms": {
|
||||||
|
"GET /users/me + GET /chats, avg per request": 17.2,
|
||||||
|
"max": 23.5
|
||||||
|
},
|
||||||
|
"api_log_db_unavailable_by_error_type": {
|
||||||
|
"QueryCanceled": 216
|
||||||
|
},
|
||||||
|
"error_trace": {
|
||||||
|
"trace_id": "237e2d6b1c7db22adbb7d4752fa47f2",
|
||||||
|
"server_ms": 5013.3,
|
||||||
|
"slowest_db_span": "INSERT",
|
||||||
|
"slowest_db_ms": 5001.3,
|
||||||
|
"db_span_status": "ERROR",
|
||||||
|
"db_share": 0.998,
|
||||||
|
"tree": [
|
||||||
|
"- [api] POST /api/v1/chats/{chat_id}/messages (SERVER, 5013.3 ms) ERROR http.response.status_code=503 error.type=503",
|
||||||
|
" - [api] redis GET (CLIENT, 1.1 ms)",
|
||||||
|
" - [api] SELECT (CLIENT, 1.6 ms)",
|
||||||
|
" - [api] INSERT (CLIENT, 5001.3 ms) ERROR"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"handler_cpu_contrast": {
|
||||||
|
"trace_id": "4a222e478c2d51006538a7bfeb485eb1",
|
||||||
|
"server_ms": 21.2,
|
||||||
|
"argon2id_ms": 16.4,
|
||||||
|
"db_ms_total": 0.7,
|
||||||
|
"tree": [
|
||||||
|
"- [api] POST /api/v1/users (SERVER, 21.2 ms) http.response.status_code=201",
|
||||||
|
" - [api] argon2id hash (INTERNAL, 16.4 ms)",
|
||||||
|
" - [api] INSERT (CLIENT, 0.7 ms)"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"during_lock": {
|
||||||
|
"requests": 328,
|
||||||
|
"codes": {
|
||||||
|
"503": 320,
|
||||||
|
"201": 8
|
||||||
|
},
|
||||||
|
"success_ratio": 0.0244,
|
||||||
|
"success_rps": 0.0,
|
||||||
|
"p95_ms_successful": 2156.0
|
||||||
|
},
|
||||||
|
"resolved_after_unlock_s": 50.1,
|
||||||
|
"after": {
|
||||||
|
"requests": 5269,
|
||||||
|
"codes": {
|
||||||
|
"201": 5269
|
||||||
|
},
|
||||||
|
"success_ratio": 1.0,
|
||||||
|
"success_rps": 116.8,
|
||||||
|
"p95_ms_successful": 31.0
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,64 @@
|
|||||||
|
# incident-db: lab6 incident 2: slow PostgreSQL (table lock) -> high error rate firing, cause from traces -> resolved
|
||||||
|
# date: 2026-11-01 20:18:39, commit: 470d9d6 + uncommitted changes
|
||||||
|
# command: python.exe student-tests/incidents.py db-slow
|
||||||
|
# exit code: 0, 285.7 s
|
||||||
|
|
||||||
|
=== db-slow
|
||||||
|
[ 0.0s] load: 8 threads POST /messages
|
||||||
|
[ 25.0s] baseline: {'requests': 2310, 'codes': {'201': 2310}, 'success_ratio': 1.0, 'success_rps': 115.5, 'p95_ms_successful': 32.0, 'server_p95_s': 0.019}
|
||||||
|
[ 25.0s] LOCK TABLE messages IN ACCESS EXCLUSIVE MODE for 210 s (PostgreSQL alive, writes wait)
|
||||||
|
[ 25.1s] 5xx ratio=None p95=0.019 s pool used=0.0 waiting=0.0
|
||||||
|
[ 30.1s] 5xx ratio=0.0 p95=0.02 s pool used=1.0 waiting=0.0
|
||||||
|
[ 35.1s] 5xx ratio=0.0 p95=0.02 s pool used=1.0 waiting=0.0
|
||||||
|
[ 40.1s] 5xx ratio=0.0 p95=0.02 s pool used=8.0 waiting=0.0
|
||||||
|
[ 45.2s] 5xx ratio=0.0 p95=0.02 s pool used=8.0 waiting=0.0
|
||||||
|
[ 50.2s] 5xx ratio=0.0 p95=0.021 s pool used=8.0 waiting=0.0
|
||||||
|
[ 55.3s] 5xx ratio=0.0 p95=0.021 s pool used=8.0 waiting=0.0
|
||||||
|
[ 60.3s] 5xx ratio=0.0 p95=0.022 s pool used=8.0 waiting=0.0
|
||||||
|
[ 65.3s] 5xx ratio=0.0 p95=0.024 s pool used=8.0 waiting=0.0
|
||||||
|
[ 70.4s] 5xx ratio=0.0 p95=0.045 s pool used=8.0 waiting=0.0
|
||||||
|
[ 75.4s] 5xx ratio=0.015 p95=7.18 s pool used=8.0 waiting=0.0
|
||||||
|
[ 80.4s] 5xx ratio=0.015 p95=7.234 s pool used=8.0 waiting=0.0
|
||||||
|
[ 85.4s] 5xx ratio=0.015 p95=7.375 s pool used=8.0 waiting=0.0
|
||||||
|
[ 90.4s] 5xx ratio=0.044 p95=7.375 s pool used=8.0 waiting=0.0
|
||||||
|
[ 95.5s] 5xx ratio=0.044 p95=7.375 s pool used=8.0 waiting=0.0
|
||||||
|
[ 100.5s] 5xx ratio=0.044 p95=7.375 s pool used=8.0 waiting=0.0
|
||||||
|
[ 105.5s] 5xx ratio=1.0 p95=7.375 s pool used=8.0 waiting=0.0
|
||||||
|
[ 110.6s] 5xx ratio=1.0 p95=7.375 s pool used=8.0 waiting=0.0
|
||||||
|
[ 115.6s] 5xx ratio=1.0 p95=7.375 s pool used=8.0 waiting=0.0
|
||||||
|
[ 120.7s] 5xx ratio=1.0 p95=7.375 s pool used=8.0 waiting=0.0
|
||||||
|
[ 125.7s] 5xx ratio=1.0 p95=7.375 s pool used=8.0 waiting=0.0
|
||||||
|
[ 130.7s] 5xx ratio=1.0 p95=7.375 s pool used=8.0 waiting=0.0
|
||||||
|
[ 135.7s] 5xx ratio=1.0 p95=7.375 s pool used=8.0 waiting=0.0
|
||||||
|
[ 140.8s] 5xx ratio=1.0 p95=7.375 s pool used=8.0 waiting=0.0
|
||||||
|
[ 145.8s] 5xx ratio=1.0 p95=7.375 s pool used=8.0 waiting=0.0
|
||||||
|
[ 150.8s] 5xx ratio=1.0 p95=7.375 s pool used=8.0 waiting=0.0
|
||||||
|
[ 155.8s] 5xx ratio=1.0 p95=7.375 s pool used=8.0 waiting=0.0
|
||||||
|
[ 160.8s] 5xx ratio=1.0 p95=7.375 s pool used=8.0 waiting=0.0
|
||||||
|
[ 165.8s] ALERT MessengerHighErrorRate firing (140.8 s after lock)
|
||||||
|
[ 172.2s] diagnosis: {"http_5xx_ratio_1m": 1.0, "http_p95_messages_s": 7.375, "pool_used": 8.0, "pool_wait_p95_s": 0.001, "status_codes_1m": {"201": 0.0, "200": 0.0, "404": 0.0, "400": 0.0, "409": 0.0, "503": 1.6}, "reads_during_lock_ms": {"GET /users/me + GET /chats, avg per request": 17.2, "max": 23.5}, "api_log_db_unavailable_by_error_type": {"QueryCanceled": 216}}
|
||||||
|
[ 172.2s] error trace (DB): {"trace_id": "237e2d6b1c7db22adbb7d4752fa47f2", "server_ms": 5013.3, "slowest_db_span": "INSERT", "slowest_db_ms": 5001.3, "db_span_status": "ERROR", "db_share": 0.998}
|
||||||
|
- [api] POST /api/v1/chats/{chat_id}/messages (SERVER, 5013.3 ms) ERROR http.response.status_code=503 error.type=503
|
||||||
|
- [api] redis GET (CLIENT, 1.1 ms)
|
||||||
|
- [api] SELECT (CLIENT, 1.6 ms)
|
||||||
|
- [api] INSERT (CLIENT, 5001.3 ms) ERROR
|
||||||
|
[ 172.2s] contrast, slow handler (CPU): {"trace_id": "4a222e478c2d51006538a7bfeb485eb1", "server_ms": 21.2, "argon2id_ms": 16.4, "db_ms_total": 0.7}
|
||||||
|
- [api] POST /api/v1/users (SERVER, 21.2 ms) http.response.status_code=201
|
||||||
|
- [api] argon2id hash (INTERNAL, 16.4 ms)
|
||||||
|
- [api] INSERT (CLIENT, 0.7 ms)
|
||||||
|
[ 235.5s] lock released (psql exit 0), 210.5 s after LOCK
|
||||||
|
[ 235.5s] ... MessengerHighErrorRate: firing (0 s)
|
||||||
|
[ 240.5s] ... MessengerHighErrorRate: firing (5 s)
|
||||||
|
[ 245.5s] ... MessengerHighErrorRate: firing (10 s)
|
||||||
|
[ 250.5s] ... MessengerHighErrorRate: firing (15 s)
|
||||||
|
[ 255.5s] ... MessengerHighErrorRate: firing (20 s)
|
||||||
|
[ 260.6s] ... MessengerHighErrorRate: firing (25 s)
|
||||||
|
[ 265.6s] ... MessengerHighErrorRate: firing (30 s)
|
||||||
|
[ 270.6s] ... MessengerHighErrorRate: firing (35 s)
|
||||||
|
[ 275.6s] ... MessengerHighErrorRate: firing (40 s)
|
||||||
|
[ 280.6s] ... MessengerHighErrorRate: firing (45 s)
|
||||||
|
[ 285.6s] ALERT MessengerHighErrorRate resolved 50.1 s after unlock
|
||||||
|
[ 285.6s] load: during lock {'requests': 328, 'codes': {'503': 320, '201': 8}, 'success_ratio': 0.0244, 'success_rps': 0.0, 'p95_ms_successful': 2156.0}; after {'requests': 5269, 'codes': {'201': 5269}, 'success_ratio': 1.0, 'success_rps': 116.8, 'p95_ms_successful': 31.0}
|
||||||
|
[ 285.6s] saved C:\Users\Gigabyte\Desktop\lab6\mtusi2026lab6\evidence\incident-db-slow.json
|
||||||
|
PASS: slow DB -> 503 -> MessengerHighErrorRate firing; traces show the time in the INSERT span (not the handler); unlock -> resolved
|
||||||
|
1 passed, 0 failed
|
||||||
@@ -0,0 +1,273 @@
|
|||||||
|
{
|
||||||
|
"scenario": "telemetry-down",
|
||||||
|
"date": "2026-11-01 20:18:47",
|
||||||
|
"stopped": [
|
||||||
|
"tempo",
|
||||||
|
"loki"
|
||||||
|
],
|
||||||
|
"outage_s": 150,
|
||||||
|
"baseline": {
|
||||||
|
"requests": 899,
|
||||||
|
"codes": {
|
||||||
|
"201": 899
|
||||||
|
},
|
||||||
|
"success_ratio": 1.0,
|
||||||
|
"success_rps": 59.9,
|
||||||
|
"p95_ms_successful": 31.0
|
||||||
|
},
|
||||||
|
"collector_heap_before_mib": 35.5,
|
||||||
|
"collector_container_before_mib": 75.9,
|
||||||
|
"during": {
|
||||||
|
"requests": 9555,
|
||||||
|
"codes": {
|
||||||
|
"201": 9555
|
||||||
|
},
|
||||||
|
"success_ratio": 1.0,
|
||||||
|
"success_rps": 59.6,
|
||||||
|
"p95_ms_successful": 31.0
|
||||||
|
},
|
||||||
|
"samples": [
|
||||||
|
{
|
||||||
|
"t": 0.0,
|
||||||
|
"collector_heap_mib": 56.4,
|
||||||
|
"collector_rss_mib": 237.6,
|
||||||
|
"collector_container_mib": 76.0,
|
||||||
|
"queue": {
|
||||||
|
"otlp_http/loki": 12.0,
|
||||||
|
"otlp_http/prometheus": 0.0,
|
||||||
|
"otlp_grpc/tempo": 5.0
|
||||||
|
},
|
||||||
|
"alert": "inactive"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"t": 12.1,
|
||||||
|
"collector_heap_mib": 45.4,
|
||||||
|
"collector_rss_mib": 237.8,
|
||||||
|
"collector_container_mib": 76.0,
|
||||||
|
"queue": {
|
||||||
|
"otlp_http/loki": 20.0,
|
||||||
|
"otlp_http/prometheus": 0.0,
|
||||||
|
"otlp_grpc/tempo": 9.0
|
||||||
|
},
|
||||||
|
"alert": "pending"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"t": 24.2,
|
||||||
|
"collector_heap_mib": 54.2,
|
||||||
|
"collector_rss_mib": 237.8,
|
||||||
|
"collector_container_mib": 76.0,
|
||||||
|
"queue": {
|
||||||
|
"otlp_http/loki": 27.0,
|
||||||
|
"otlp_http/prometheus": 0.0,
|
||||||
|
"otlp_grpc/tempo": 12.0
|
||||||
|
},
|
||||||
|
"alert": "pending"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"t": 36.2,
|
||||||
|
"collector_heap_mib": 54.2,
|
||||||
|
"collector_rss_mib": 237.8,
|
||||||
|
"collector_container_mib": 78.1,
|
||||||
|
"queue": {
|
||||||
|
"otlp_http/loki": 35.0,
|
||||||
|
"otlp_http/prometheus": 0.0,
|
||||||
|
"otlp_grpc/tempo": 15.0
|
||||||
|
},
|
||||||
|
"alert": "pending"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"t": 48.7,
|
||||||
|
"collector_heap_mib": 62.9,
|
||||||
|
"collector_rss_mib": 240.4,
|
||||||
|
"collector_container_mib": 82.2,
|
||||||
|
"queue": {
|
||||||
|
"otlp_http/loki": 35.0,
|
||||||
|
"otlp_http/prometheus": 0.0,
|
||||||
|
"otlp_grpc/tempo": 15.0
|
||||||
|
},
|
||||||
|
"alert": "pending"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"t": 60.8,
|
||||||
|
"collector_heap_mib": 51.2,
|
||||||
|
"collector_rss_mib": 244.4,
|
||||||
|
"collector_container_mib": 82.2,
|
||||||
|
"queue": {
|
||||||
|
"otlp_http/loki": 42.0,
|
||||||
|
"otlp_http/prometheus": 0.0,
|
||||||
|
"otlp_grpc/tempo": 19.0
|
||||||
|
},
|
||||||
|
"alert": "pending"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"t": 73.1,
|
||||||
|
"collector_heap_mib": 59.9,
|
||||||
|
"collector_rss_mib": 244.4,
|
||||||
|
"collector_container_mib": 82.3,
|
||||||
|
"queue": {
|
||||||
|
"otlp_http/loki": 50.0,
|
||||||
|
"otlp_http/prometheus": 0.0,
|
||||||
|
"otlp_grpc/tempo": 22.0
|
||||||
|
},
|
||||||
|
"alert": "firing"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"t": 85.2,
|
||||||
|
"collector_heap_mib": 70.5,
|
||||||
|
"collector_rss_mib": 248.4,
|
||||||
|
"collector_container_mib": 88.7,
|
||||||
|
"queue": {
|
||||||
|
"otlp_http/loki": 57.0,
|
||||||
|
"otlp_http/prometheus": 0.0,
|
||||||
|
"otlp_grpc/tempo": 26.0
|
||||||
|
},
|
||||||
|
"alert": "firing"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"t": 97.7,
|
||||||
|
"collector_heap_mib": 70.5,
|
||||||
|
"collector_rss_mib": 248.4,
|
||||||
|
"collector_container_mib": 94.5,
|
||||||
|
"queue": {
|
||||||
|
"otlp_http/loki": 63.0,
|
||||||
|
"otlp_http/prometheus": 0.0,
|
||||||
|
"otlp_grpc/tempo": 27.0
|
||||||
|
},
|
||||||
|
"alert": "firing"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"t": 109.8,
|
||||||
|
"collector_heap_mib": 56.3,
|
||||||
|
"collector_rss_mib": 256.7,
|
||||||
|
"collector_container_mib": 94.5,
|
||||||
|
"queue": {
|
||||||
|
"otlp_http/loki": 63.0,
|
||||||
|
"otlp_http/prometheus": 0.0,
|
||||||
|
"otlp_grpc/tempo": 27.0
|
||||||
|
},
|
||||||
|
"alert": "firing"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"t": 122.1,
|
||||||
|
"collector_heap_mib": 67.5,
|
||||||
|
"collector_rss_mib": 256.7,
|
||||||
|
"collector_container_mib": 94.5,
|
||||||
|
"queue": {
|
||||||
|
"otlp_http/loki": 70.0,
|
||||||
|
"otlp_http/prometheus": 0.0,
|
||||||
|
"otlp_grpc/tempo": 31.0
|
||||||
|
},
|
||||||
|
"alert": "firing"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"t": 134.2,
|
||||||
|
"collector_heap_mib": 76.1,
|
||||||
|
"collector_rss_mib": 256.9,
|
||||||
|
"collector_container_mib": 98.8,
|
||||||
|
"queue": {
|
||||||
|
"otlp_http/loki": 78.0,
|
||||||
|
"otlp_http/prometheus": 0.0,
|
||||||
|
"otlp_grpc/tempo": 34.0
|
||||||
|
},
|
||||||
|
"alert": "firing"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"t": 146.2,
|
||||||
|
"collector_heap_mib": 86.9,
|
||||||
|
"collector_rss_mib": 267.1,
|
||||||
|
"collector_container_mib": 107.0,
|
||||||
|
"queue": {
|
||||||
|
"otlp_http/loki": 85.0,
|
||||||
|
"otlp_http/prometheus": 0.0,
|
||||||
|
"otlp_grpc/tempo": 38.0
|
||||||
|
},
|
||||||
|
"alert": "firing"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"collector_heap_max_mib": 86.9,
|
||||||
|
"collector_rss_max_mib": 267.1,
|
||||||
|
"collector_container_max_mib": 107.0,
|
||||||
|
"TelemetryExportFailing_firing_at_s": 73.1,
|
||||||
|
"queue_drained_after_start_s": 60.1,
|
||||||
|
"probe_traces": [
|
||||||
|
{
|
||||||
|
"sent_at_s": 0.0,
|
||||||
|
"http": 201,
|
||||||
|
"trace_in_tempo": true
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"sent_at_s": 12.1,
|
||||||
|
"http": 201,
|
||||||
|
"trace_in_tempo": true
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"sent_at_s": 24.2,
|
||||||
|
"http": 201,
|
||||||
|
"trace_in_tempo": true
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"sent_at_s": 36.2,
|
||||||
|
"http": 201,
|
||||||
|
"trace_in_tempo": true
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"sent_at_s": 48.7,
|
||||||
|
"http": 201,
|
||||||
|
"trace_in_tempo": true
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"sent_at_s": 60.8,
|
||||||
|
"http": 201,
|
||||||
|
"trace_in_tempo": true
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"sent_at_s": 73.1,
|
||||||
|
"http": 201,
|
||||||
|
"trace_in_tempo": true
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"sent_at_s": 85.1,
|
||||||
|
"http": 201,
|
||||||
|
"trace_in_tempo": true
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"sent_at_s": 97.7,
|
||||||
|
"http": 201,
|
||||||
|
"trace_in_tempo": true
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"sent_at_s": 109.8,
|
||||||
|
"http": 201,
|
||||||
|
"trace_in_tempo": true
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"sent_at_s": 122.1,
|
||||||
|
"http": 201,
|
||||||
|
"trace_in_tempo": true
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"sent_at_s": 134.2,
|
||||||
|
"http": 201,
|
||||||
|
"trace_in_tempo": true
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"sent_at_s": 146.2,
|
||||||
|
"http": 201,
|
||||||
|
"trace_in_tempo": true
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"lost_in_collector": {
|
||||||
|
"spans": 4096,
|
||||||
|
"log_records": 235
|
||||||
|
},
|
||||||
|
"resolved_after_start_s": 165.6,
|
||||||
|
"after": {
|
||||||
|
"requests": 9370,
|
||||||
|
"codes": {
|
||||||
|
"201": 9370
|
||||||
|
},
|
||||||
|
"success_ratio": 1.0,
|
||||||
|
"success_rps": 58.3,
|
||||||
|
"p95_ms_successful": 31.0
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,47 @@
|
|||||||
|
# incident-telemetry: lab6: Tempo+Loki down -> API unaffected, collector memory bounded, losses counted
|
||||||
|
# date: 2026-11-01 20:25:06, commit: 470d9d6 + uncommitted changes
|
||||||
|
# command: python.exe student-tests/incidents.py telemetry-down
|
||||||
|
# exit code: 0, 379.8 s
|
||||||
|
|
||||||
|
=== telemetry-down
|
||||||
|
[ 22.5s] baseline: {'requests': 899, 'codes': {'201': 899}, 'success_ratio': 1.0, 'success_rps': 59.9, 'p95_ms_successful': 31.0}, collector heap 35.5 MiB, container 75.9 MiB
|
||||||
|
[ 53.8s] docker compose stop tempo loki: stopped
|
||||||
|
[ 55.9s] {'t': 0.0, 'collector_heap_mib': 56.4, 'collector_rss_mib': 237.6, 'collector_container_mib': 76.0, 'queue': {'otlp_http/loki': 12.0, 'otlp_http/prometheus': 0.0, 'otlp_grpc/tempo': 5.0}, 'alert': 'inactive'}
|
||||||
|
[ 68.0s] {'t': 12.1, 'collector_heap_mib': 45.4, 'collector_rss_mib': 237.8, 'collector_container_mib': 76.0, 'queue': {'otlp_http/loki': 20.0, 'otlp_http/prometheus': 0.0, 'otlp_grpc/tempo': 9.0}, 'alert': 'pending'}
|
||||||
|
[ 80.0s] {'t': 24.2, 'collector_heap_mib': 54.2, 'collector_rss_mib': 237.8, 'collector_container_mib': 76.0, 'queue': {'otlp_http/loki': 27.0, 'otlp_http/prometheus': 0.0, 'otlp_grpc/tempo': 12.0}, 'alert': 'pending'}
|
||||||
|
[ 92.5s] {'t': 36.2, 'collector_heap_mib': 54.2, 'collector_rss_mib': 237.8, 'collector_container_mib': 78.1, 'queue': {'otlp_http/loki': 35.0, 'otlp_http/prometheus': 0.0, 'otlp_grpc/tempo': 15.0}, 'alert': 'pending'}
|
||||||
|
[ 104.5s] {'t': 48.7, 'collector_heap_mib': 62.9, 'collector_rss_mib': 240.4, 'collector_container_mib': 82.2, 'queue': {'otlp_http/loki': 35.0, 'otlp_http/prometheus': 0.0, 'otlp_grpc/tempo': 15.0}, 'alert': 'pending'}
|
||||||
|
[ 116.9s] {'t': 60.8, 'collector_heap_mib': 51.2, 'collector_rss_mib': 244.4, 'collector_container_mib': 82.2, 'queue': {'otlp_http/loki': 42.0, 'otlp_http/prometheus': 0.0, 'otlp_grpc/tempo': 19.0}, 'alert': 'pending'}
|
||||||
|
[ 128.9s] {'t': 73.1, 'collector_heap_mib': 59.9, 'collector_rss_mib': 244.4, 'collector_container_mib': 82.3, 'queue': {'otlp_http/loki': 50.0, 'otlp_http/prometheus': 0.0, 'otlp_grpc/tempo': 22.0}, 'alert': 'firing'}
|
||||||
|
[ 141.5s] {'t': 85.2, 'collector_heap_mib': 70.5, 'collector_rss_mib': 248.4, 'collector_container_mib': 88.7, 'queue': {'otlp_http/loki': 57.0, 'otlp_http/prometheus': 0.0, 'otlp_grpc/tempo': 26.0}, 'alert': 'firing'}
|
||||||
|
[ 153.5s] {'t': 97.7, 'collector_heap_mib': 70.5, 'collector_rss_mib': 248.4, 'collector_container_mib': 94.5, 'queue': {'otlp_http/loki': 63.0, 'otlp_http/prometheus': 0.0, 'otlp_grpc/tempo': 27.0}, 'alert': 'firing'}
|
||||||
|
[ 165.9s] {'t': 109.8, 'collector_heap_mib': 56.3, 'collector_rss_mib': 256.7, 'collector_container_mib': 94.5, 'queue': {'otlp_http/loki': 63.0, 'otlp_http/prometheus': 0.0, 'otlp_grpc/tempo': 27.0}, 'alert': 'firing'}
|
||||||
|
[ 178.0s] {'t': 122.1, 'collector_heap_mib': 67.5, 'collector_rss_mib': 256.7, 'collector_container_mib': 94.5, 'queue': {'otlp_http/loki': 70.0, 'otlp_http/prometheus': 0.0, 'otlp_grpc/tempo': 31.0}, 'alert': 'firing'}
|
||||||
|
[ 190.0s] {'t': 134.2, 'collector_heap_mib': 76.1, 'collector_rss_mib': 256.9, 'collector_container_mib': 98.8, 'queue': {'otlp_http/loki': 78.0, 'otlp_http/prometheus': 0.0, 'otlp_grpc/tempo': 34.0}, 'alert': 'firing'}
|
||||||
|
[ 202.6s] {'t': 146.2, 'collector_heap_mib': 86.9, 'collector_rss_mib': 267.1, 'collector_container_mib': 107.0, 'queue': {'otlp_http/loki': 85.0, 'otlp_http/prometheus': 0.0, 'otlp_grpc/tempo': 38.0}, 'alert': 'firing'}
|
||||||
|
[ 214.0s] docker compose start tempo loki
|
||||||
|
[ 274.1s] exporter queues drained 60.1 s after start
|
||||||
|
[ 289.4s] ... TelemetryExportFailing: firing (0 s)
|
||||||
|
[ 294.4s] ... TelemetryExportFailing: firing (5 s)
|
||||||
|
[ 299.5s] ... TelemetryExportFailing: firing (10 s)
|
||||||
|
[ 304.5s] ... TelemetryExportFailing: firing (15 s)
|
||||||
|
[ 309.5s] ... TelemetryExportFailing: firing (20 s)
|
||||||
|
[ 314.5s] ... TelemetryExportFailing: firing (25 s)
|
||||||
|
[ 319.5s] ... TelemetryExportFailing: firing (30 s)
|
||||||
|
[ 324.5s] ... TelemetryExportFailing: firing (35 s)
|
||||||
|
[ 329.5s] ... TelemetryExportFailing: firing (40 s)
|
||||||
|
[ 334.5s] ... TelemetryExportFailing: firing (45 s)
|
||||||
|
[ 339.6s] ... TelemetryExportFailing: firing (50 s)
|
||||||
|
[ 344.6s] ... TelemetryExportFailing: firing (55 s)
|
||||||
|
[ 349.6s] ... TelemetryExportFailing: firing (60 s)
|
||||||
|
[ 354.6s] ... TelemetryExportFailing: firing (65 s)
|
||||||
|
[ 359.6s] ... TelemetryExportFailing: firing (70 s)
|
||||||
|
[ 364.6s] ... TelemetryExportFailing: firing (75 s)
|
||||||
|
[ 369.6s] ... TelemetryExportFailing: firing (80 s)
|
||||||
|
[ 374.7s] ... TelemetryExportFailing: firing (85 s)
|
||||||
|
[ 379.7s] API during outage: {'requests': 9555, 'codes': {'201': 9555}, 'success_ratio': 1.0, 'success_rps': 59.6, 'p95_ms_successful': 31.0}
|
||||||
|
[ 379.7s] probe traces: 13/13 reached Tempo (lost: sent at [] s)
|
||||||
|
[ 379.7s] lost in Collector (send_failed+enqueue_failed): {'spans': 4096, 'log_records': 235}; collector heap 35.5 -> max 86.9 MiB (memory_limiter 200 MiB), container max 107.0 MiB
|
||||||
|
[ 379.7s] saved C:\Users\Gigabyte\Desktop\lab6\mtusi2026lab6\evidence\incident-telemetry-down.json
|
||||||
|
PASS: Tempo+Loki down -> API unaffected, collector memory bounded, losses counted; recovery -> resolved
|
||||||
|
1 passed, 0 failed
|
||||||
@@ -0,0 +1,50 @@
|
|||||||
|
{
|
||||||
|
"scenario": "worker-stop",
|
||||||
|
"date": "2026-11-01 20:09:59",
|
||||||
|
"firing_after_stop_s": {
|
||||||
|
"MessengerWorkerNoConsumers": 101.7,
|
||||||
|
"MessengerQueueLagGrowing": 131.8
|
||||||
|
},
|
||||||
|
"diagnosis": {
|
||||||
|
"queue_consumers": 0.0,
|
||||||
|
"broker_queue_depth": 6.0,
|
||||||
|
"jobs_backlog_queued": 3.0,
|
||||||
|
"oldest_job_age_s": 127.3,
|
||||||
|
"outbox_oldest_age_s": 0.0,
|
||||||
|
"published_ok_last_5m": 16.41239671427261,
|
||||||
|
"worker_metrics_last_seen_s": 131.0,
|
||||||
|
"last_worker_log": {
|
||||||
|
"event": "service.stopped",
|
||||||
|
"message": "stopped"
|
||||||
|
},
|
||||||
|
"upload_trace_span_names": [
|
||||||
|
"INSERT",
|
||||||
|
"NOTIFY",
|
||||||
|
"POST /api/v1/chats/{chat_id}/attachments",
|
||||||
|
"S3 PutObject",
|
||||||
|
"SELECT",
|
||||||
|
"UPDATE",
|
||||||
|
"create attachments",
|
||||||
|
"inspect image",
|
||||||
|
"publish attachments",
|
||||||
|
"redis GET",
|
||||||
|
"republish lost message"
|
||||||
|
],
|
||||||
|
"upload_trace_has_consumer_span": false
|
||||||
|
},
|
||||||
|
"ready_after_start_s": [
|
||||||
|
4.2,
|
||||||
|
4.2,
|
||||||
|
4.7
|
||||||
|
],
|
||||||
|
"resolved_after_start_s": {
|
||||||
|
"MessengerWorkerNoConsumers": 29.7,
|
||||||
|
"MessengerQueueLagGrowing": 29.7
|
||||||
|
},
|
||||||
|
"after_recovery": {
|
||||||
|
"jobs_backlog_queued": 0.0,
|
||||||
|
"oldest_job_age_s": 0.0,
|
||||||
|
"queue_consumers": 2.0
|
||||||
|
},
|
||||||
|
"time_to_ready_p95_5m_s": 282.0
|
||||||
|
}
|
||||||
@@ -0,0 +1,49 @@
|
|||||||
|
# incident-worker: lab6 incident 1: worker stopped -> queue lag + no consumers firing -> resolved
|
||||||
|
# date: 2026-11-01 20:12:40, commit: 470d9d6 + uncommitted changes
|
||||||
|
# command: python.exe student-tests/incidents.py worker-stop
|
||||||
|
# exit code: 0, 161.8 s
|
||||||
|
|
||||||
|
=== worker-stop
|
||||||
|
[ 0.1s] baseline: consumers=2.0, oldest job age=0.0 s
|
||||||
|
[ 1.2s] docker compose stop worker
|
||||||
|
[ 1.3s] uploaded 3 attachments: 202 queued (API accepts without worker)
|
||||||
|
[ 1.4s] consumers=2.0 backlog queued=0.0 oldest age=0.0 s, alerts: {}
|
||||||
|
[ 6.4s] consumers=2.0 backlog queued=0.0 oldest age=0.0 s, alerts: {}
|
||||||
|
[ 11.4s] consumers=0.0 backlog queued=3.0 oldest age=7.1 s, alerts: {}
|
||||||
|
[ 16.5s] consumers=0.0 backlog queued=3.0 oldest age=7.1 s, alerts: {}
|
||||||
|
[ 21.5s] consumers=0.0 backlog queued=3.0 oldest age=17.1 s, alerts: {}
|
||||||
|
[ 26.5s] consumers=0.0 backlog queued=3.0 oldest age=17.1 s, alerts: {}
|
||||||
|
[ 31.5s] consumers=0.0 backlog queued=3.0 oldest age=27.1 s, alerts: {}
|
||||||
|
[ 36.5s] consumers=0.0 backlog queued=3.0 oldest age=27.1 s, alerts: {}
|
||||||
|
[ 41.6s] consumers=0.0 backlog queued=3.0 oldest age=37.1 s, alerts: {'MessengerWorkerNoConsumers': 'pending'}
|
||||||
|
[ 46.6s] consumers=0.0 backlog queued=3.0 oldest age=37.1 s, alerts: {'MessengerWorkerNoConsumers': 'pending'}
|
||||||
|
[ 51.6s] consumers=0.0 backlog queued=3.0 oldest age=47.1 s, alerts: {'MessengerWorkerNoConsumers': 'pending'}
|
||||||
|
[ 56.6s] consumers=0.0 backlog queued=3.0 oldest age=47.1 s, alerts: {'MessengerWorkerNoConsumers': 'pending'}
|
||||||
|
[ 61.6s] consumers=0.0 backlog queued=3.0 oldest age=57.1 s, alerts: {'MessengerWorkerNoConsumers': 'pending'}
|
||||||
|
[ 66.6s] consumers=0.0 backlog queued=3.0 oldest age=57.1 s, alerts: {'MessengerWorkerNoConsumers': 'pending'}
|
||||||
|
[ 71.7s] consumers=0.0 backlog queued=3.0 oldest age=67.2 s, alerts: {'MessengerQueueLagGrowing': 'pending', 'MessengerWorkerNoConsumers': 'pending'}
|
||||||
|
[ 76.7s] consumers=0.0 backlog queued=3.0 oldest age=67.2 s, alerts: {'MessengerQueueLagGrowing': 'pending', 'MessengerWorkerNoConsumers': 'pending'}
|
||||||
|
[ 81.7s] consumers=0.0 backlog queued=3.0 oldest age=77.2 s, alerts: {'MessengerQueueLagGrowing': 'pending', 'MessengerWorkerNoConsumers': 'pending'}
|
||||||
|
[ 86.7s] consumers=0.0 backlog queued=3.0 oldest age=77.2 s, alerts: {'MessengerQueueLagGrowing': 'pending', 'MessengerWorkerNoConsumers': 'pending'}
|
||||||
|
[ 91.8s] consumers=0.0 backlog queued=3.0 oldest age=87.2 s, alerts: {'MessengerQueueLagGrowing': 'pending', 'MessengerWorkerNoConsumers': 'pending'}
|
||||||
|
[ 96.8s] consumers=0.0 backlog queued=3.0 oldest age=87.2 s, alerts: {'MessengerQueueLagGrowing': 'pending', 'MessengerWorkerNoConsumers': 'pending'}
|
||||||
|
[ 101.8s] ALERT MessengerWorkerNoConsumers firing (101.7 s after stop)
|
||||||
|
[ 101.8s] consumers=0.0 backlog queued=3.0 oldest age=97.2 s, alerts: {'MessengerQueueLagGrowing': 'pending', 'MessengerWorkerNoConsumers': 'firing'}
|
||||||
|
[ 106.8s] consumers=0.0 backlog queued=3.0 oldest age=97.2 s, alerts: {'MessengerQueueLagGrowing': 'pending', 'MessengerWorkerNoConsumers': 'firing'}
|
||||||
|
[ 111.8s] consumers=0.0 backlog queued=3.0 oldest age=107.2 s, alerts: {'MessengerQueueLagGrowing': 'pending', 'MessengerWorkerNoConsumers': 'firing'}
|
||||||
|
[ 116.8s] consumers=0.0 backlog queued=3.0 oldest age=107.2 s, alerts: {'MessengerQueueLagGrowing': 'pending', 'MessengerWorkerNoConsumers': 'firing'}
|
||||||
|
[ 121.8s] consumers=0.0 backlog queued=3.0 oldest age=117.3 s, alerts: {'MessengerQueueLagGrowing': 'pending', 'MessengerWorkerNoConsumers': 'firing'}
|
||||||
|
[ 126.9s] consumers=0.0 backlog queued=3.0 oldest age=117.3 s, alerts: {'MessengerQueueLagGrowing': 'pending', 'MessengerWorkerNoConsumers': 'firing'}
|
||||||
|
[ 131.9s] ALERT MessengerQueueLagGrowing firing (131.8 s after stop)
|
||||||
|
[ 132.0s] diagnosis: {"queue_consumers": 0.0, "broker_queue_depth": 6.0, "jobs_backlog_queued": 3.0, "oldest_job_age_s": 127.3, "outbox_oldest_age_s": 0.0, "published_ok_last_5m": 16.41239671427261, "worker_metrics_last_seen_s": 131.0, "last_worker_log": {"event": "service.stopped", "message": "stopped"}, "upload_trace_span_names": ["INSERT", "NOTIFY", "POST /api/v1/chats/{chat_id}/attachments", "S3 PutObject", "SELECT", "UPDATE", "create attachments", "inspect image", "publish attachments", "redis GET", "republish lost message"], "upload_trace_has_consumer_span": false}
|
||||||
|
[ 135.3s] docker compose start worker
|
||||||
|
[ 136.6s] all attachments ready 4.7 s after worker start; backlog now 3.0
|
||||||
|
[ 136.6s] ... MessengerWorkerNoConsumers: firing (0 s)
|
||||||
|
[ 141.6s] ... MessengerWorkerNoConsumers: firing (5 s)
|
||||||
|
[ 146.6s] ... MessengerWorkerNoConsumers: firing (10 s)
|
||||||
|
[ 151.7s] ... MessengerWorkerNoConsumers: firing (15 s)
|
||||||
|
[ 156.7s] ... MessengerWorkerNoConsumers: firing (20 s)
|
||||||
|
[ 161.7s] alerts resolved: {'MessengerWorkerNoConsumers': 29.7, 'MessengerQueueLagGrowing': 29.7}; now {'jobs_backlog_queued': 0.0, 'oldest_job_age_s': 0.0, 'queue_consumers': 2.0}
|
||||||
|
[ 161.7s] saved C:\Users\Gigabyte\Desktop\lab6\mtusi2026lab6\evidence\incident-worker-stop.json
|
||||||
|
PASS: worker stop -> no consumers + growing queue lag -> firing; cause from metrics/logs/trace; start -> backlog drained -> resolved
|
||||||
|
1 passed, 0 failed
|
||||||
@@ -0,0 +1,8 @@
|
|||||||
|
# jobs-concurrency: lab5: WORKER_CONCURRENCY consumers, prefetch 1, bounded processing
|
||||||
|
# date: 2026-11-01 20:38:38, commit: 470d9d6 + uncommitted changes
|
||||||
|
# command: python.exe student-tests/test_jobs.py concurrency
|
||||||
|
# exit code: 0, 15.8 s
|
||||||
|
|
||||||
|
--- concurrency: Ограниченная конкуренция: WORKER_CONCURRENCY потребителей с prefetch=1; одновременно processing <= 2.
|
||||||
|
2 consumers, prefetch_count 1, manual ack; 8 queued 3000x2000 JPEG drained in 2.4 s after worker start, peak processing 2 of 8 (6 samples)
|
||||||
|
1 passed, 0 failed
|
||||||
@@ -0,0 +1,15 @@
|
|||||||
|
# jobs-crash: lab5: worker crash after S3 PUT / after COMMIT before ack, staging cleanup, frozen worker lease
|
||||||
|
# date: 2026-11-01 20:33:58, commit: 470d9d6 + uncommitted changes
|
||||||
|
# command: python.exe student-tests/test_jobs.py crash-after-put crash-after-commit staging lease
|
||||||
|
# exit code: 0, 157.0 s
|
||||||
|
|
||||||
|
--- crash-after-put: Worker падает после записи объектов в S3, до COMMIT/ack: повтор завершает тот же результат.
|
||||||
|
worker killed after S3 PUT (before COMMIT and ack): redelivery saw a live lease (no-op), lease expired after 15 s, attempt 2 rewrote the same keys -> one set; events ['created', 'claimed', 'duplicate_ignored', 'lease_expired', 'claimed', 'ready']
|
||||||
|
--- crash-after-commit: Worker падает после COMMIT результата, до ack: повторная доставка ready-задачи — no-op с ack.
|
||||||
|
worker killed after COMMIT, before ack: broker redelivered, worker acked as no-op (1:duplicate_ignored:status=ready); queue empty (0 ready, 0 unacked)
|
||||||
|
--- staging: Janitor доделывает очистку staging, если worker упал до неё (после crash-after-commit).
|
||||||
|
staging source still present after the crash (worker died before cleanup); waiting 60 s window
|
||||||
|
janitor removed the staging source of the ready attachment (this pass); source_key NULL, object gone
|
||||||
|
--- lease: Зависший worker: аренда не продлевается -> dispatcher возвращает job -> другой поток завершает;
|
||||||
|
frozen attempt 1 stopped renewing; lease expired, recovered by dispatcher, attempt 2 ready after 17 s; frozen attempt woke up -> stale_result_rejected; events ['created', 'claimed', 'lease_expired', 'claimed', 'ready', 'stale_result_rejected']
|
||||||
|
4 passed, 0 failed
|
||||||
@@ -0,0 +1,13 @@
|
|||||||
|
# jobs-janitor-replay: lab5: janitor does not delete objects re-written by replay (review fix)
|
||||||
|
# date: 2026-11-01 20:38:23, commit: 470d9d6 + uncommitted changes
|
||||||
|
# command: python.exe student-tests/test_janitor_replay.py
|
||||||
|
# exit code: 0, 187.0 s
|
||||||
|
|
||||||
|
race-replayable: failed/retry_exhausted after a crash post-PUT; leftover objects ['v1/crop.jpg', 'v1/optimized.webp', 'v1/original', 'v1/thumbnail.jpg']
|
||||||
|
race-terminal: failed/retry_exhausted after a crash post-PUT; leftover objects ['v1/crop.jpg', 'v1/optimized.webp', 'v1/original', 'v1/thumbnail.jpg']
|
||||||
|
waiting 86 s until the leftovers are older than grace (120 s)
|
||||||
|
janitor pass over the bucket: did not try to delete any object of the replayable failed attachment
|
||||||
|
ops replay -> job 09f5cf6c-bca3-4759-9930-8d18968ed1be ready
|
||||||
|
after the janitor pass: original/thumbnail/crop/optimized -> 200, 4 objects in S3; janitor report: orphans 0, deleted 0
|
||||||
|
retention expired (STAGING_RETENTION_SECONDS=1 for this pass): staging source removed, leftovers of the failed attachment swept; replay refused: "source file is no longer in staging (retention expired)"
|
||||||
|
1 passed, 0 failed
|
||||||
@@ -0,0 +1,11 @@
|
|||||||
|
# jobs-limits: lab5: worker limits - bytes, pixels, memory (RLIMIT_AS), time (JOB_TIMEOUT_SECONDS)
|
||||||
|
# date: 2026-11-01 20:30:08, commit: 470d9d6 + uncommitted changes
|
||||||
|
# command: python.exe student-tests/test_jobs.py limits
|
||||||
|
# exit code: 0, 72.7 s
|
||||||
|
|
||||||
|
--- limits: Лимиты: bytes и пиксели на API; память/пиксели/время процесса обработки на worker.
|
||||||
|
10 MiB + 8 bytes -> 413 before any job is created
|
||||||
|
imaging.py on 8000x5000 PNG: exit 0, peak RSS 226.9 MiB (RLIMIT_AS 512 MiB); same file with MAX_IMAGE_PIXELS=10M -> exit 4 (image_too_large)
|
||||||
|
memory bomb in the imaging process -> failed/resource_limit after attempt 1; worker itself survived
|
||||||
|
imaging hang -> killed after 60 s (processing_timeout, temporary), lease renewed meanwhile (no lease_expired), attempt 2 ready after 62 s
|
||||||
|
1 passed, 0 failed
|
||||||
@@ -0,0 +1,9 @@
|
|||||||
|
# jobs-outbox: lab5: dispatcher crash between COMMIT and publish / after confirm
|
||||||
|
# date: 2026-11-01 20:31:21, commit: 470d9d6 + uncommitted changes
|
||||||
|
# command: python.exe student-tests/test_jobs.py outbox-crash
|
||||||
|
# exit code: 0, 17.2 s
|
||||||
|
|
||||||
|
--- outbox-crash: Разрыв COMMIT/publish: dispatcher падает после COMMIT задания до publish и после confirm до отметки.
|
||||||
|
dispatcher killed between COMMIT and publish; restarted by policy, outbox re-published -> job d1eb476a-fbe1-4271-861f-74f3e943b8cc ready, attempt 1, no manual action
|
||||||
|
dispatcher killed after publisher confirm, before marking outbox -> message published twice; worker: ['created', 'claimed', 'ready', 'duplicate_ignored'] -> one result set
|
||||||
|
1 passed, 0 failed
|
||||||
@@ -0,0 +1,16 @@
|
|||||||
|
# jobs-recovery: lab5: worker down -> 202 -> restart api+broker -> ready; broker down -> outbox waits
|
||||||
|
# date: 2026-11-01 20:31:04, commit: 470d9d6 + uncommitted changes
|
||||||
|
# command: python.exe student-tests/test_jobs.py worker-down broker-down
|
||||||
|
# exit code: 0, 55.2 s
|
||||||
|
|
||||||
|
--- worker-down: Защита п.2-3: worker остановлен -> 202 -> сообщение с вложением -> рестарт API+broker -> worker -> ready.
|
||||||
|
worker stopped
|
||||||
|
upload while worker is down -> 202 queued, attachment 0d7622f1-7c85-4c52-9dc0-2dad8f729c8d
|
||||||
|
message with attachment 201, content 409, other messages 201; job 6a45774d-0d88-46ec-87b9-70fb4c122bbd published, queue messages/unacked = (1, 0)
|
||||||
|
rabbitmq and api restarted with volumes; queue still holds 1 message(s); status queued
|
||||||
|
worker started -> same job 6a45774d-0d88-46ec-87b9-70fb4c122bbd ready, thumbnail downloadable; staging object deleted
|
||||||
|
--- broker-down: Брокер недоступен: API принимает (outbox), dispatcher копит и публикует после восстановления.
|
||||||
|
rabbitmq stopped
|
||||||
|
upload -> 202 and /health/ready 200 without broker; outbox row pending for job 34eb3c30-1942-4777-bce2-1ea2a4eca4c9
|
||||||
|
rabbitmq started -> dispatcher published the pending row -> ready
|
||||||
|
2 passed, 0 failed
|
||||||
@@ -0,0 +1,15 @@
|
|||||||
|
# jobs-retry: lab5: backoff+jitter, retry_exhausted, replay, permanent errors, DLQ
|
||||||
|
# date: 2026-11-01 20:35:16, commit: 470d9d6 + uncommitted changes
|
||||||
|
# command: python.exe student-tests/test_jobs.py retry exhausted-replay permanent dlq
|
||||||
|
# exit code: 0, 77.6 s
|
||||||
|
|
||||||
|
--- retry: Временные ошибки S3: backoff с jitter, затем ready; задержки видны в журнале.
|
||||||
|
2 x storage_unavailable -> retries after 1.90 s and 3.20 s (equal jitter: 1..2 s, 2..4 s), attempt 3 ready
|
||||||
|
--- exhausted-replay: Исчерпание попыток -> failed/retry_exhausted, реестр failed, явный replay; replay ready — отказ.
|
||||||
|
5 temporary errors -> failed/retry_exhausted (public error_code), content 409, attach 409, listed by ops.py failed (replayable)
|
||||||
|
ops replay -> same job_id d8ab860c-eb40-4df2-8d98-67aade24f2f2 attempt 6 ready; second replay refused: "job is ready, replay is only for failed jobs"
|
||||||
|
--- permanent: Постоянная ошибка (битое изображение в worker) -> failed сразу, без повторов.
|
||||||
|
invalid image in the worker -> failed/invalid_image at attempt 1, no retry (permanent error)
|
||||||
|
--- dlq: Невалидные сообщения и сообщения о неизвестной job -> DLQ; dlq-replay их не исполняет.
|
||||||
|
schema-invalid message and unknown job_id -> reject -> DLQ (2 message(s)); dlq-replay kept them: ['invalid message is kept for analysis', 'unknown job']
|
||||||
|
4 passed, 0 failed
|
||||||
@@ -0,0 +1,14 @@
|
|||||||
|
# jobs-variants: lab5: thumbnail/crop/optimized, EXIF orientation, alpha, no upscale, metadata stripped
|
||||||
|
# date: 2026-11-01 20:28:56, commit: 470d9d6 + uncommitted changes
|
||||||
|
# command: python.exe student-tests/test_jobs.py variants
|
||||||
|
# exit code: 0, 20.1 s
|
||||||
|
|
||||||
|
--- variants: thumbnail/crop/optimized по контракту, EXIF orientation, alpha -> белый, без увеличения, без метаданных.
|
||||||
|
thumb320: thumbnail 256x160, crop 128x128, optimized 320x200; original byte-identical; EXIF/XMP in variants: 0; S3 keys ['v1/crop.jpg', 'v1/optimized.webp', 'v1/original', 'v1/thumbnail.jpg']
|
||||||
|
exif6: thumbnail 192x256, crop 128x128, optimized 300x400; original byte-identical; EXIF/XMP in variants: 0; S3 keys ['v1/crop.jpg', 'v1/optimized.webp', 'v1/original', 'v1/thumbnail.jpg']
|
||||||
|
alpha: thumbnail 256x170, crop 128x128, optimized 300x200; original byte-identical; EXIF/XMP in variants: 0; S3 keys ['v1/crop.jpg', 'v1/optimized.webp', 'v1/original', 'v1/thumbnail.jpg']
|
||||||
|
tiny: thumbnail 40x30, crop 128x128, optimized 40x30; original byte-identical; EXIF/XMP in variants: 0; S3 keys ['v1/crop.jpg', 'v1/optimized.webp', 'v1/original', 'v1/thumbnail.jpg']
|
||||||
|
large: thumbnail 256x170, crop 128x128, optimized 1280x853; original byte-identical; EXIF/XMP in variants: 0; S3 keys ['v1/crop.jpg', 'v1/optimized.webp', 'v1/original', 'v1/thumbnail.jpg']
|
||||||
|
strip: thumbnail 256x1, crop 128x128, optimized 1280x1; original byte-identical; EXIF/XMP in variants: 0; S3 keys ['v1/crop.jpg', 'v1/optimized.webp', 'v1/original', 'v1/thumbnail.jpg']
|
||||||
|
unknown variant name -> 400; variants carry their own size/sha256, original fields describe the upload
|
||||||
|
1 passed, 0 failed
|
||||||
@@ -0,0 +1,17 @@
|
|||||||
|
{
|
||||||
|
"lab": 6,
|
||||||
|
"workload": "closed-loop POST messages; no automatic retries",
|
||||||
|
"requested_duration_seconds": 30.0,
|
||||||
|
"elapsed_seconds": 30.031,
|
||||||
|
"concurrency": 4,
|
||||||
|
"successful_requests": 7178,
|
||||||
|
"failed_requests": 0,
|
||||||
|
"error_ratio": 0.0,
|
||||||
|
"successful_rps": 239.02,
|
||||||
|
"successful_latency_ms": {
|
||||||
|
"p50": 16.0,
|
||||||
|
"p95": 32.0,
|
||||||
|
"p99": 47.0
|
||||||
|
},
|
||||||
|
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
|
||||||
|
}
|
||||||
@@ -0,0 +1,22 @@
|
|||||||
|
# load: public load generator, 30 s, concurrency 4
|
||||||
|
# date: 2026-11-01 20:44:10, commit: 470d9d6 + uncommitted changes
|
||||||
|
# command: python.exe tests/load.py --base-url http://127.0.0.1:8080 --duration 30 --concurrency 4 --output evidence/load.json
|
||||||
|
# exit code: 0, 30.3 s
|
||||||
|
|
||||||
|
{
|
||||||
|
"lab": 6,
|
||||||
|
"workload": "closed-loop POST messages; no automatic retries",
|
||||||
|
"requested_duration_seconds": 30.0,
|
||||||
|
"elapsed_seconds": 30.031,
|
||||||
|
"concurrency": 4,
|
||||||
|
"successful_requests": 7178,
|
||||||
|
"failed_requests": 0,
|
||||||
|
"error_ratio": 0.0,
|
||||||
|
"successful_rps": 239.02,
|
||||||
|
"successful_latency_ms": {
|
||||||
|
"p50": 16.0,
|
||||||
|
"p95": 32.0,
|
||||||
|
"p99": 47.0
|
||||||
|
},
|
||||||
|
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
|
||||||
|
}
|
||||||
@@ -0,0 +1,96 @@
|
|||||||
|
# attachments: lab4: files, limits, damaged images, MIME, filenames, access, attachment_ids
|
||||||
|
# date: 2026-11-01 00:52:47, commit: unknown
|
||||||
|
# command: python student-tests/test_attachments.py
|
||||||
|
# exit code: 0, 13.5 s
|
||||||
|
|
||||||
|
sample.png: 111817 bytes, sha256 99ea7533c441..., downloaded by the other member, same bytes
|
||||||
|
sample.jpg: 1492 bytes, sha256 c310b9a1c208..., downloaded by the other member, same bytes
|
||||||
|
progressive.jpg: 1081 bytes, sha256 913df01aec84..., downloaded by the other member, same bytes
|
||||||
|
interlaced.png: 79 bytes, sha256 38149222f78f..., downloaded by the other member, same bytes
|
||||||
|
1x1.png: 67 bytes, sha256 eaa4a94ea300..., downloaded by the other member, same bytes
|
||||||
|
PASS: PNG, JPEG, progressive JPEG, interlaced PNG and 1x1 PNG are stored and returned byte for byte
|
||||||
|
exactly 10485760 bytes (10 MiB) -> 202, then ready
|
||||||
|
10485761 bytes (10 MiB + 1) with Content-Length -> 413 before reading the body
|
||||||
|
PASS: the 10 MiB limit is exact
|
||||||
|
12 MiB without Content-Length (chunked) -> 413 as soon as the 10 MiB limit is passed
|
||||||
|
PASS: oversized streamed body is cut off, the API keeps working
|
||||||
|
8000x5000 = 40 000 000 pixels (38957 bytes compressed) -> 202, ready f30c028f
|
||||||
|
8001x5000 = 40 005 000 pixels (38962 bytes) -> 413
|
||||||
|
header claims 100000x100000 in 65 bytes -> 413 from the header, nothing is decoded
|
||||||
|
PASS: pixel limit is checked before decompression
|
||||||
|
PNG cut in half (55908 bytes, image/png) -> 415
|
||||||
|
PNG without IEND (111805 bytes, image/png) -> 415
|
||||||
|
PNG with a flipped byte in IDAT (CRC mismatch) (111817 bytes, image/png) -> 415
|
||||||
|
PNG with garbage after IEND (111824 bytes, image/png) -> 415
|
||||||
|
PNG signature only (8 bytes, image/png) -> 415
|
||||||
|
PNG with less pixel data than the header says (70 bytes, image/png) -> 415
|
||||||
|
JPEG cut in half (746 bytes, image/jpeg) -> 415
|
||||||
|
JPEG header only (20 bytes, image/jpeg) -> 415
|
||||||
|
random bytes (1600 bytes, image/png) -> 415
|
||||||
|
plain text (28 bytes, image/jpeg) -> 415
|
||||||
|
empty body -> 400
|
||||||
|
PASS: damaged and foreign files are rejected with 415, never 500
|
||||||
|
PLTE 771 bytes = 257 entries (review case) (850 bytes, valid CRCs) -> 415
|
||||||
|
PLTE length not a multiple of 3 (83 bytes, valid CRCs) -> 415
|
||||||
|
empty PLTE (79 bytes, valid CRCs) -> 415
|
||||||
|
palette larger than bit depth allows (depth 1, 3 entries) (88 bytes, valid CRCs) -> 415
|
||||||
|
palette image without PLTE (67 bytes, valid CRCs) -> 415
|
||||||
|
PLTE in a grayscale image (85 bytes, valid CRCs) -> 415
|
||||||
|
PLTE after IDAT (85 bytes, valid CRCs) -> 415
|
||||||
|
two PLTE chunks (103 bytes, valid CRCs) -> 415
|
||||||
|
tRNS longer than the palette (100 bytes, valid CRCs) -> 415
|
||||||
|
tRNS before PLTE (98 bytes, valid CRCs) -> 415
|
||||||
|
tRNS of wrong size for RGB (4 bytes instead of 6) (85 bytes, valid CRCs) -> 415
|
||||||
|
tRNS in an image with alpha channel (84 bytes, valid CRCs) -> 415
|
||||||
|
IEND with data (70 bytes, valid CRCs) -> 415
|
||||||
|
unknown critical chunk (81 bytes, valid CRCs) -> 415
|
||||||
|
IDAT chunks not consecutive (95 bytes, valid CRCs) -> 415
|
||||||
|
gAMA after IDAT (85 bytes, valid CRCs) -> 415
|
||||||
|
gAMA of 2 bytes instead of 4 (83 bytes, valid CRCs) -> 415
|
||||||
|
3 MiB metadata chunk before IDAT (3145809 bytes, valid CRCs) -> 415
|
||||||
|
private chunk after IDAT (181 bytes, valid CRCs) -> 415
|
||||||
|
animated PNG frame (fdAT) (91 bytes, valid CRCs) -> 415
|
||||||
|
tIME of 2 bytes instead of 7 (83 bytes, valid CRCs) -> 415
|
||||||
|
tEXt without keyword separator (86 bytes, valid CRCs) -> 415
|
||||||
|
zTXt with broken zlib after IDAT (91 bytes, valid CRCs) -> 415
|
||||||
|
zTXt text bomb: 50 MB of text in ~50 KB after IDAT (48694 bytes, valid CRCs) -> 415
|
||||||
|
palette 256 entries + tRNS + tEXt and tIME after IDAT -> 201, downloaded byte for byte
|
||||||
|
palette depth 2 with 4 entries -> 201, downloaded byte for byte
|
||||||
|
RGB with 6-byte tRNS and gAMA -> 201, downloaded byte for byte
|
||||||
|
compressed zTXt (100 KB of text) and iTXt after IDAT -> 201, downloaded byte for byte
|
||||||
|
PASS: chunk contents are checked, not only CRCs; valid PNGs with metadata are still accepted
|
||||||
|
PNG bytes sent as image/jpeg -> 415
|
||||||
|
JPEG bytes sent as image/png -> 415
|
||||||
|
GIF bytes sent as image/png -> 415
|
||||||
|
PNG sent as image/gif -> 415
|
||||||
|
PNG sent as text/html -> 415
|
||||||
|
PNG sent as application/octet-stream -> 415
|
||||||
|
PNG sent as multipart/form-data -> 415
|
||||||
|
PNG sent as 'IMAGE/PNG; charset=binary' -> 201, stored as image/png
|
||||||
|
PASS: the type is taken from the content; a declared type that does not match is refused
|
||||||
|
13 dangerous or invalid names (path, slashes, NUL, control characters, empty, 256 chars) -> 400
|
||||||
|
no filename -> 400
|
||||||
|
Unicode, 255 chars, leading dots, %20 and ? are kept as display names only
|
||||||
|
PASS: filename never becomes a path; dangerous names are refused
|
||||||
|
outsider -> 404 for metadata and bytes (same as a non-existent id); no token -> 401; bad UUID -> 400
|
||||||
|
upload into someone else's / unknown chat -> 404, without token -> 401
|
||||||
|
variant=preview -> 400 (not in the contract); original/thumbnail/crop/optimized -> 200 (lab5)
|
||||||
|
PASS: only chat members can see an attachment
|
||||||
|
text='' with 2 attachments -> 201, order kept; any member may attach a file uploaded by another member
|
||||||
|
history returns attachment_ids for every message, [] for a message without files; reuse is allowed
|
||||||
|
attachment of another chat (author is a member of both) -> 404
|
||||||
|
own + foreign attachment -> 404
|
||||||
|
unknown attachment_id -> 404
|
||||||
|
outsider referencing an attachment of the chat -> 404
|
||||||
|
duplicate ids -> 400
|
||||||
|
11 ids -> 400
|
||||||
|
bad UUID -> 400
|
||||||
|
empty text without attachments -> 400
|
||||||
|
whitespace text without attachments -> 400
|
||||||
|
PASS: only attachments of the same chat can be linked; rejected requests change nothing
|
||||||
|
same key + same attachments -> same 201; other attachments -> 409; missing field == []
|
||||||
|
PASS: Idempotency-Key covers attachment_ids
|
||||||
|
sent 1.1 MB of a declared 5 MB body and closed the connection
|
||||||
|
the API keeps serving uploads; nothing of the aborted request is visible (DB/S3 side is checked by test_consistency.py)
|
||||||
|
PASS: an interrupted upload is dropped
|
||||||
|
12 passed, 0 failed
|
||||||
@@ -0,0 +1,6 @@
|
|||||||
|
# check: offline contract check (make check)
|
||||||
|
# date: 2026-11-01 00:22:29, commit: unknown
|
||||||
|
# command: python scripts/check_contract.py
|
||||||
|
# exit code: 0, 0.1 s
|
||||||
|
|
||||||
|
PASS: offline skeleton structure, local refs and Python syntax; application NOT tested.
|
||||||
@@ -0,0 +1,28 @@
|
|||||||
|
# configs: lab6: collector/prometheus/loki/tempo configs by their own tools, alert unit tests, Grafana provisioning
|
||||||
|
# date: 2026-11-01 00:22:31, commit: unknown
|
||||||
|
# command: python student-tests/check_configs.py
|
||||||
|
# exit code: 0, 0.7 s
|
||||||
|
|
||||||
|
--- otelcol-contrib validate (infra/otel/collector.yaml): exit 0
|
||||||
|
|
||||||
|
--- promtool check config (infra/prometheus/prometheus.yml): exit 0
|
||||||
|
|
||||||
|
Checking /home/claude/work/mtusi2026lab6/infra/prometheus/rules/alerts.yml
|
||||||
|
SUCCESS: 6 rules found
|
||||||
|
|
||||||
|
Checking /home/claude/work/mtusi2026lab6/infra/prometheus/rules/recording.yml
|
||||||
|
SUCCESS: 10 rules found
|
||||||
|
--- promtool check rules (recording + alerts): exit 0
|
||||||
|
Checking /home/claude/work/mtusi2026lab6/infra/prometheus/rules/alerts.yml
|
||||||
|
SUCCESS: 6 rules found
|
||||||
|
|
||||||
|
Checking /home/claude/work/mtusi2026lab6/infra/prometheus/rules/recording.yml
|
||||||
|
SUCCESS: 10 rules found
|
||||||
|
--- promtool test rules (unit tests of alert thresholds/windows/for): exit 0
|
||||||
|
SUCCESS
|
||||||
|
--- loki -verify-config (infra/loki/loki.yaml): exit 0
|
||||||
|
|
||||||
|
--- tempo -config.verify (infra/tempo/tempo.yaml): exit 0
|
||||||
|
|
||||||
|
--- Grafana: not reachable at http://127.0.0.1:3000, skipped (local binaries mode)
|
||||||
|
PASS: collector, prometheus (config, rules, unit tests), loki and tempo configs are valid
|
||||||
@@ -0,0 +1,36 @@
|
|||||||
|
# dashboard: lab6: every dashboard/rule query runs on live data, key signals present, no forbidden labels
|
||||||
|
# date: 2026-11-01 00:23:49, commit: unknown
|
||||||
|
# command: python student-tests/check_dashboard.py
|
||||||
|
# exit code: 0, 0.5 s
|
||||||
|
|
||||||
|
PromQL: 54 dashboard expressions + 16 rule expressions executed, 0 errors, 15 empty now (no traffic of that kind / Docker-only / no incident)
|
||||||
|
empty now: CPU контейнеров / лимит
|
||||||
|
empty now: Collector: потери и отказы
|
||||||
|
empty now: SDK приложений: экспорт и потери
|
||||||
|
empty now: alerts.yml: MessengerDbPoolSaturated
|
||||||
|
empty now: alerts.yml: MessengerHighErrorRate
|
||||||
|
empty now: alerts.yml: MessengerQueueLagGrowing
|
||||||
|
empty now: alerts.yml: MessengerWorkerNoConsumers
|
||||||
|
empty now: alerts.yml: TelemetryCollectorDown
|
||||||
|
empty now: alerts.yml: TelemetryExportFailing
|
||||||
|
empty now: Алерты Prometheus (pending/firing)
|
||||||
|
empty now: Память: RSS процесса и контейнер
|
||||||
|
LogQL ok: Предупреждения и ошибки (Loki): {service_name=~"messenger-.*"} | detected_level=~"warn|error"
|
||||||
|
TraceQL ok: Трассы с ошибками (Tempo, TraceQL): { status = error } -> 5 trace(s)
|
||||||
|
|
||||||
|
data: http_server_request_duration_seconds_count{job="messenger-api"} 39 series <- HTTP requests/duration (3)
|
||||||
|
data: messenger_job_duration_seconds_count 3 series <- jobs processed/failed + duration (3)
|
||||||
|
data: messenger_attachment_time_to_ready_seconds_count 1 series <- end-to-end time to ready (4)
|
||||||
|
data: messenger_queue_depth 2 series <- queue depth (4)
|
||||||
|
data: messenger_queue_oldest_age_seconds 1 series <- oldest job age (4)
|
||||||
|
data: db_client_connection_count 4 series <- DB pool used/idle (4)
|
||||||
|
data: db_client_connection_pending_requests 2 series <- DB pool waiting (4)
|
||||||
|
data: messenger_worker_busy 1 series <- worker busy (4)
|
||||||
|
data: messenger_worker_capacity 1 series <- worker capacity (4)
|
||||||
|
data: process_cpu_time_seconds_total 12 series <- process CPU (4)
|
||||||
|
data: process_memory_usage_bytes 6 series <- process RAM (4)
|
||||||
|
data: otelcol_exporter_queue_size 3 series <- collector queue (4)
|
||||||
|
|
||||||
|
labels used by application metrics: __name__, cpu_mode, db_client_connection_pool_name, db_client_connection_state, deployment_environment_name, error_type, http_request_method, http_response_status_class, http_response_status_code, http_route, instance, job, job_type, le, messaging_destination_name, otel_component_name, otel_component_type, otel_span_parent_origin, otel_span_sampling_result, outcome, server_address, server_port, service_version, status, telemetry_sdk_language, telemetry_sdk_name, telemetry_sdk_version
|
||||||
|
http_route values (templates, not URLs): ['/api/v1/attachments/{attachment_id}', '/api/v1/attachments/{attachment_id}/content', '/api/v1/auth/sessions', '/api/v1/auth/sessions/current', '/api/v1/chats', '/api/v1/chats/{chat_id}', '/api/v1/chats/{chat_id}/attachments', '/api/v1/chats/{chat_id}/messages', '/api/v1/users', '/api/v1/users/me', '/health/live', '/health/ready']
|
||||||
|
PASS: every dashboard/rule query runs, key signals have data, no forbidden labels
|
||||||
@@ -0,0 +1,22 @@
|
|||||||
|
# idempotency: Idempotency-Key, parallel same key via replicas
|
||||||
|
# date: 2026-11-01 00:52:50, commit: unknown
|
||||||
|
# command: python student-tests/test_idempotency.py
|
||||||
|
# exit code: 0, 2.2 s
|
||||||
|
|
||||||
|
first -> 201; same JSON written differently -> 201; plain retry -> 201
|
||||||
|
PASS: a retry with the same key returns the original 201 and message, no duplicate
|
||||||
|
20 parallel POSTs with one key: statuses {201: 20}, replicas {'a1a1a1a1a1a1': 10, 'a2a2a2a2a2a2': 10}, different bodies in answers: 1, messages in history: 1
|
||||||
|
PASS: parallel requests with one key through different replicas create exactly one message
|
||||||
|
20 parallel POSTs, one key, two different bodies: {('body A', 409): 10, ('body B', 201): 10}; saved: ['body B']
|
||||||
|
PASS: the first body wins, the same body gets 201, the other body gets 409
|
||||||
|
one key: alice in chat 1, bob in chat 1, alice in chat 2 -> 3 messages
|
||||||
|
PASS: the key is scoped to the current user and chat; another body with a used key -> 409
|
||||||
|
missing, empty, 129 chars, space, '/', non-ASCII, '"', '+' -> 400; 128 chars of A-Z a-z 0-9 . _ : - -> 201
|
||||||
|
PASS: Idempotency-Key is required and checked: [A-Za-z0-9._:-]{1,128}
|
||||||
|
key used by a 400 and by a 404 first, then valid request -> 201
|
||||||
|
PASS: rejected requests (400, 404) do not store a result for the key
|
||||||
|
same key and body by an outsider -> 404; by alice after logout -> 401
|
||||||
|
PASS: a retry checks the session and chat access again, the saved answer is not leaked
|
||||||
|
POST sent, connection closed without reading the answer; retry -> 201; messages: 1
|
||||||
|
PASS: an unconfirmed POST can be safely retried with the same key
|
||||||
|
8 passed, 0 failed
|
||||||
@@ -0,0 +1,39 @@
|
|||||||
|
{
|
||||||
|
"scenario": "collector-down",
|
||||||
|
"date": "2026-11-01 00:39:14",
|
||||||
|
"outage_s": 90,
|
||||||
|
"baseline": {
|
||||||
|
"requests": 1002,
|
||||||
|
"codes": {
|
||||||
|
"201": 1002
|
||||||
|
},
|
||||||
|
"success_ratio": 1.0,
|
||||||
|
"success_rps": 66.8,
|
||||||
|
"p95_ms_successful": 15.2
|
||||||
|
},
|
||||||
|
"during": {
|
||||||
|
"requests": 6263,
|
||||||
|
"codes": {
|
||||||
|
"201": 6263
|
||||||
|
},
|
||||||
|
"success_ratio": 1.0,
|
||||||
|
"success_rps": 67.4,
|
||||||
|
"p95_ms_successful": 14.3
|
||||||
|
},
|
||||||
|
"api_mib_before": 280.5,
|
||||||
|
"api_mib_max": 292.5,
|
||||||
|
"sdk_lost_spans_api": {
|
||||||
|
"processor_queue_full": 0,
|
||||||
|
"exporter_failed": 24576
|
||||||
|
},
|
||||||
|
"sdk_exported_ok_after_restart": 10524,
|
||||||
|
"after": {
|
||||||
|
"requests": 1703,
|
||||||
|
"codes": {
|
||||||
|
"201": 1703
|
||||||
|
},
|
||||||
|
"success_ratio": 1.0,
|
||||||
|
"success_rps": 68.1,
|
||||||
|
"p95_ms_successful": 13.8
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,22 @@
|
|||||||
|
# incident-collector: lab6: Collector down -> API unaffected, SDK queue bounded, drops counted
|
||||||
|
# date: 2026-11-01 00:41:37, commit: unknown
|
||||||
|
# command: python student-tests/incidents.py collector-down
|
||||||
|
# exit code: 0, 143.3 s
|
||||||
|
|
||||||
|
=== collector-down
|
||||||
|
[ 20.3s] baseline: {'requests': 1002, 'codes': {'201': 1002}, 'success_ratio': 1.0, 'success_rps': 66.8, 'p95_ms_successful': 15.2}, api containers 280.5 MiB
|
||||||
|
[ 20.6s] docker compose stop otel-collector
|
||||||
|
[ 20.8s] {'t': 0.3, 'api_mib': 283.0}
|
||||||
|
[ 31.0s] {'t': 10.5, 'api_mib': 283.2}
|
||||||
|
[ 41.3s] {'t': 20.8, 'api_mib': 288.6}
|
||||||
|
[ 51.5s] {'t': 31.0, 'api_mib': 289.0}
|
||||||
|
[ 61.7s] {'t': 41.2, 'api_mib': 289.7}
|
||||||
|
[ 71.9s] {'t': 51.4, 'api_mib': 292.5}
|
||||||
|
[ 82.1s] {'t': 61.6, 'api_mib': 292.4}
|
||||||
|
[ 92.4s] {'t': 71.9, 'api_mib': 292.3}
|
||||||
|
[ 102.6s] {'t': 82.1, 'api_mib': 291.2}
|
||||||
|
[ 113.2s] docker compose start otel-collector
|
||||||
|
[ 143.2s] API during collector outage: {'requests': 6263, 'codes': {'201': 6263}, 'success_ratio': 1.0, 'success_rps': 67.4, 'p95_ms_successful': 14.3}; api memory 280.5 -> max 292.5 MiB; SDK lost spans (api): {'processor_queue_full': 0, 'exporter_failed': 24576}; exported OK in the last minute after restart: 10524
|
||||||
|
[ 143.2s] saved /home/claude/work/mtusi2026lab6/evidence/native-sandbox/incident-collector-down.json
|
||||||
|
PASS: Collector down -> API unaffected, SDK memory bounded, lost spans counted, export resumes after start
|
||||||
|
1 passed, 0 failed
|
||||||
@@ -0,0 +1,82 @@
|
|||||||
|
{
|
||||||
|
"scenario": "db-slow",
|
||||||
|
"date": "2026-11-01 00:42:49",
|
||||||
|
"lock_seconds": 210,
|
||||||
|
"baseline": {
|
||||||
|
"requests": 2648,
|
||||||
|
"codes": {
|
||||||
|
"201": 2648
|
||||||
|
},
|
||||||
|
"success_ratio": 1.0,
|
||||||
|
"success_rps": 132.4,
|
||||||
|
"p95_ms_successful": 18.5,
|
||||||
|
"server_p95_s": 0.0214
|
||||||
|
},
|
||||||
|
"firing_after_lock_s": 120.1,
|
||||||
|
"diagnosis": {
|
||||||
|
"http_5xx_ratio_1m": 1.0,
|
||||||
|
"http_p95_messages_s": 7.375,
|
||||||
|
"pool_used": 8.0,
|
||||||
|
"pool_wait_p95_s": 0.001,
|
||||||
|
"status_codes_1m": {
|
||||||
|
"201": 0.0,
|
||||||
|
"200": 0.0,
|
||||||
|
"404": 0.0,
|
||||||
|
"400": 0.0,
|
||||||
|
"409": 0.0,
|
||||||
|
"503": 1.6
|
||||||
|
},
|
||||||
|
"reads_during_lock_ms": {
|
||||||
|
"GET /users/me + GET /chats, avg per request": 4.7,
|
||||||
|
"max": 5.4
|
||||||
|
},
|
||||||
|
"api_log_db_unavailable_by_error_type": {
|
||||||
|
"QueryCanceled": 184
|
||||||
|
},
|
||||||
|
"error_trace": {
|
||||||
|
"trace_id": "1c7f8dd06427574a5f155558aa4cd975",
|
||||||
|
"server_ms": 5006.7,
|
||||||
|
"slowest_db_span": "INSERT",
|
||||||
|
"slowest_db_ms": 5000.6,
|
||||||
|
"db_span_status": "ERROR",
|
||||||
|
"db_share": 0.999,
|
||||||
|
"tree": [
|
||||||
|
"- [api] POST /api/v1/chats/{chat_id}/messages (SERVER, 5006.7 ms) ERROR http.response.status_code=503 error.type=503",
|
||||||
|
" - [api] redis GET (CLIENT, 0.2 ms)",
|
||||||
|
" - [api] SELECT (CLIENT, 0.4 ms)",
|
||||||
|
" - [api] INSERT (CLIENT, 5000.6 ms) ERROR"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"handler_cpu_contrast": {
|
||||||
|
"trace_id": "a6da2e3fe7b0c229773f36fa507293ca",
|
||||||
|
"server_ms": 34.0,
|
||||||
|
"argon2id_ms": 29.0,
|
||||||
|
"db_ms_total": 0.6,
|
||||||
|
"tree": [
|
||||||
|
"- [api] POST /api/v1/users (SERVER, 34.0 ms) http.response.status_code=201",
|
||||||
|
" - [api] argon2id hash (INTERNAL, 29.0 ms)",
|
||||||
|
" - [api] INSERT (CLIENT, 0.6 ms)"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"during_lock": {
|
||||||
|
"requests": 326,
|
||||||
|
"codes": {
|
||||||
|
"503": 320,
|
||||||
|
"201": 6
|
||||||
|
},
|
||||||
|
"success_ratio": 0.0184,
|
||||||
|
"success_rps": 0.0,
|
||||||
|
"p95_ms_successful": 2718.4
|
||||||
|
},
|
||||||
|
"resolved_after_unlock_s": 30.0,
|
||||||
|
"after": {
|
||||||
|
"requests": 3298,
|
||||||
|
"codes": {
|
||||||
|
"201": 3298
|
||||||
|
},
|
||||||
|
"success_ratio": 1.0,
|
||||||
|
"success_rps": 131.9,
|
||||||
|
"p95_ms_successful": 21.0
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,56 @@
|
|||||||
|
# incident-db: lab6 incident 2: slow PostgreSQL (table lock) -> high error rate firing, cause from traces -> resolved
|
||||||
|
# date: 2026-11-01 00:47:15, commit: unknown
|
||||||
|
# command: python student-tests/incidents.py db-slow
|
||||||
|
# exit code: 0, 265.4 s
|
||||||
|
|
||||||
|
=== db-slow
|
||||||
|
[ 0.1s] load: 8 threads POST /messages
|
||||||
|
[ 25.1s] baseline: {'requests': 2648, 'codes': {'201': 2648}, 'success_ratio': 1.0, 'success_rps': 132.4, 'p95_ms_successful': 18.5, 'server_p95_s': 0.0214}
|
||||||
|
[ 25.1s] LOCK TABLE messages IN ACCESS EXCLUSIVE MODE for 210 s (PostgreSQL alive, writes wait)
|
||||||
|
[ 25.1s] 5xx ratio=0.0 p95=0.021 s pool used=0.0 waiting=0.0
|
||||||
|
[ 30.1s] 5xx ratio=0.0 p95=0.022 s pool used=8.0 waiting=0.0
|
||||||
|
[ 35.1s] 5xx ratio=0.0 p95=0.022 s pool used=8.0 waiting=0.0
|
||||||
|
[ 40.1s] 5xx ratio=0.0 p95=0.022 s pool used=8.0 waiting=0.0
|
||||||
|
[ 45.1s] 5xx ratio=0.005 p95=0.022 s pool used=8.0 waiting=0.0
|
||||||
|
[ 50.1s] 5xx ratio=0.005 p95=0.023 s pool used=8.0 waiting=0.0
|
||||||
|
[ 55.1s] 5xx ratio=0.005 p95=0.023 s pool used=8.0 waiting=0.0
|
||||||
|
[ 60.1s] 5xx ratio=0.01 p95=0.024 s pool used=8.0 waiting=0.0
|
||||||
|
[ 65.1s] 5xx ratio=0.01 p95=0.024 s pool used=8.0 waiting=0.0
|
||||||
|
[ 70.1s] 5xx ratio=0.01 p95=5.688 s pool used=8.0 waiting=0.0
|
||||||
|
[ 75.2s] 5xx ratio=0.069 p95=5.688 s pool used=8.0 waiting=0.0
|
||||||
|
[ 80.2s] 5xx ratio=0.069 p95=7.375 s pool used=8.0 waiting=0.0
|
||||||
|
[ 85.2s] 5xx ratio=0.069 p95=7.375 s pool used=8.0 waiting=0.0
|
||||||
|
[ 90.2s] 5xx ratio=1.0 p95=7.375 s pool used=8.0 waiting=0.0
|
||||||
|
[ 95.2s] 5xx ratio=1.0 p95=7.375 s pool used=8.0 waiting=0.0
|
||||||
|
[ 100.2s] 5xx ratio=1.0 p95=7.375 s pool used=8.0 waiting=0.0
|
||||||
|
[ 105.2s] 5xx ratio=1.0 p95=7.375 s pool used=8.0 waiting=0.0
|
||||||
|
[ 110.2s] 5xx ratio=1.0 p95=7.375 s pool used=8.0 waiting=0.0
|
||||||
|
[ 115.2s] 5xx ratio=1.0 p95=7.375 s pool used=8.0 waiting=0.0
|
||||||
|
[ 120.2s] 5xx ratio=1.0 p95=7.375 s pool used=8.0 waiting=0.0
|
||||||
|
[ 125.2s] 5xx ratio=1.0 p95=7.375 s pool used=8.0 waiting=0.0
|
||||||
|
[ 130.2s] 5xx ratio=1.0 p95=7.375 s pool used=8.0 waiting=0.0
|
||||||
|
[ 135.2s] 5xx ratio=1.0 p95=7.375 s pool used=8.0 waiting=0.0
|
||||||
|
[ 140.2s] 5xx ratio=1.0 p95=7.375 s pool used=8.0 waiting=0.0
|
||||||
|
[ 145.2s] ALERT MessengerHighErrorRate firing (120.1 s after lock)
|
||||||
|
[ 151.5s] diagnosis: {"http_5xx_ratio_1m": 1.0, "http_p95_messages_s": 7.375, "pool_used": 8.0, "pool_wait_p95_s": 0.001, "status_codes_1m": {"201": 0.0, "200": 0.0, "404": 0.0, "400": 0.0, "409": 0.0, "503": 1.6}, "reads_during_lock_ms": {"GET /users/me + GET /chats, avg per request": 4.7, "max": 5.4}, "api_log_db_unavailable_by_error_type": {"QueryCanceled": 184}}
|
||||||
|
[ 151.5s] error trace (DB): {"trace_id": "1c7f8dd06427574a5f155558aa4cd975", "server_ms": 5006.7, "slowest_db_span": "INSERT", "slowest_db_ms": 5000.6, "db_span_status": "ERROR", "db_share": 0.999}
|
||||||
|
- [api] POST /api/v1/chats/{chat_id}/messages (SERVER, 5006.7 ms) ERROR http.response.status_code=503 error.type=503
|
||||||
|
- [api] redis GET (CLIENT, 0.2 ms)
|
||||||
|
- [api] SELECT (CLIENT, 0.4 ms)
|
||||||
|
- [api] INSERT (CLIENT, 5000.6 ms) ERROR
|
||||||
|
[ 151.5s] contrast, slow handler (CPU): {"trace_id": "a6da2e3fe7b0c229773f36fa507293ca", "server_ms": 34.0, "argon2id_ms": 29.0, "db_ms_total": 0.6}
|
||||||
|
- [api] POST /api/v1/users (SERVER, 34.0 ms) http.response.status_code=201
|
||||||
|
- [api] argon2id hash (INTERNAL, 29.0 ms)
|
||||||
|
- [api] INSERT (CLIENT, 0.6 ms)
|
||||||
|
[ 235.4s] lock released (psql exit 0), 210.3 s after LOCK
|
||||||
|
[ 235.4s] ... MessengerHighErrorRate: firing (0 s)
|
||||||
|
[ 240.4s] ... MessengerHighErrorRate: firing (5 s)
|
||||||
|
[ 245.4s] ... MessengerHighErrorRate: firing (10 s)
|
||||||
|
[ 250.4s] ... MessengerHighErrorRate: firing (15 s)
|
||||||
|
[ 255.4s] ... MessengerHighErrorRate: firing (20 s)
|
||||||
|
[ 260.4s] ... MessengerHighErrorRate: firing (25 s)
|
||||||
|
[ 265.4s] ALERT MessengerHighErrorRate resolved 30.0 s after unlock
|
||||||
|
[ 265.4s] load: during lock {'requests': 326, 'codes': {'503': 320, '201': 6}, 'success_ratio': 0.0184, 'success_rps': 0.0, 'p95_ms_successful': 2718.4}; after {'requests': 3298, 'codes': {'201': 3298}, 'success_ratio': 1.0, 'success_rps': 131.9, 'p95_ms_successful': 21.0}
|
||||||
|
[ 265.4s] saved /home/claude/work/mtusi2026lab6/evidence/native-sandbox/incident-db-slow.json
|
||||||
|
PASS: slow DB -> 503 -> MessengerHighErrorRate firing; traces show the time in the INSERT span (not the handler); unlock -> resolved
|
||||||
|
1 passed, 0 failed
|
||||||
@@ -0,0 +1,307 @@
|
|||||||
|
{
|
||||||
|
"scenario": "telemetry-down",
|
||||||
|
"date": "2026-11-01 00:32:59",
|
||||||
|
"stopped": [
|
||||||
|
"tempo",
|
||||||
|
"loki"
|
||||||
|
],
|
||||||
|
"outage_s": 150,
|
||||||
|
"baseline": {
|
||||||
|
"requests": 955,
|
||||||
|
"codes": {
|
||||||
|
"201": 955
|
||||||
|
},
|
||||||
|
"success_ratio": 1.0,
|
||||||
|
"success_rps": 63.7,
|
||||||
|
"p95_ms_successful": 17.9
|
||||||
|
},
|
||||||
|
"collector_heap_before_mib": 44.1,
|
||||||
|
"collector_container_before_mib": 224.4,
|
||||||
|
"during": {
|
||||||
|
"requests": 11500,
|
||||||
|
"codes": {
|
||||||
|
"201": 11500
|
||||||
|
},
|
||||||
|
"success_ratio": 1.0,
|
||||||
|
"success_rps": 68.3,
|
||||||
|
"p95_ms_successful": 13.2
|
||||||
|
},
|
||||||
|
"samples": [
|
||||||
|
{
|
||||||
|
"t": 0.0,
|
||||||
|
"collector_heap_mib": 39.8,
|
||||||
|
"collector_rss_mib": 228.5,
|
||||||
|
"collector_container_mib": 228.6,
|
||||||
|
"queue": {
|
||||||
|
"otlp_http/loki": 0.0,
|
||||||
|
"otlp_http/prometheus": 0.0,
|
||||||
|
"otlp_grpc/tempo": 6.0
|
||||||
|
},
|
||||||
|
"alert": "inactive"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"t": 10.1,
|
||||||
|
"collector_heap_mib": 52.2,
|
||||||
|
"collector_rss_mib": 228.6,
|
||||||
|
"collector_container_mib": 228.6,
|
||||||
|
"queue": {
|
||||||
|
"otlp_http/loki": 5.0,
|
||||||
|
"otlp_http/prometheus": 0.0,
|
||||||
|
"otlp_grpc/tempo": 10.0
|
||||||
|
},
|
||||||
|
"alert": "inactive"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"t": 20.3,
|
||||||
|
"collector_heap_mib": 52.2,
|
||||||
|
"collector_rss_mib": 228.6,
|
||||||
|
"collector_container_mib": 228.7,
|
||||||
|
"queue": {
|
||||||
|
"otlp_http/loki": 5.0,
|
||||||
|
"otlp_http/prometheus": 0.0,
|
||||||
|
"otlp_grpc/tempo": 10.0
|
||||||
|
},
|
||||||
|
"alert": "inactive"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"t": 30.4,
|
||||||
|
"collector_heap_mib": 63.3,
|
||||||
|
"collector_rss_mib": 230.2,
|
||||||
|
"collector_container_mib": 233.6,
|
||||||
|
"queue": {
|
||||||
|
"otlp_http/loki": 12.0,
|
||||||
|
"otlp_http/prometheus": 0.0,
|
||||||
|
"otlp_grpc/tempo": 14.0
|
||||||
|
},
|
||||||
|
"alert": "inactive"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"t": 40.6,
|
||||||
|
"collector_heap_mib": 48.5,
|
||||||
|
"collector_rss_mib": 234.5,
|
||||||
|
"collector_container_mib": 234.5,
|
||||||
|
"queue": {
|
||||||
|
"otlp_http/loki": 20.0,
|
||||||
|
"otlp_http/prometheus": 0.0,
|
||||||
|
"otlp_grpc/tempo": 18.0
|
||||||
|
},
|
||||||
|
"alert": "pending"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"t": 50.7,
|
||||||
|
"collector_heap_mib": 48.5,
|
||||||
|
"collector_rss_mib": 234.5,
|
||||||
|
"collector_container_mib": 234.6,
|
||||||
|
"queue": {
|
||||||
|
"otlp_http/loki": 20.0,
|
||||||
|
"otlp_http/prometheus": 0.0,
|
||||||
|
"otlp_grpc/tempo": 18.0
|
||||||
|
},
|
||||||
|
"alert": "pending"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"t": 60.8,
|
||||||
|
"collector_heap_mib": 59.5,
|
||||||
|
"collector_rss_mib": 234.6,
|
||||||
|
"collector_container_mib": 234.9,
|
||||||
|
"queue": {
|
||||||
|
"otlp_http/loki": 27.0,
|
||||||
|
"otlp_http/prometheus": 0.0,
|
||||||
|
"otlp_grpc/tempo": 22.0
|
||||||
|
},
|
||||||
|
"alert": "pending"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"t": 71.0,
|
||||||
|
"collector_heap_mib": 70.7,
|
||||||
|
"collector_rss_mib": 239.4,
|
||||||
|
"collector_container_mib": 242.2,
|
||||||
|
"queue": {
|
||||||
|
"otlp_http/loki": 35.0,
|
||||||
|
"otlp_http/prometheus": 0.0,
|
||||||
|
"otlp_grpc/tempo": 26.0
|
||||||
|
},
|
||||||
|
"alert": "pending"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"t": 81.2,
|
||||||
|
"collector_heap_mib": 70.7,
|
||||||
|
"collector_rss_mib": 239.4,
|
||||||
|
"collector_container_mib": 247.4,
|
||||||
|
"queue": {
|
||||||
|
"otlp_http/loki": 35.0,
|
||||||
|
"otlp_http/prometheus": 0.0,
|
||||||
|
"otlp_grpc/tempo": 26.0
|
||||||
|
},
|
||||||
|
"alert": "pending"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"t": 91.3,
|
||||||
|
"collector_heap_mib": 58.1,
|
||||||
|
"collector_rss_mib": 247.4,
|
||||||
|
"collector_container_mib": 247.4,
|
||||||
|
"queue": {
|
||||||
|
"otlp_http/loki": 42.0,
|
||||||
|
"otlp_http/prometheus": 0.0,
|
||||||
|
"otlp_grpc/tempo": 30.0
|
||||||
|
},
|
||||||
|
"alert": "pending"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"t": 101.4,
|
||||||
|
"collector_heap_mib": 69.3,
|
||||||
|
"collector_rss_mib": 247.5,
|
||||||
|
"collector_container_mib": 247.5,
|
||||||
|
"queue": {
|
||||||
|
"otlp_http/loki": 50.0,
|
||||||
|
"otlp_http/prometheus": 0.0,
|
||||||
|
"otlp_grpc/tempo": 32.0
|
||||||
|
},
|
||||||
|
"alert": "firing"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"t": 111.6,
|
||||||
|
"collector_heap_mib": 69.3,
|
||||||
|
"collector_rss_mib": 247.5,
|
||||||
|
"collector_container_mib": 249.7,
|
||||||
|
"queue": {
|
||||||
|
"otlp_http/loki": 50.0,
|
||||||
|
"otlp_http/prometheus": 0.0,
|
||||||
|
"otlp_grpc/tempo": 32.0
|
||||||
|
},
|
||||||
|
"alert": "firing"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"t": 121.8,
|
||||||
|
"collector_heap_mib": 80.2,
|
||||||
|
"collector_rss_mib": 249.9,
|
||||||
|
"collector_container_mib": 258.8,
|
||||||
|
"queue": {
|
||||||
|
"otlp_http/loki": 57.0,
|
||||||
|
"otlp_http/prometheus": 0.0,
|
||||||
|
"otlp_grpc/tempo": 36.0
|
||||||
|
},
|
||||||
|
"alert": "firing"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"t": 131.9,
|
||||||
|
"collector_heap_mib": 92.7,
|
||||||
|
"collector_rss_mib": 262.3,
|
||||||
|
"collector_container_mib": 265.4,
|
||||||
|
"queue": {
|
||||||
|
"otlp_http/loki": 63.0,
|
||||||
|
"otlp_http/prometheus": 0.0,
|
||||||
|
"otlp_grpc/tempo": 40.0
|
||||||
|
},
|
||||||
|
"alert": "firing"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"t": 142.1,
|
||||||
|
"collector_heap_mib": 92.7,
|
||||||
|
"collector_rss_mib": 262.3,
|
||||||
|
"collector_container_mib": 273.3,
|
||||||
|
"queue": {
|
||||||
|
"otlp_http/loki": 63.0,
|
||||||
|
"otlp_http/prometheus": 0.0,
|
||||||
|
"otlp_grpc/tempo": 40.0
|
||||||
|
},
|
||||||
|
"alert": "firing"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"collector_heap_max_mib": 92.7,
|
||||||
|
"collector_rss_max_mib": 262.3,
|
||||||
|
"collector_container_max_mib": 273.3,
|
||||||
|
"TelemetryExportFailing_firing_at_s": 101.4,
|
||||||
|
"queue_drained_after_start_s": 80.0,
|
||||||
|
"probe_traces": [
|
||||||
|
{
|
||||||
|
"sent_at_s": 0.0,
|
||||||
|
"http": 201,
|
||||||
|
"trace_in_tempo": true
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"sent_at_s": 10.1,
|
||||||
|
"http": 201,
|
||||||
|
"trace_in_tempo": true
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"sent_at_s": 20.3,
|
||||||
|
"http": 201,
|
||||||
|
"trace_in_tempo": true
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"sent_at_s": 30.4,
|
||||||
|
"http": 201,
|
||||||
|
"trace_in_tempo": true
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"sent_at_s": 40.6,
|
||||||
|
"http": 201,
|
||||||
|
"trace_in_tempo": true
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"sent_at_s": 50.7,
|
||||||
|
"http": 201,
|
||||||
|
"trace_in_tempo": true
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"sent_at_s": 60.8,
|
||||||
|
"http": 201,
|
||||||
|
"trace_in_tempo": true
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"sent_at_s": 71.0,
|
||||||
|
"http": 201,
|
||||||
|
"trace_in_tempo": true
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"sent_at_s": 81.2,
|
||||||
|
"http": 201,
|
||||||
|
"trace_in_tempo": true
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"sent_at_s": 91.3,
|
||||||
|
"http": 201,
|
||||||
|
"trace_in_tempo": true
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"sent_at_s": 101.4,
|
||||||
|
"http": 201,
|
||||||
|
"trace_in_tempo": true
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"sent_at_s": 111.6,
|
||||||
|
"http": 201,
|
||||||
|
"trace_in_tempo": true
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"sent_at_s": 121.7,
|
||||||
|
"http": 201,
|
||||||
|
"trace_in_tempo": true
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"sent_at_s": 131.9,
|
||||||
|
"http": 201,
|
||||||
|
"trace_in_tempo": true
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"sent_at_s": 142.1,
|
||||||
|
"http": 201,
|
||||||
|
"trace_in_tempo": false
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"lost_in_collector": {
|
||||||
|
"spans": 4096,
|
||||||
|
"log_records": 269
|
||||||
|
},
|
||||||
|
"resolved_after_start_s": 155.4,
|
||||||
|
"after": {
|
||||||
|
"requests": 10186,
|
||||||
|
"codes": {
|
||||||
|
"201": 10186
|
||||||
|
},
|
||||||
|
"success_ratio": 1.0,
|
||||||
|
"success_rps": 67.7,
|
||||||
|
"p95_ms_successful": 14.2
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,41 @@
|
|||||||
|
# incident-telemetry: lab6: Tempo+Loki down -> API unaffected, collector memory bounded, losses counted
|
||||||
|
# date: 2026-11-01 00:39:14, commit: unknown
|
||||||
|
# command: python student-tests/incidents.py telemetry-down
|
||||||
|
# exit code: 0, 374.8 s
|
||||||
|
|
||||||
|
=== telemetry-down
|
||||||
|
[ 20.2s] baseline: {'requests': 955, 'codes': {'201': 955}, 'success_ratio': 1.0, 'success_rps': 63.7, 'p95_ms_successful': 17.9}, collector heap 44.1 MiB, container 224.4 MiB
|
||||||
|
[ 50.9s] docker compose stop tempo loki: stopped
|
||||||
|
[ 51.1s] {'t': 0.0, 'collector_heap_mib': 39.8, 'collector_rss_mib': 228.5, 'collector_container_mib': 228.6, 'queue': {'otlp_http/loki': 0.0, 'otlp_http/prometheus': 0.0, 'otlp_grpc/tempo': 6.0}, 'alert': 'inactive'}
|
||||||
|
[ 61.2s] {'t': 10.1, 'collector_heap_mib': 52.2, 'collector_rss_mib': 228.6, 'collector_container_mib': 228.6, 'queue': {'otlp_http/loki': 5.0, 'otlp_http/prometheus': 0.0, 'otlp_grpc/tempo': 10.0}, 'alert': 'inactive'}
|
||||||
|
[ 71.3s] {'t': 20.3, 'collector_heap_mib': 52.2, 'collector_rss_mib': 228.6, 'collector_container_mib': 228.7, 'queue': {'otlp_http/loki': 5.0, 'otlp_http/prometheus': 0.0, 'otlp_grpc/tempo': 10.0}, 'alert': 'inactive'}
|
||||||
|
[ 81.5s] {'t': 30.4, 'collector_heap_mib': 63.3, 'collector_rss_mib': 230.2, 'collector_container_mib': 233.6, 'queue': {'otlp_http/loki': 12.0, 'otlp_http/prometheus': 0.0, 'otlp_grpc/tempo': 14.0}, 'alert': 'inactive'}
|
||||||
|
[ 91.6s] {'t': 40.6, 'collector_heap_mib': 48.5, 'collector_rss_mib': 234.5, 'collector_container_mib': 234.5, 'queue': {'otlp_http/loki': 20.0, 'otlp_http/prometheus': 0.0, 'otlp_grpc/tempo': 18.0}, 'alert': 'pending'}
|
||||||
|
[ 101.8s] {'t': 50.7, 'collector_heap_mib': 48.5, 'collector_rss_mib': 234.5, 'collector_container_mib': 234.6, 'queue': {'otlp_http/loki': 20.0, 'otlp_http/prometheus': 0.0, 'otlp_grpc/tempo': 18.0}, 'alert': 'pending'}
|
||||||
|
[ 112.0s] {'t': 60.8, 'collector_heap_mib': 59.5, 'collector_rss_mib': 234.6, 'collector_container_mib': 234.9, 'queue': {'otlp_http/loki': 27.0, 'otlp_http/prometheus': 0.0, 'otlp_grpc/tempo': 22.0}, 'alert': 'pending'}
|
||||||
|
[ 122.1s] {'t': 71.0, 'collector_heap_mib': 70.7, 'collector_rss_mib': 239.4, 'collector_container_mib': 242.2, 'queue': {'otlp_http/loki': 35.0, 'otlp_http/prometheus': 0.0, 'otlp_grpc/tempo': 26.0}, 'alert': 'pending'}
|
||||||
|
[ 132.2s] {'t': 81.2, 'collector_heap_mib': 70.7, 'collector_rss_mib': 239.4, 'collector_container_mib': 247.4, 'queue': {'otlp_http/loki': 35.0, 'otlp_http/prometheus': 0.0, 'otlp_grpc/tempo': 26.0}, 'alert': 'pending'}
|
||||||
|
[ 142.4s] {'t': 91.3, 'collector_heap_mib': 58.1, 'collector_rss_mib': 247.4, 'collector_container_mib': 247.4, 'queue': {'otlp_http/loki': 42.0, 'otlp_http/prometheus': 0.0, 'otlp_grpc/tempo': 30.0}, 'alert': 'pending'}
|
||||||
|
[ 152.5s] {'t': 101.4, 'collector_heap_mib': 69.3, 'collector_rss_mib': 247.5, 'collector_container_mib': 247.5, 'queue': {'otlp_http/loki': 50.0, 'otlp_http/prometheus': 0.0, 'otlp_grpc/tempo': 32.0}, 'alert': 'firing'}
|
||||||
|
[ 162.7s] {'t': 111.6, 'collector_heap_mib': 69.3, 'collector_rss_mib': 247.5, 'collector_container_mib': 249.7, 'queue': {'otlp_http/loki': 50.0, 'otlp_http/prometheus': 0.0, 'otlp_grpc/tempo': 32.0}, 'alert': 'firing'}
|
||||||
|
[ 172.9s] {'t': 121.8, 'collector_heap_mib': 80.2, 'collector_rss_mib': 249.9, 'collector_container_mib': 258.8, 'queue': {'otlp_http/loki': 57.0, 'otlp_http/prometheus': 0.0, 'otlp_grpc/tempo': 36.0}, 'alert': 'firing'}
|
||||||
|
[ 183.0s] {'t': 131.9, 'collector_heap_mib': 92.7, 'collector_rss_mib': 262.3, 'collector_container_mib': 265.4, 'queue': {'otlp_http/loki': 63.0, 'otlp_http/prometheus': 0.0, 'otlp_grpc/tempo': 40.0}, 'alert': 'firing'}
|
||||||
|
[ 193.2s] {'t': 142.1, 'collector_heap_mib': 92.7, 'collector_rss_mib': 262.3, 'collector_container_mib': 273.3, 'queue': {'otlp_http/loki': 63.0, 'otlp_http/prometheus': 0.0, 'otlp_grpc/tempo': 40.0}, 'alert': 'firing'}
|
||||||
|
[ 219.3s] docker compose start tempo loki
|
||||||
|
[ 299.3s] exporter queues drained 80.0 s after start
|
||||||
|
[ 324.7s] ... TelemetryExportFailing: firing (0 s)
|
||||||
|
[ 329.7s] ... TelemetryExportFailing: firing (5 s)
|
||||||
|
[ 334.7s] ... TelemetryExportFailing: firing (10 s)
|
||||||
|
[ 339.7s] ... TelemetryExportFailing: firing (15 s)
|
||||||
|
[ 344.7s] ... TelemetryExportFailing: firing (20 s)
|
||||||
|
[ 349.7s] ... TelemetryExportFailing: firing (25 s)
|
||||||
|
[ 354.7s] ... TelemetryExportFailing: firing (30 s)
|
||||||
|
[ 359.7s] ... TelemetryExportFailing: firing (35 s)
|
||||||
|
[ 364.7s] ... TelemetryExportFailing: firing (40 s)
|
||||||
|
[ 369.7s] ... TelemetryExportFailing: firing (45 s)
|
||||||
|
[ 374.7s] API during outage: {'requests': 11500, 'codes': {'201': 11500}, 'success_ratio': 1.0, 'success_rps': 68.3, 'p95_ms_successful': 13.2}
|
||||||
|
[ 374.7s] probe traces: 14/15 reached Tempo (lost: sent at [142.1] s)
|
||||||
|
[ 374.7s] lost in Collector (send_failed+enqueue_failed): {'spans': 4096, 'log_records': 269}; collector heap 44.1 -> max 92.7 MiB (memory_limiter 200 MiB), container max 273.3 MiB
|
||||||
|
[ 374.7s] saved /home/claude/work/mtusi2026lab6/evidence/native-sandbox/incident-telemetry-down.json
|
||||||
|
PASS: Tempo+Loki down -> API unaffected, collector memory bounded, losses counted; recovery -> resolved
|
||||||
|
1 passed, 0 failed
|
||||||
@@ -0,0 +1,49 @@
|
|||||||
|
{
|
||||||
|
"scenario": "worker-stop",
|
||||||
|
"date": "2026-11-01 00:26:35",
|
||||||
|
"firing_after_stop_s": {
|
||||||
|
"MessengerWorkerNoConsumers": 66.4,
|
||||||
|
"MessengerQueueLagGrowing": 111.4
|
||||||
|
},
|
||||||
|
"diagnosis": {
|
||||||
|
"queue_consumers": 0.0,
|
||||||
|
"broker_queue_depth": 3.0,
|
||||||
|
"jobs_backlog_queued": 3.0,
|
||||||
|
"oldest_job_age_s": 97.3,
|
||||||
|
"outbox_oldest_age_s": 0.0,
|
||||||
|
"published_ok_last_5m": 3.102218085931441,
|
||||||
|
"worker_metrics_last_seen_s": 110.0,
|
||||||
|
"last_worker_log": {
|
||||||
|
"event": "service.stopped",
|
||||||
|
"message": "stopped"
|
||||||
|
},
|
||||||
|
"upload_trace_span_names": [
|
||||||
|
"INSERT",
|
||||||
|
"NOTIFY",
|
||||||
|
"POST /api/v1/chats/{chat_id}/attachments",
|
||||||
|
"S3 PutObject",
|
||||||
|
"SELECT",
|
||||||
|
"UPDATE",
|
||||||
|
"create attachments",
|
||||||
|
"inspect image",
|
||||||
|
"publish attachments",
|
||||||
|
"redis GET"
|
||||||
|
],
|
||||||
|
"upload_trace_has_consumer_span": false
|
||||||
|
},
|
||||||
|
"ready_after_start_s": [
|
||||||
|
1.5,
|
||||||
|
1.6,
|
||||||
|
2.0
|
||||||
|
],
|
||||||
|
"resolved_after_start_s": {
|
||||||
|
"MessengerWorkerNoConsumers": 17.0,
|
||||||
|
"MessengerQueueLagGrowing": 17.0
|
||||||
|
},
|
||||||
|
"after_recovery": {
|
||||||
|
"jobs_backlog_queued": 0.0,
|
||||||
|
"oldest_job_age_s": 0.0,
|
||||||
|
"queue_consumers": 2.0
|
||||||
|
},
|
||||||
|
"time_to_ready_p95_5m_s": 117.0
|
||||||
|
}
|
||||||
@@ -0,0 +1,43 @@
|
|||||||
|
# incident-worker: lab6 incident 1: worker stopped -> queue lag + no consumers firing -> resolved
|
||||||
|
# date: 2026-11-01 00:28:44, commit: unknown
|
||||||
|
# command: python student-tests/incidents.py worker-stop
|
||||||
|
# exit code: 0, 128.6 s
|
||||||
|
|
||||||
|
=== worker-stop
|
||||||
|
[ 0.1s] baseline: consumers=2.0, oldest job age=0.0 s
|
||||||
|
[ 1.3s] docker compose stop worker
|
||||||
|
[ 1.4s] uploaded 3 attachments: 202 queued (API accepts without worker)
|
||||||
|
[ 1.4s] consumers=2.0 backlog queued=0.0 oldest age=0.0 s, alerts: {}
|
||||||
|
[ 6.4s] consumers=0.0 backlog queued=3.0 oldest age=2.1 s, alerts: {'MessengerWorkerNoConsumers': 'pending'}
|
||||||
|
[ 11.4s] consumers=0.0 backlog queued=3.0 oldest age=2.1 s, alerts: {'MessengerWorkerNoConsumers': 'pending'}
|
||||||
|
[ 16.4s] consumers=0.0 backlog queued=3.0 oldest age=12.1 s, alerts: {'MessengerWorkerNoConsumers': 'pending'}
|
||||||
|
[ 21.4s] consumers=0.0 backlog queued=3.0 oldest age=12.1 s, alerts: {'MessengerWorkerNoConsumers': 'pending'}
|
||||||
|
[ 26.4s] consumers=0.0 backlog queued=3.0 oldest age=22.1 s, alerts: {'MessengerWorkerNoConsumers': 'pending'}
|
||||||
|
[ 31.4s] consumers=0.0 backlog queued=3.0 oldest age=22.1 s, alerts: {'MessengerWorkerNoConsumers': 'pending'}
|
||||||
|
[ 36.4s] consumers=0.0 backlog queued=3.0 oldest age=27.1 s, alerts: {'MessengerWorkerNoConsumers': 'pending'}
|
||||||
|
[ 41.4s] consumers=0.0 backlog queued=3.0 oldest age=27.1 s, alerts: {'MessengerWorkerNoConsumers': 'pending'}
|
||||||
|
[ 46.4s] consumers=0.0 backlog queued=3.0 oldest age=37.1 s, alerts: {'MessengerWorkerNoConsumers': 'pending'}
|
||||||
|
[ 51.4s] consumers=0.0 backlog queued=3.0 oldest age=37.1 s, alerts: {'MessengerQueueLagGrowing': 'pending', 'MessengerWorkerNoConsumers': 'pending'}
|
||||||
|
[ 56.4s] consumers=0.0 backlog queued=3.0 oldest age=47.2 s, alerts: {'MessengerQueueLagGrowing': 'pending', 'MessengerWorkerNoConsumers': 'pending'}
|
||||||
|
[ 61.5s] consumers=0.0 backlog queued=3.0 oldest age=47.2 s, alerts: {'MessengerQueueLagGrowing': 'pending', 'MessengerWorkerNoConsumers': 'pending'}
|
||||||
|
[ 66.5s] ALERT MessengerWorkerNoConsumers firing (66.4 s after stop)
|
||||||
|
[ 66.5s] consumers=0.0 backlog queued=3.0 oldest age=57.2 s, alerts: {'MessengerQueueLagGrowing': 'pending', 'MessengerWorkerNoConsumers': 'firing'}
|
||||||
|
[ 71.5s] consumers=0.0 backlog queued=3.0 oldest age=57.2 s, alerts: {'MessengerQueueLagGrowing': 'pending', 'MessengerWorkerNoConsumers': 'firing'}
|
||||||
|
[ 76.5s] consumers=0.0 backlog queued=3.0 oldest age=67.2 s, alerts: {'MessengerQueueLagGrowing': 'pending', 'MessengerWorkerNoConsumers': 'firing'}
|
||||||
|
[ 81.5s] consumers=0.0 backlog queued=3.0 oldest age=67.2 s, alerts: {'MessengerQueueLagGrowing': 'pending', 'MessengerWorkerNoConsumers': 'firing'}
|
||||||
|
[ 86.5s] consumers=0.0 backlog queued=3.0 oldest age=77.2 s, alerts: {'MessengerQueueLagGrowing': 'pending', 'MessengerWorkerNoConsumers': 'firing'}
|
||||||
|
[ 91.5s] consumers=0.0 backlog queued=3.0 oldest age=77.2 s, alerts: {'MessengerQueueLagGrowing': 'pending', 'MessengerWorkerNoConsumers': 'firing'}
|
||||||
|
[ 96.5s] consumers=0.0 backlog queued=3.0 oldest age=87.2 s, alerts: {'MessengerQueueLagGrowing': 'pending', 'MessengerWorkerNoConsumers': 'firing'}
|
||||||
|
[ 101.5s] consumers=0.0 backlog queued=3.0 oldest age=87.2 s, alerts: {'MessengerQueueLagGrowing': 'pending', 'MessengerWorkerNoConsumers': 'firing'}
|
||||||
|
[ 106.5s] consumers=0.0 backlog queued=3.0 oldest age=97.3 s, alerts: {'MessengerQueueLagGrowing': 'pending', 'MessengerWorkerNoConsumers': 'firing'}
|
||||||
|
[ 111.5s] ALERT MessengerQueueLagGrowing firing (111.4 s after stop)
|
||||||
|
[ 111.5s] diagnosis: {"queue_consumers": 0.0, "broker_queue_depth": 3.0, "jobs_backlog_queued": 3.0, "oldest_job_age_s": 97.3, "outbox_oldest_age_s": 0.0, "published_ok_last_5m": 3.102218085931441, "worker_metrics_last_seen_s": 110.0, "last_worker_log": {"event": "service.stopped", "message": "stopped"}, "upload_trace_span_names": ["INSERT", "NOTIFY", "POST /api/v1/chats/{chat_id}/attachments", "S3 PutObject", "SELECT", "UPDATE", "create attachments", "inspect image", "publish attachments", "redis GET"], "upload_trace_has_consumer_span": false}
|
||||||
|
[ 111.6s] docker compose start worker
|
||||||
|
[ 113.5s] all attachments ready 2.0 s after worker start; backlog now 3.0
|
||||||
|
[ 113.5s] ... MessengerWorkerNoConsumers: firing (0 s)
|
||||||
|
[ 118.5s] ... MessengerWorkerNoConsumers: firing (5 s)
|
||||||
|
[ 123.5s] ... MessengerWorkerNoConsumers: firing (10 s)
|
||||||
|
[ 128.5s] alerts resolved: {'MessengerWorkerNoConsumers': 17.0, 'MessengerQueueLagGrowing': 17.0}; now {'jobs_backlog_queued': 0.0, 'oldest_job_age_s': 0.0, 'queue_consumers': 2.0}
|
||||||
|
[ 128.5s] saved /home/claude/work/mtusi2026lab6/evidence/native-sandbox/incident-worker-stop.json
|
||||||
|
PASS: worker stop -> no consumers + growing queue lag -> firing; cause from metrics/logs/trace; start -> backlog drained -> resolved
|
||||||
|
1 passed, 0 failed
|
||||||
@@ -0,0 +1,8 @@
|
|||||||
|
# jobs-concurrency: lab5: WORKER_CONCURRENCY consumers, prefetch 1, bounded processing
|
||||||
|
# date: 2026-11-01 00:52:34, commit: unknown
|
||||||
|
# command: python student-tests/test_jobs.py concurrency
|
||||||
|
# exit code: 0, 7.1 s
|
||||||
|
|
||||||
|
--- concurrency: Ограниченная конкуренция: WORKER_CONCURRENCY потребителей с prefetch=1; одновременно processing <= 2.
|
||||||
|
2 consumers, prefetch_count 1, manual ack; 8 queued 3000x2000 JPEG drained in 4.1 s after worker start, peak processing 2 of 8 (22 samples)
|
||||||
|
1 passed, 0 failed
|
||||||
@@ -0,0 +1,9 @@
|
|||||||
|
# jobs-outbox: lab5: dispatcher crash between COMMIT and publish / after confirm
|
||||||
|
# date: 2026-11-01 00:51:38, commit: unknown
|
||||||
|
# command: python student-tests/test_jobs.py outbox-crash
|
||||||
|
# exit code: 0, 7.4 s
|
||||||
|
|
||||||
|
--- outbox-crash: Разрыв COMMIT/publish: dispatcher падает после COMMIT задания до publish и после confirm до отметки.
|
||||||
|
dispatcher killed between COMMIT and publish; restarted by policy, outbox re-published -> job 18ab06f7-657b-4508-9ba5-844bd2c7ba2e ready, attempt 1, no manual action
|
||||||
|
dispatcher killed after publisher confirm, before marking outbox -> message published twice; worker: ['created', 'claimed', 'ready', 'duplicate_ignored'] -> one result set
|
||||||
|
1 passed, 0 failed
|
||||||
@@ -0,0 +1,16 @@
|
|||||||
|
# jobs-recovery: lab5: worker down -> 202 -> restart api+broker -> ready; broker down -> outbox waits
|
||||||
|
# date: 2026-11-01 00:48:13, commit: unknown
|
||||||
|
# command: python student-tests/test_jobs.py worker-down broker-down
|
||||||
|
# exit code: 0, 44.2 s
|
||||||
|
|
||||||
|
--- worker-down: Защита п.2-3: worker остановлен -> 202 -> сообщение с вложением -> рестарт API+broker -> worker -> ready.
|
||||||
|
worker stopped
|
||||||
|
upload while worker is down -> 202 queued, attachment 748b0b56-e355-48a0-9ac6-4b731a3432a9
|
||||||
|
message with attachment 201, content 409, other messages 201; job 87ddc8fa-f8fe-4241-bf18-95fd2d4a2762 published, queue messages/unacked = (1, 0)
|
||||||
|
rabbitmq and api restarted with volumes; queue still holds 1 message(s); status queued
|
||||||
|
worker started -> same job 87ddc8fa-f8fe-4241-bf18-95fd2d4a2762 ready, thumbnail downloadable; staging object deleted
|
||||||
|
--- broker-down: Брокер недоступен: API принимает (outbox), dispatcher копит и публикует после восстановления.
|
||||||
|
rabbitmq stopped
|
||||||
|
upload -> 202 and /health/ready 200 without broker; outbox row pending for job 5784e1e7-7173-4a27-a3ad-72216f4a43a9
|
||||||
|
rabbitmq started -> dispatcher published the pending row -> ready
|
||||||
|
2 passed, 0 failed
|
||||||
@@ -0,0 +1,15 @@
|
|||||||
|
# jobs-retry: lab5: backoff+jitter, retry_exhausted, replay, permanent errors, DLQ
|
||||||
|
# date: 2026-11-01 00:52:27, commit: unknown
|
||||||
|
# command: python student-tests/test_jobs.py retry exhausted-replay permanent dlq
|
||||||
|
# exit code: 0, 49.1 s
|
||||||
|
|
||||||
|
--- retry: Временные ошибки S3: backoff с jitter, затем ready; задержки видны в журнале.
|
||||||
|
2 x storage_unavailable -> retries after 1.40 s and 3.50 s (equal jitter: 1..2 s, 2..4 s), attempt 3 ready
|
||||||
|
--- exhausted-replay: Исчерпание попыток -> failed/retry_exhausted, реестр failed, явный replay; replay ready — отказ.
|
||||||
|
5 temporary errors -> failed/retry_exhausted (public error_code), content 409, attach 409, listed by ops.py failed (replayable)
|
||||||
|
ops replay -> same job_id fce6904a-fe4b-429c-821a-e3b55a43b0ef attempt 6 ready; second replay refused: "job is ready, replay is only for failed jobs"
|
||||||
|
--- permanent: Постоянная ошибка (битое изображение в worker) -> failed сразу, без повторов.
|
||||||
|
invalid image in the worker -> failed/invalid_image at attempt 1, no retry (permanent error)
|
||||||
|
--- dlq: Невалидные сообщения и сообщения о неизвестной job -> DLQ; dlq-replay их не исполняет.
|
||||||
|
schema-invalid message and unknown job_id -> reject -> DLQ (2 message(s)); dlq-replay kept them: ['invalid message is kept for analysis', 'unknown job']
|
||||||
|
4 passed, 0 failed
|
||||||
@@ -0,0 +1,14 @@
|
|||||||
|
# jobs-variants: lab5: thumbnail/crop/optimized, EXIF orientation, alpha, no upscale, metadata stripped
|
||||||
|
# date: 2026-11-01 00:47:29, commit: unknown
|
||||||
|
# command: python student-tests/test_jobs.py variants
|
||||||
|
# exit code: 0, 14.3 s
|
||||||
|
|
||||||
|
--- variants: thumbnail/crop/optimized по контракту, EXIF orientation, alpha -> белый, без увеличения, без метаданных.
|
||||||
|
thumb320: thumbnail 256x160, crop 128x128, optimized 320x200; original byte-identical; EXIF/XMP in variants: 0; S3 keys ['v1/crop.jpg', 'v1/optimized.webp', 'v1/original', 'v1/thumbnail.jpg']
|
||||||
|
exif6: thumbnail 192x256, crop 128x128, optimized 300x400; original byte-identical; EXIF/XMP in variants: 0; S3 keys ['v1/crop.jpg', 'v1/optimized.webp', 'v1/original', 'v1/thumbnail.jpg']
|
||||||
|
alpha: thumbnail 256x170, crop 128x128, optimized 300x200; original byte-identical; EXIF/XMP in variants: 0; S3 keys ['v1/crop.jpg', 'v1/optimized.webp', 'v1/original', 'v1/thumbnail.jpg']
|
||||||
|
tiny: thumbnail 40x30, crop 128x128, optimized 40x30; original byte-identical; EXIF/XMP in variants: 0; S3 keys ['v1/crop.jpg', 'v1/optimized.webp', 'v1/original', 'v1/thumbnail.jpg']
|
||||||
|
large: thumbnail 256x170, crop 128x128, optimized 1280x853; original byte-identical; EXIF/XMP in variants: 0; S3 keys ['v1/crop.jpg', 'v1/optimized.webp', 'v1/original', 'v1/thumbnail.jpg']
|
||||||
|
strip: thumbnail 256x1, crop 128x128, optimized 1280x1; original byte-identical; EXIF/XMP in variants: 0; S3 keys ['v1/crop.jpg', 'v1/optimized.webp', 'v1/original', 'v1/thumbnail.jpg']
|
||||||
|
unknown variant name -> 400; variants carry their own size/sha256, original fields describe the upload
|
||||||
|
1 passed, 0 failed
|
||||||
@@ -0,0 +1,17 @@
|
|||||||
|
{
|
||||||
|
"lab": 6,
|
||||||
|
"workload": "closed-loop POST messages; no automatic retries",
|
||||||
|
"requested_duration_seconds": 30.0,
|
||||||
|
"elapsed_seconds": 30.083,
|
||||||
|
"concurrency": 4,
|
||||||
|
"successful_requests": 8762,
|
||||||
|
"failed_requests": 0,
|
||||||
|
"error_ratio": 0.0,
|
||||||
|
"successful_rps": 291.265,
|
||||||
|
"successful_latency_ms": {
|
||||||
|
"p50": 11.557,
|
||||||
|
"p95": 26.659,
|
||||||
|
"p99": 56.425
|
||||||
|
},
|
||||||
|
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
|
||||||
|
}
|
||||||
@@ -0,0 +1,22 @@
|
|||||||
|
# load: public load generator, 30 s, concurrency 4
|
||||||
|
# date: 2026-11-01 00:42:08, commit: unknown
|
||||||
|
# command: python tests/load.py --base-url http://127.0.0.1:8080 --duration 30 --concurrency 4 --output evidence/load.json
|
||||||
|
# exit code: 0, 30.4 s
|
||||||
|
|
||||||
|
{
|
||||||
|
"lab": 6,
|
||||||
|
"workload": "closed-loop POST messages; no automatic retries",
|
||||||
|
"requested_duration_seconds": 30.0,
|
||||||
|
"elapsed_seconds": 30.083,
|
||||||
|
"concurrency": 4,
|
||||||
|
"successful_requests": 8762,
|
||||||
|
"failed_requests": 0,
|
||||||
|
"error_ratio": 0.0,
|
||||||
|
"successful_rps": 291.265,
|
||||||
|
"successful_latency_ms": {
|
||||||
|
"p50": 11.557,
|
||||||
|
"p95": 26.659,
|
||||||
|
"p99": 56.425
|
||||||
|
},
|
||||||
|
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
|
||||||
|
}
|
||||||
@@ -0,0 +1,10 @@
|
|||||||
|
# race: lab2 races via the balancer
|
||||||
|
# date: 2026-11-01 00:52:53, commit: unknown
|
||||||
|
# command: python student-tests/test_race.py
|
||||||
|
# exit code: 0, 1.5 s
|
||||||
|
|
||||||
|
sent 100 messages from 10 clients in 0.34 s; read 100 in 4 pages, unique ids: 100
|
||||||
|
PASS: 100 concurrent messages from 10 clients, no losses and no duplicates
|
||||||
|
20 parallel registrations of one username: 201 x1, 409 x19
|
||||||
|
PASS: concurrent registration created exactly one user
|
||||||
|
2 passed, 0 failed
|
||||||
@@ -0,0 +1,6 @@
|
|||||||
|
# smoke: public smoke via the balancer (make test) with telemetry ON, with attachment
|
||||||
|
# date: 2026-11-01 00:22:30, commit: unknown
|
||||||
|
# command: python tests/smoke.py --base-url http://127.0.0.1:8080
|
||||||
|
# exit code: 0, 1.7 s
|
||||||
|
|
||||||
|
PASS: lab6 public API scenario; infrastructure and grading evidence remain separate.
|
||||||
@@ -0,0 +1,26 @@
|
|||||||
|
# student-tests: lab1/lab2 API tests via the balancer
|
||||||
|
# date: 2026-11-01 00:52:52, commit: unknown
|
||||||
|
# command: python student-tests/test_api.py
|
||||||
|
# exit code: 0, 2.0 s
|
||||||
|
|
||||||
|
PASS: duplicate username returns 409
|
||||||
|
PASS: invalid JSON returns 400
|
||||||
|
PASS: wrong and extra fields return 400
|
||||||
|
PASS: foreign chat returns 404
|
||||||
|
PASS: empty and too long messages return 400, 4000 chars is ok
|
||||||
|
PASS: pagination works across multiple pages, foreign cursor returns 400
|
||||||
|
PASS: empty cursor (?cursor= and ?cursor) returns 400, no cursor returns the first page
|
||||||
|
PASS: chat list pagination
|
||||||
|
PASS: unicode text is stored as is
|
||||||
|
PASS: strings with U+0000 are stored and returned as is
|
||||||
|
PASS: user response has no password or hash
|
||||||
|
PASS: password length 12..128, blank password is rejected
|
||||||
|
PASS: Basic login returns opaque Bearer token with expiry
|
||||||
|
PASS: wrong password and unknown user both return the same 401
|
||||||
|
PASS: malformed Basic header returns 401
|
||||||
|
PASS: password with colon and unicode works
|
||||||
|
PASS: business endpoints need Bearer; Basic and X-User-Id give 401
|
||||||
|
PASS: X-User-Id does not change the current user
|
||||||
|
PASS: 401 comes before 400 and 404
|
||||||
|
PASS: logout revokes only the current session
|
||||||
|
20 passed, 0 failed
|
||||||
@@ -0,0 +1,48 @@
|
|||||||
|
# Сводка проверок (нативная песочница, без Docker)
|
||||||
|
|
||||||
|
Стенд теми же процессами без контейнеров: PostgreSQL 16, Redis 7, RabbitMQ 3.12, SeaweedFS 3.80, HAProxy 2.8, 2 реплики API, dispatcher, worker, janitor (Python 3.11, requirements.txt), otelcol-contrib 0.164.0, Prometheus 3.15.1, Loki 3.7.9, Tempo 3.1.1 с конфигами из infra/ (адреса -> 127.0.0.1). Команды `docker compose` тестов выполнялись через подмену на управление процессами. Машина: Linux x86_64, 2 vCPU, 8 GiB RAM.
|
||||||
|
|
||||||
|
| Шаг | Что проверяет | Результат | Время, с | Файл |
|
||||||
|
| --- | --- | --- | --- | --- |
|
||||||
|
| check | offline contract check (make check) | PASS | 0 | evidence/native-sandbox/check.txt |
|
||||||
|
| smoke | public smoke via the balancer (make test) with telemetry ON, with attachment | PASS | 2 | evidence/native-sandbox/smoke.txt |
|
||||||
|
| configs | lab6: collector/prometheus/loki/tempo configs by their own tools, alert unit tests, Grafana provisioning | PASS | 1 | evidence/native-sandbox/configs.txt |
|
||||||
|
| telemetry | lab6: request<->log<->trace, upload->publish->consume->S3/DB chain, retry, replay link, no secrets | PASS | 77 | evidence/native-sandbox/telemetry.txt |
|
||||||
|
| dashboard | lab6: every dashboard/rule query runs on live data, key signals present, no forbidden labels | PASS | 0 | evidence/native-sandbox/dashboard.txt |
|
||||||
|
| telemetry-history | lab6: traces/logs/metrics survive restart of prometheus/loki/tempo (volumes) | PASS | 52 | evidence/native-sandbox/telemetry-history.txt |
|
||||||
|
| incident-worker | lab6 incident 1: worker stopped -> queue lag + no consumers firing -> resolved | PASS | 129 | evidence/native-sandbox/incident-worker.txt |
|
||||||
|
| incident-db | lab6 incident 2: slow PostgreSQL (table lock) -> high error rate firing, cause from traces -> resolved | PASS | 265 | evidence/native-sandbox/incident-db.txt |
|
||||||
|
| incident-telemetry | lab6: Tempo+Loki down -> API unaffected, collector memory bounded, losses counted | PASS | 375 | evidence/native-sandbox/incident-telemetry.txt |
|
||||||
|
| incident-collector | lab6: Collector down -> API unaffected, SDK queue bounded, drops counted | PASS | 143 | evidence/native-sandbox/incident-collector.txt |
|
||||||
|
| jobs-variants | lab5: thumbnail/crop/optimized, EXIF orientation, alpha, no upscale, metadata stripped | PASS | 14 | evidence/native-sandbox/jobs-variants.txt |
|
||||||
|
| jobs-recovery | lab5: worker down -> 202 -> restart api+broker -> ready; broker down -> outbox waits | PASS | 44 | evidence/native-sandbox/jobs-recovery.txt |
|
||||||
|
| jobs-outbox | lab5: dispatcher crash between COMMIT and publish / after confirm | PASS | 7 | evidence/native-sandbox/jobs-outbox.txt |
|
||||||
|
| jobs-retry | lab5: backoff+jitter, retry_exhausted, replay, permanent errors, DLQ | PASS | 49 | evidence/native-sandbox/jobs-retry.txt |
|
||||||
|
| jobs-concurrency | lab5: WORKER_CONCURRENCY consumers, prefetch 1, bounded processing | PASS | 7 | evidence/native-sandbox/jobs-concurrency.txt |
|
||||||
|
| attachments | lab4: files, limits, damaged images, MIME, filenames, access, attachment_ids | PASS | 14 | evidence/native-sandbox/attachments.txt |
|
||||||
|
| idempotency | Idempotency-Key, parallel same key via replicas | PASS | 2 | evidence/native-sandbox/idempotency.txt |
|
||||||
|
| student-tests | lab1/lab2 API tests via the balancer | PASS | 2 | evidence/native-sandbox/student-tests.txt |
|
||||||
|
| race | lab2 races via the balancer | PASS | 2 | evidence/native-sandbox/race.txt |
|
||||||
|
| load | public load generator, 30 s, concurrency 4 | PASS | 30 | evidence/native-sandbox/load.txt |
|
||||||
|
|
||||||
|
# Замеры
|
||||||
|
|
||||||
|
Алерты на контролируемых инцидентах (`student-tests/incidents.py`, состояние из Prometheus /api/v1/alerts):
|
||||||
|
|
||||||
|
| Инцидент | Воздействие | Алерт | firing через, с | resolved через, с после устранения |
|
||||||
|
| --- | --- | --- | --- | --- |
|
||||||
|
| 1. worker | `docker compose stop worker` + 3 загрузки | MessengerWorkerNoConsumers | 66.4 (от stop) | 17.0 (от start) |
|
||||||
|
| 1. worker | `docker compose stop worker` + 3 загрузки | MessengerQueueLagGrowing | 111.4 (от stop) | 17.0 (от start) |
|
||||||
|
| 2. PostgreSQL | `LOCK TABLE messages IN ACCESS EXCLUSIVE MODE` на 210 с под нагрузкой 8 потоков | MessengerHighErrorRate | 120.1 (от LOCK) | 30.0 (от COMMIT блокировки) |
|
||||||
|
| 3. телеметрия | `docker compose stop tempo loki` на 150 с | TelemetryExportFailing | 101.4 (от stop) | 155.4 (от start) |
|
||||||
|
|
||||||
|
Инцидент 1, наблюдения в момент firing: consumers очереди = 0.0, сообщений в RabbitMQ = 3.0, queued заданий = 3.0, возраст старейшего = 97.3 с, возраст outbox = 0.0 с (публикация работает), метрики worker не обновлялись 110.0 с, последний лог worker: event=`service.stopped`; трасса загрузки содержит `publish attachments`, но не содержит `process attachments.process.v1`. После start: вложения ready через 2.0 с, backlog 0.0.
|
||||||
|
|
||||||
|
Инцидент 2, наблюдения: доля 5xx (1m) = 1.0, коды POST /messages за 1m = {'201': 0.0, '200': 0.0, '404': 0.0, '400': 0.0, '409': 0.0, '503': 1.6}, p95 POST /messages = 7.375 с, чтения других таблиц (GET /users/me, /chats) во время блокировки {'GET /users/me + GET /chats, avg per request': 4.7, 'max': 5.4} мс, занято соединений пула = 8.0, p95 ожидания пула = 0.001 с, логи API event=db.unavailable по error_type: {'QueryCanceled': 184}. Error-трасса 1c7f8dd06427574a5f155558aa4cd975: server-спан 5006.7 мс, из них `INSERT` 5000.6 мс (доля 0.999, статус ERROR). Контраст «медленный обработчик» (трасса a6da2e3fe7b0c229773f36fa507293ca, POST /users): server 34.0 мс, `argon2id hash` 29.0 мс, все спаны БД 0.6 мс. Нагрузка: до 132.4 успешных записей/с, во время блокировки {'503': 320, '201': 6}, после 131.9/с без ошибок.
|
||||||
|
|
||||||
|
| Отказ телеметрии | API во время отказа | Память | Потери телеметрии |
|
||||||
|
| --- | --- | --- | --- |
|
||||||
|
| `stop tempo loki` на 150 с (окно повторов Collector 120 с) | 11500 запросов, коды {'201': 11500}, доля успеха 1.0 | heap Collector 44.1 -> max 92.7 MiB (limit 200), контейнер max 273.3 MiB | send/enqueue failed в Collector: {'spans': 4096, 'log_records': 269}; пробные трассы дошли 14/15, потеряны отправленные на [142.1] с |
|
||||||
|
| `stop otel-collector` на 90 с | 6263 запросов, коды {'201': 6263}, доля успеха 1.0 | api 280.5 -> max 292.5 MiB (2 реплики) | SDK api потерял спанов: {'processor_queue_full': 0, 'exporter_failed': 24576} (queue_full — очередь 2048 полна, exporter_failed — Collector не принял пакет); после start экспорт возобновился (10524 спанов за минуту) |
|
||||||
|
|
||||||
|
Публичный генератор нагрузки при включённой телеметрии (`tests/load.py`, closed loop, concurrency 4, 30 с, 2 реплики): 291.265 успешных POST/с, ошибок 0, p50 11.557 мс, p95 26.659 мс, p99 56.425 мс (песочница 2 vCPU на всё: стенд + хранилища + клиент; не сравнивать с Docker-машиной).
|
||||||
@@ -0,0 +1,9 @@
|
|||||||
|
# telemetry-history: lab6: traces/logs/metrics survive restart of prometheus/loki/tempo (volumes)
|
||||||
|
# date: 2026-11-01 00:26:35, commit: unknown
|
||||||
|
# command: python student-tests/test_telemetry.py history
|
||||||
|
# exit code: 0, 52.2 s
|
||||||
|
|
||||||
|
before restart: trace 52668075eebc7fd4938de9690ba2aad1 found, its log found, sum(http_server_request_duration_seconds_count{job="messenger-api"}) at t-30s = 719
|
||||||
|
after restart of prometheus, loki, tempo: trace 52668075eebc7fd4938de9690ba2aad1 (4 spans), 1 log record(s), metric at t-30s = 719 (same value)
|
||||||
|
PASS: persistent volumes keep traces, logs and metrics across restarts of the stores
|
||||||
|
1 passed, 0 failed
|
||||||
@@ -0,0 +1,145 @@
|
|||||||
|
# telemetry: lab6: request<->log<->trace, upload->publish->consume->S3/DB chain, retry, replay link, no secrets
|
||||||
|
# date: 2026-11-01 00:23:48, commit: unknown
|
||||||
|
# command: python student-tests/test_telemetry.py request upload-chain retry-chain replay-link
|
||||||
|
# exit code: 0, 77.1 s
|
||||||
|
|
||||||
|
POST message: 201, X-Request-Id=4d281ed1-39de-484a-add2-8480ef33ab9c, trace_id=97ebaba763fec926fa1e1ed3fc0fae7a (client traceparent)
|
||||||
|
- [api] POST /api/v1/chats/{chat_id}/messages (SERVER, 5.8 ms) http.response.status_code=201
|
||||||
|
- [api] redis GET (CLIENT, 0.3 ms)
|
||||||
|
- [api] SELECT (CLIENT, 0.5 ms)
|
||||||
|
- [api] INSERT (CLIENT, 0.8 ms)
|
||||||
|
Loki: 'POST /api/v1/chats/{chat_id}/messages -> 201 (5.9 ms)' event=http.request severity=INFO service=messenger-api trace_id=97ebaba763fec926fa1e1ed3fc0fae7a span_id=82ab66080cb50d2a request_id=4d281ed1-39de-484a-add2-8480ef33ab9c
|
||||||
|
PASS: request -> server span with DB/Redis children -> structured log with the same trace_id/span_id and request_id; no message text, password or token in logs/span attributes
|
||||||
|
- [api] POST /api/v1/chats/{chat_id}/attachments (SERVER, 64.6 ms) http.response.status_code=202
|
||||||
|
- [api] redis GET (CLIENT, 0.4 ms)
|
||||||
|
- [api] SELECT (CLIENT, 0.5 ms)
|
||||||
|
- [api] inspect image (INTERNAL, 45.7 ms)
|
||||||
|
- [api] INSERT (CLIENT, 1.1 ms)
|
||||||
|
- [api] S3 PutObject (CLIENT, 5.1 ms) aws.s3.key=staging/f5bdb54e2b234121b7e13ef3007a03b1
|
||||||
|
- [api] UPDATE (CLIENT, 0.9 ms)
|
||||||
|
- [api] create attachments (PRODUCER, 3.5 ms) messenger.job.attempt=1
|
||||||
|
- [api] INSERT (CLIENT, 0.9 ms)
|
||||||
|
- [api] INSERT (CLIENT, 0.9 ms)
|
||||||
|
- [api] INSERT (CLIENT, 0.8 ms)
|
||||||
|
- [api] NOTIFY (CLIENT, 0.3 ms)
|
||||||
|
- [dispatcher] publish attachments (PRODUCER, 4.4 ms) messenger.job.attempt=1
|
||||||
|
- [dispatcher] UPDATE (CLIENT, 0.9 ms)
|
||||||
|
- [worker] process attachments.process.v1 (CONSUMER, 280.0 ms) messenger.job.attempt=1 messenger.job.outcome=ready
|
||||||
|
- [worker] SELECT (CLIENT, 1.4 ms)
|
||||||
|
- [worker] UPDATE (CLIENT, 0.8 ms)
|
||||||
|
- [worker] UPDATE (CLIENT, 1.4 ms)
|
||||||
|
- [worker] INSERT (CLIENT, 0.7 ms)
|
||||||
|
- [worker] S3 GetObject (CLIENT, 2.9 ms) aws.s3.key=staging/f5bdb54e2b234121b7e13ef3007a03b1
|
||||||
|
- [worker] verify source sha256 (INTERNAL, 0.1 ms)
|
||||||
|
- [worker] imaging (INTERNAL, 238.2 ms)
|
||||||
|
- [worker] S3 PutObject (CLIENT, 4.2 ms) aws.s3.key=attachments/016e0413-eeaa-481a-bef5-27270d212a71/v1/original
|
||||||
|
- [worker] S3 PutObject (CLIENT, 3.4 ms) aws.s3.key=attachments/016e0413-eeaa-481a-bef5-27270d212a71/v1/optimized.webp
|
||||||
|
- [worker] S3 PutObject (CLIENT, 3.2 ms) aws.s3.key=attachments/016e0413-eeaa-481a-bef5-27270d212a71/v1/thumbnail.jpg
|
||||||
|
- [worker] S3 PutObject (CLIENT, 2.9 ms) aws.s3.key=attachments/016e0413-eeaa-481a-bef5-27270d212a71/v1/crop.jpg
|
||||||
|
- [worker] commit result (INTERNAL, 7.5 ms)
|
||||||
|
- [worker] SELECT (CLIENT, 0.7 ms)
|
||||||
|
- [worker] INSERT (CLIENT, 1.0 ms)
|
||||||
|
- [worker] INSERT (CLIENT, 0.6 ms)
|
||||||
|
- [worker] INSERT (CLIENT, 0.7 ms)
|
||||||
|
- [worker] UPDATE (CLIENT, 1.0 ms)
|
||||||
|
- [worker] UPDATE (CLIENT, 0.7 ms)
|
||||||
|
- [worker] INSERT (CLIENT, 0.4 ms)
|
||||||
|
- [worker] cleanup staging (INTERNAL, 4.1 ms)
|
||||||
|
- [worker] S3 DeleteObject (CLIENT, 2.4 ms) aws.s3.key=staging/f5bdb54e2b234121b7e13ef3007a03b1
|
||||||
|
- [worker] UPDATE (CLIENT, 1.4 ms)
|
||||||
|
worker logs with trace_id 0eecc096d0191a19e41ad1547110183c: ['job.claimed', 'job.ready']
|
||||||
|
PASS: one trace 0eecc096d0191a19e41ad1547110183c: API 202 -> outbox create -> dispatcher publish -> worker consume -> 4 S3 objects + COMMIT ready (37 spans, services: messenger-api, messenger-dispatcher, messenger-worker)
|
||||||
|
- [api] POST /api/v1/chats/{chat_id}/attachments (SERVER, 17.4 ms) http.response.status_code=202
|
||||||
|
- [api] redis GET (CLIENT, 0.3 ms)
|
||||||
|
- [api] SELECT (CLIENT, 0.5 ms)
|
||||||
|
- [api] inspect image (INTERNAL, 0.3 ms)
|
||||||
|
- [api] INSERT (CLIENT, 0.7 ms)
|
||||||
|
- [api] S3 PutObject (CLIENT, 5.0 ms) aws.s3.key=staging/e638ce1aa4c645a6bbc8e93427d644df
|
||||||
|
- [api] UPDATE (CLIENT, 0.6 ms)
|
||||||
|
- [api] create attachments (PRODUCER, 2.9 ms) messenger.job.attempt=1
|
||||||
|
- [api] INSERT (CLIENT, 0.6 ms)
|
||||||
|
- [api] INSERT (CLIENT, 0.5 ms)
|
||||||
|
- [api] INSERT (CLIENT, 0.5 ms)
|
||||||
|
- [api] NOTIFY (CLIENT, 0.2 ms)
|
||||||
|
- [dispatcher] publish attachments (PRODUCER, 3.2 ms) messenger.job.attempt=1
|
||||||
|
- [dispatcher] UPDATE (CLIENT, 0.5 ms)
|
||||||
|
- [worker] process attachments.process.v1 (CONSUMER, 255.7 ms) ERROR messenger.job.attempt=1 messenger.job.outcome=retry error.type=storage_unavailable
|
||||||
|
- [worker] SELECT (CLIENT, 0.8 ms)
|
||||||
|
- [worker] UPDATE (CLIENT, 0.5 ms)
|
||||||
|
- [worker] UPDATE (CLIENT, 0.5 ms)
|
||||||
|
- [worker] INSERT (CLIENT, 0.4 ms)
|
||||||
|
- [worker] S3 GetObject (CLIENT, 3.4 ms) aws.s3.key=staging/e638ce1aa4c645a6bbc8e93427d644df
|
||||||
|
- [worker] verify source sha256 (INTERNAL, 0.0 ms)
|
||||||
|
- [worker] imaging (INTERNAL, 238.9 ms)
|
||||||
|
- [worker] SELECT (CLIENT, 0.6 ms)
|
||||||
|
- [worker] UPDATE (CLIENT, 0.8 ms)
|
||||||
|
- [worker] UPDATE (CLIENT, 0.5 ms)
|
||||||
|
- [worker] create attachments (PRODUCER, 1.5 ms) messenger.job.attempt=2
|
||||||
|
- [worker] INSERT (CLIENT, 0.8 ms)
|
||||||
|
- [worker] NOTIFY (CLIENT, 0.2 ms)
|
||||||
|
- [dispatcher] publish attachments (PRODUCER, 3.3 ms) messenger.job.attempt=2
|
||||||
|
- [dispatcher] UPDATE (CLIENT, 0.6 ms)
|
||||||
|
- [worker] process attachments.process.v1 (CONSUMER, 284.4 ms) messenger.job.attempt=2 messenger.job.outcome=ready
|
||||||
|
- [worker] SELECT (CLIENT, 0.7 ms)
|
||||||
|
- [worker] UPDATE (CLIENT, 0.8 ms)
|
||||||
|
- [worker] UPDATE (CLIENT, 0.7 ms)
|
||||||
|
- [worker] INSERT (CLIENT, 0.6 ms)
|
||||||
|
- [worker] S3 GetObject (CLIENT, 3.5 ms) aws.s3.key=staging/e638ce1aa4c645a6bbc8e93427d644df
|
||||||
|
- [worker] verify source sha256 (INTERNAL, 0.1 ms)
|
||||||
|
- [worker] imaging (INTERNAL, 250.7 ms)
|
||||||
|
- [worker] S3 PutObject (CLIENT, 4.7 ms) aws.s3.key=attachments/9e5dd23f-0ecc-4bdf-b8de-87d529fd54b0/v1/original
|
||||||
|
- [worker] S3 PutObject (CLIENT, 3.2 ms) aws.s3.key=attachments/9e5dd23f-0ecc-4bdf-b8de-87d529fd54b0/v1/optimized.webp
|
||||||
|
- [worker] S3 PutObject (CLIENT, 3.0 ms) aws.s3.key=attachments/9e5dd23f-0ecc-4bdf-b8de-87d529fd54b0/v1/thumbnail.jpg
|
||||||
|
- [worker] S3 PutObject (CLIENT, 2.9 ms) aws.s3.key=attachments/9e5dd23f-0ecc-4bdf-b8de-87d529fd54b0/v1/crop.jpg
|
||||||
|
- [worker] commit result (INTERNAL, 6.6 ms)
|
||||||
|
- [worker] SELECT (CLIENT, 0.6 ms)
|
||||||
|
- [worker] INSERT (CLIENT, 0.7 ms)
|
||||||
|
- [worker] INSERT (CLIENT, 0.6 ms)
|
||||||
|
- [worker] INSERT (CLIENT, 0.8 ms)
|
||||||
|
- [worker] UPDATE (CLIENT, 0.8 ms)
|
||||||
|
- [worker] UPDATE (CLIENT, 0.5 ms)
|
||||||
|
- [worker] INSERT (CLIENT, 0.5 ms)
|
||||||
|
- [worker] cleanup staging (INTERNAL, 4.6 ms)
|
||||||
|
- [worker] S3 DeleteObject (CLIENT, 2.7 ms) aws.s3.key=staging/e638ce1aa4c645a6bbc8e93427d644df
|
||||||
|
- [worker] UPDATE (CLIENT, 1.3 ms)
|
||||||
|
- [worker] INSERT (CLIENT, 0.5 ms)
|
||||||
|
worker log: 'job 9a82f341-ec57-49e5-aa86-6263681529c1 attempt 1: storage_unavailable (temporary) -> retry' error_type=storage_unavailable
|
||||||
|
messenger_job_duration_seconds_count{outcome="retry"}: 0 -> 1
|
||||||
|
PASS: retry in one trace 1abe27ebd519ac0854a844d18d679bf6: attempt 1 ERROR storage_unavailable -> create -> publish -> attempt 2 ready; retry counted in metrics and logged
|
||||||
|
original trace 5b3ea90278cb5e18b6ad08cd32349c0e: 5 attempts, last outcome failed
|
||||||
|
- [ops] ops replay (INTERNAL, 16.5 ms)
|
||||||
|
- [ops] SELECT (CLIENT, 2.3 ms)
|
||||||
|
- [ops] UPDATE (CLIENT, 0.9 ms)
|
||||||
|
- [ops] UPDATE (CLIENT, 0.8 ms)
|
||||||
|
- [ops] INSERT (CLIENT, 0.7 ms)
|
||||||
|
- [ops] create attachments (PRODUCER, 1.1 ms) links=1 messenger.job.attempt=6
|
||||||
|
- [ops] INSERT (CLIENT, 0.7 ms)
|
||||||
|
- [ops] NOTIFY (CLIENT, 0.2 ms)
|
||||||
|
- [dispatcher] publish attachments (PRODUCER, 5.3 ms) messenger.job.attempt=6
|
||||||
|
- [worker] process attachments.process.v1 (CONSUMER, 281.7 ms) messenger.job.attempt=6 messenger.job.outcome=ready
|
||||||
|
- [worker] SELECT (CLIENT, 0.4 ms)
|
||||||
|
- [worker] UPDATE (CLIENT, 0.4 ms)
|
||||||
|
- [worker] UPDATE (CLIENT, 0.6 ms)
|
||||||
|
- [worker] INSERT (CLIENT, 0.2 ms)
|
||||||
|
- [worker] S3 GetObject (CLIENT, 2.7 ms) aws.s3.key=staging/4a1c95c1e3ea46c1b51982aa45344f00
|
||||||
|
- [worker] verify source sha256 (INTERNAL, 0.0 ms)
|
||||||
|
- [worker] imaging (INTERNAL, 251.3 ms)
|
||||||
|
- [worker] S3 PutObject (CLIENT, 3.9 ms) aws.s3.key=attachments/5f4e2b56-4497-4ff8-9c9f-df41939a2520/v1/original
|
||||||
|
- [worker] S3 PutObject (CLIENT, 2.9 ms) aws.s3.key=attachments/5f4e2b56-4497-4ff8-9c9f-df41939a2520/v1/optimized.webp
|
||||||
|
- [worker] S3 PutObject (CLIENT, 2.8 ms) aws.s3.key=attachments/5f4e2b56-4497-4ff8-9c9f-df41939a2520/v1/thumbnail.jpg
|
||||||
|
- [worker] S3 PutObject (CLIENT, 2.5 ms) aws.s3.key=attachments/5f4e2b56-4497-4ff8-9c9f-df41939a2520/v1/crop.jpg
|
||||||
|
- [worker] commit result (INTERNAL, 6.0 ms)
|
||||||
|
- [worker] SELECT (CLIENT, 0.6 ms)
|
||||||
|
- [worker] INSERT (CLIENT, 0.7 ms)
|
||||||
|
- [worker] INSERT (CLIENT, 0.6 ms)
|
||||||
|
- [worker] INSERT (CLIENT, 0.3 ms)
|
||||||
|
- [worker] UPDATE (CLIENT, 0.7 ms)
|
||||||
|
- [worker] UPDATE (CLIENT, 0.5 ms)
|
||||||
|
- [worker] INSERT (CLIENT, 0.4 ms)
|
||||||
|
- [worker] cleanup staging (INTERNAL, 5.9 ms)
|
||||||
|
- [worker] S3 DeleteObject (CLIENT, 3.3 ms) aws.s3.key=staging/4a1c95c1e3ea46c1b51982aa45344f00
|
||||||
|
- [worker] UPDATE (CLIENT, 2.1 ms)
|
||||||
|
- [dispatcher] UPDATE (CLIENT, 1.3 ms)
|
||||||
|
replay trace 6d0327f64ceb5758c4a19b30aceed617: create span links -> ['5b3ea90278cb5e18b6ad08cd32349c0e'] (original upload trace 5b3ea90278cb5e18b6ad08cd32349c0e)
|
||||||
|
PASS: replay = new trace (root 'ops replay') with a link to the original upload; worker finished it
|
||||||
|
4 passed, 0 failed
|
||||||
@@ -0,0 +1,40 @@
|
|||||||
|
# persistence: public persistence test incl. S3 + proof of real restarts (make test-persistence)
|
||||||
|
# date: 2026-11-01 20:46:08, commit: 470d9d6 + uncommitted changes
|
||||||
|
# command: python.exe student-tests/run_persistence.py
|
||||||
|
# exit code: 0, 62.0 s
|
||||||
|
|
||||||
|
before: container, ID, started at
|
||||||
|
messenger-lab6-api-1 171a473b0d28 2026-11-01T17:43:16.63584434Z
|
||||||
|
messenger-lab6-api-2 1db2fedff78f 2026-11-01T17:43:36.466962173Z
|
||||||
|
messenger-lab6-dispatcher-1 2f899202c4d1 2026-11-01T17:31:14.251298706Z
|
||||||
|
messenger-lab6-grafana-1 23e7b8bb1b3a 2026-11-01T17:06:55.450862118Z
|
||||||
|
messenger-lab6-janitor-1 6dad6b402ed9 2026-11-01T17:07:17.636225869Z
|
||||||
|
messenger-lab6-lb-1 0bffe8d8d7d2 2026-11-01T17:07:20.27137747Z
|
||||||
|
messenger-lab6-loki-1 c62735c8baca 2026-11-01T17:22:20.690589644Z
|
||||||
|
messenger-lab6-otel-collector-1 8c416bdc0221 2026-11-01T17:27:23.730596273Z
|
||||||
|
messenger-lab6-postgres-1 ced386ea9bbd 2026-11-01T17:45:04.263026279Z
|
||||||
|
messenger-lab6-prometheus-1 451bf5466e96 2026-11-01T17:09:18.447414724Z
|
||||||
|
messenger-lab6-rabbitmq-1 ec502659b532 2026-11-01T17:30:47.892983401Z
|
||||||
|
messenger-lab6-redis-1 3c003ba9b32a 2026-11-01T17:44:43.823155102Z
|
||||||
|
messenger-lab6-s3-1 1e70a509e403 2026-11-01T17:40:41.919464505Z
|
||||||
|
messenger-lab6-tempo-1 3daf3848457f 2026-11-01T17:22:20.681510324Z
|
||||||
|
messenger-lab6-worker-1 cbd8138493bf 2026-11-01T17:38:33.102546081Z
|
||||||
|
Executing explicit persistence hook; hook output suppressed to avoid leaking secrets.␍
|
||||||
|
PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence.
|
||||||
|
after (hook + waiting for readiness took 60.9 s): container, ID, started at, restarted?
|
||||||
|
messenger-lab6-api-1 171a473b0d28 2026-11-01T17:46:02.488153329Z same container: True, restarted: True
|
||||||
|
messenger-lab6-api-2 1db2fedff78f 2026-11-01T17:46:02.317979748Z same container: True, restarted: True
|
||||||
|
messenger-lab6-dispatcher-1 2f899202c4d1 2026-11-01T17:46:00.495060123Z same container: True, restarted: True
|
||||||
|
messenger-lab6-grafana-1 23e7b8bb1b3a 2026-11-01T17:45:57.449417453Z same container: True, restarted: True
|
||||||
|
messenger-lab6-janitor-1 6dad6b402ed9 2026-11-01T17:45:57.621731679Z same container: True, restarted: True
|
||||||
|
messenger-lab6-lb-1 0bffe8d8d7d2 2026-11-01T17:46:05.984667885Z same container: True, restarted: True
|
||||||
|
messenger-lab6-loki-1 c62735c8baca 2026-11-01T17:45:21.206788727Z same container: True, restarted: True
|
||||||
|
messenger-lab6-otel-collector-1 8c416bdc0221 2026-11-01T17:45:56.829306342Z same container: True, restarted: True
|
||||||
|
messenger-lab6-postgres-1 ced386ea9bbd 2026-11-01T17:45:19.869545292Z same container: True, restarted: True
|
||||||
|
messenger-lab6-prometheus-1 451bf5466e96 2026-11-01T17:45:20.880337926Z same container: True, restarted: True
|
||||||
|
messenger-lab6-rabbitmq-1 ec502659b532 2026-11-01T17:45:21.007029551Z same container: True, restarted: True
|
||||||
|
messenger-lab6-redis-1 3c003ba9b32a 2026-11-01T17:45:19.748047785Z same container: True, restarted: True
|
||||||
|
messenger-lab6-s3-1 1e70a509e403 2026-11-01T17:45:29.752000764Z same container: True, restarted: True
|
||||||
|
messenger-lab6-tempo-1 3daf3848457f 2026-11-01T17:45:50.068926735Z same container: True, restarted: True
|
||||||
|
messenger-lab6-worker-1 cbd8138493bf 2026-11-01T17:46:00.664535867Z same container: True, restarted: True
|
||||||
|
PASS: the hook restarted postgres, redis, s3, rabbitmq, api, janitor, dispatcher, worker, lb, otel-collector, prometheus, loki, tempo, grafana (all api replicas; same containers, new start time); data, the attachment bytes and the old token survived
|
||||||
@@ -0,0 +1,10 @@
|
|||||||
|
# race: lab2 races via the balancer
|
||||||
|
# date: 2026-11-01 20:42:41, commit: 470d9d6 + uncommitted changes
|
||||||
|
# command: python.exe student-tests/test_race.py
|
||||||
|
# exit code: 0, 1.2 s
|
||||||
|
|
||||||
|
sent 100 messages from 10 clients in 0.30 s; read 100 in 4 pages, unique ids: 100
|
||||||
|
PASS: 100 concurrent messages from 10 clients, no losses and no duplicates
|
||||||
|
20 parallel registrations of one username: 201 x1, 409 x19
|
||||||
|
PASS: concurrent registration created exactly one user
|
||||||
|
2 passed, 0 failed
|
||||||
@@ -0,0 +1,14 @@
|
|||||||
|
# redis: Redis as the session store
|
||||||
|
# date: 2026-11-01 20:42:56, commit: 470d9d6 + uncommitted changes
|
||||||
|
# command: python.exe student-tests/redis_info.py
|
||||||
|
# exit code: 0, 3.2 s
|
||||||
|
|
||||||
|
session key: session:0fe0e5550e07d95d...; value is the user id: True
|
||||||
|
TTL of the key, seconds: 3599
|
||||||
|
sessions in Redis: 147
|
||||||
|
config appendonly = yes
|
||||||
|
config appendfsync = always
|
||||||
|
config maxmemory-policy = noeviction
|
||||||
|
aof_enabled:1
|
||||||
|
aof_last_write_status:ok
|
||||||
|
without password: NOAUTH Authentication required.
|
||||||
@@ -0,0 +1,53 @@
|
|||||||
|
# s3-security: lab4: private bucket outside API, limited API key, no console/ports, idempotent bootstrap
|
||||||
|
# date: 2026-11-01 20:38:59, commit: 470d9d6 + uncommitted changes
|
||||||
|
# command: python.exe student-tests/test_s3_security.py
|
||||||
|
# exit code: 0, 10.5 s
|
||||||
|
|
||||||
|
anonymous get_object directly to S3 (from the data network) -> 403 AccessDenied
|
||||||
|
anonymous head_object directly to S3 (from the data network) -> 403
|
||||||
|
anonymous list_bucket directly to S3 (from the data network) -> 403 AccessDenied
|
||||||
|
anonymous put_object directly to S3 (from the data network) -> 403 AccessDenied
|
||||||
|
anonymous get_bucket_policy directly to S3 (from the data network) -> 403 AccessDenied
|
||||||
|
the same file through the API for a chat member -> 200
|
||||||
|
PASS: the bucket is private: no anonymous read, list, write or policy read
|
||||||
|
API key: list buckets -> ['messenger-attachments']
|
||||||
|
API key: get own object -> allowed
|
||||||
|
API key: put object -> allowed
|
||||||
|
API key: delete object -> allowed
|
||||||
|
API key: list bucket -> allowed
|
||||||
|
API key: create another bucket -> denied: AccessDenied
|
||||||
|
API key: make bucket public (put policy) -> denied: NotImplemented
|
||||||
|
API key: read bucket policy -> denied: NoSuchBucketPolicy
|
||||||
|
API key: delete bucket -> denied: AccessDenied
|
||||||
|
API key: change versioning -> denied: NotImplemented
|
||||||
|
PASS: the API key can only get/put/delete/list objects of its own bucket; bucket admin, policy and other buckets are denied
|
||||||
|
compose: s3 has no published ports, only the internal network 'data' (S3 8333, master UI 9333, filer UI 8888 are reachable only inside the stand)
|
||||||
|
127.0.0.1:8333 from the host -> connection refused
|
||||||
|
127.0.0.1:9333 from the host -> connection refused
|
||||||
|
127.0.0.1:8888 from the host -> connection refused
|
||||||
|
127.0.0.1:19333 from the host -> connection refused
|
||||||
|
secrets of api: ['cache_url', 'database_url', 's3_access_key_id', 's3_secret_access_key']
|
||||||
|
secrets of dispatcher: ['broker_url_dispatcher', 'database_url']
|
||||||
|
secrets of grafana: ['grafana_admin_password']
|
||||||
|
secrets of janitor: ['database_url', 's3_access_key_id', 's3_secret_access_key']
|
||||||
|
secrets of lb: -
|
||||||
|
secrets of loki: -
|
||||||
|
secrets of migrate: ['database_url']
|
||||||
|
secrets of otel-collector: -
|
||||||
|
secrets of postgres: ['postgres_password']
|
||||||
|
secrets of prometheus: -
|
||||||
|
secrets of rabbitmq: ['rabbitmq_definitions']
|
||||||
|
secrets of redis: ['redis_auth']
|
||||||
|
secrets of s3: ['s3_config']
|
||||||
|
secrets of s3-init: ['s3_access_key_id', 's3_root_password', 's3_root_user', 's3_secret_access_key']
|
||||||
|
secrets of telemetry-init: -
|
||||||
|
secrets of tempo: -
|
||||||
|
secrets of worker: ['broker_url_worker', 'database_url', 's3_access_key_id', 's3_secret_access_key']
|
||||||
|
PASS: S3 is reachable only inside the stand; admin keys are only in s3 and s3-init
|
||||||
|
s3-init: s3-init: bucket messenger-attachments exists
|
||||||
|
s3-init: s3-init: api key can put/get/list/delete objects of its bucket
|
||||||
|
s3-init: s3-init: api key cannot create buckets (AccessDenied)
|
||||||
|
s3-init: s3-init: bucket is private (anonymous list/get/put -> [403])
|
||||||
|
after the repeated bootstrap: old file downloads, new upload works
|
||||||
|
PASS: bucket/user/policy bootstrap can be repeated safely
|
||||||
|
4 passed, 0 failed
|
||||||
@@ -0,0 +1,6 @@
|
|||||||
|
# smoke: public smoke via the balancer (make test) with telemetry ON, with attachment
|
||||||
|
# date: 2026-11-01 20:07:47, commit: 470d9d6 + uncommitted changes
|
||||||
|
# command: python.exe tests/smoke.py --base-url http://127.0.0.1:8080
|
||||||
|
# exit code: 0, 1.0 s
|
||||||
|
|
||||||
|
PASS: lab6 public API scenario; infrastructure and grading evidence remain separate.
|
||||||
@@ -0,0 +1,66 @@
|
|||||||
|
# stack: stand layout: replicas, ports, limits, networks, secrets, S3, telemetry services
|
||||||
|
# date: 2026-11-01 20:07:46, commit: 470d9d6 + uncommitted changes
|
||||||
|
# command: python.exe student-tests/check_stack.py
|
||||||
|
# exit code: 0, 3.1 s
|
||||||
|
|
||||||
|
host: Windows 10, AMD64 Family 26 Model 68 Stepping 0, AuthenticAMD; client Python 3.11.1
|
||||||
|
Docker 28.1.1, Docker Compose version v2.35.1-desktop.1; Docker VM: 16 CPU, 15.2 GiB RAM, Docker Desktop
|
||||||
|
|
||||||
|
SERVICE CONTAINER ID STATUS PORTS
|
||||||
|
api 171a473b0d28 Up 26 seconds (healthy) 8080/tcp
|
||||||
|
api 1db2fedff78f Up 26 seconds (healthy) 8080/tcp
|
||||||
|
dispatcher 2f899202c4d1 Up 47 seconds (healthy)
|
||||||
|
grafana 23e7b8bb1b3a Up 49 seconds (healthy) 127.0.0.1:3000->3000/tcp
|
||||||
|
janitor 6dad6b402ed9 Up 26 seconds (healthy)
|
||||||
|
lb 0bffe8d8d7d2 Up 24 seconds (healthy) 127.0.0.1:8080->8080/tcp, 127.0.0.1:8404->8404/tcp
|
||||||
|
loki c62735c8baca Up 54 seconds 127.0.0.1:3100->3100/tcp
|
||||||
|
migrate dd9753ec4186 Exited (0) 48 seconds ago
|
||||||
|
otel-collector 8c416bdc0221 Up 55 seconds
|
||||||
|
postgres ced386ea9bbd Up 55 seconds (healthy)
|
||||||
|
prometheus 451bf5466e96 Up 55 seconds (healthy) 127.0.0.1:9090->9090/tcp
|
||||||
|
rabbitmq ec502659b532 Up 55 seconds (healthy)
|
||||||
|
redis 3c003ba9b32a Up 55 seconds (healthy)
|
||||||
|
s3 1e70a509e403 Up 55 seconds (healthy)
|
||||||
|
s3-init 64471229b155 Exited (0) 27 seconds ago
|
||||||
|
telemetry-init 809e227f3528 Exited (0) 55 seconds ago
|
||||||
|
tempo 3daf3848457f Up 54 seconds 127.0.0.1:3200->3200/tcp
|
||||||
|
worker cbd8138493bf Up 26 seconds (healthy)
|
||||||
|
|
||||||
|
container image cpus memory host ports networks
|
||||||
|
messenger-lab6-api-1 messenger-lab6-api 1.00 512 MiB - data,edge,telemetry
|
||||||
|
messenger-lab6-api-2 messenger-lab6-api 1.00 512 MiB - data,edge,telemetry
|
||||||
|
messenger-lab6-dispatcher-1 messenger-lab6-dispatcher 0.25 128 MiB - data,telemetry
|
||||||
|
messenger-lab6-grafana-1 grafana/grafana:12.4.14 0.50 384 MiB 127.0.0.1:3000 ops,telemetry
|
||||||
|
messenger-lab6-janitor-1 messenger-lab6-janitor 0.25 192 MiB - data,telemetry
|
||||||
|
messenger-lab6-lb-1 haproxy:3.2.25-alpine 0.50 128 MiB 127.0.0.1:8080,127.0.0.1:8404 edge
|
||||||
|
messenger-lab6-loki-1 grafana/loki:3.7.9 1.00 512 MiB 127.0.0.1:3100 ops,telemetry
|
||||||
|
messenger-lab6-otel-collector-1 otel/opentelemetry-collector-contrib:0.164.0 0.50 384 MiB - telemetry
|
||||||
|
messenger-lab6-postgres-1 postgres:17.11-alpine 1.00 512 MiB - data
|
||||||
|
messenger-lab6-prometheus-1 prom/prometheus:v3.15.1 1.00 512 MiB 127.0.0.1:9090 ops,telemetry
|
||||||
|
messenger-lab6-rabbitmq-1 rabbitmq:4.1.4-alpine 1.00 512 MiB - data
|
||||||
|
messenger-lab6-redis-1 redis:7.4.11-alpine 0.50 128 MiB - data
|
||||||
|
messenger-lab6-s3-1 chrislusf/seaweedfs:3.80 1.00 512 MiB - data
|
||||||
|
messenger-lab6-tempo-1 grafana/tempo:3.1.1 1.00 512 MiB 127.0.0.1:3200 ops,telemetry
|
||||||
|
messenger-lab6-worker-1 messenger-lab6-worker 2.00 1536 MiB - data,telemetry
|
||||||
|
|
||||||
|
secrets of api: ['cache_url', 'database_url', 's3_access_key_id', 's3_secret_access_key']
|
||||||
|
secrets of dispatcher: ['broker_url_dispatcher', 'database_url']
|
||||||
|
secrets of grafana: ['grafana_admin_password']
|
||||||
|
secrets of janitor: ['database_url', 's3_access_key_id', 's3_secret_access_key']
|
||||||
|
secrets of lb: -
|
||||||
|
secrets of loki: -
|
||||||
|
secrets of migrate: ['database_url']
|
||||||
|
secrets of otel-collector: -
|
||||||
|
secrets of postgres: ['postgres_password']
|
||||||
|
secrets of prometheus: -
|
||||||
|
secrets of rabbitmq: ['rabbitmq_definitions']
|
||||||
|
secrets of redis: ['redis_auth']
|
||||||
|
secrets of s3: ['s3_config']
|
||||||
|
secrets of s3-init: ['s3_access_key_id', 's3_root_password', 's3_root_user', 's3_secret_access_key']
|
||||||
|
secrets of telemetry-init: -
|
||||||
|
secrets of tempo: -
|
||||||
|
secrets of worker: ['broker_url_worker', 'database_url', 's3_access_key_id', 's3_secret_access_key']
|
||||||
|
|
||||||
|
through http://127.0.0.1:8080 one session is served by 2 replicas: ['171a473b0d28', '1db2fedff78f']
|
||||||
|
/health/ready -> 200 from 1db2fedff78f
|
||||||
|
PASS: 2 api replicas behind one entry point, no direct API ports, limits on every container, postgres/redis/s3/rabbitmq only in the internal data network, worker/dispatcher/janitor in data+telemetry, collector only in the internal telemetry network, prometheus/loki/tempo/grafana on 127.0.0.1 with persistent volumes, pinned images, S3 admin keys only in s3 and s3-init, broker credentials only in dispatcher/worker
|
||||||
@@ -0,0 +1,26 @@
|
|||||||
|
# student-tests: lab1/lab2 API tests via the balancer
|
||||||
|
# date: 2026-11-01 20:42:40, commit: 470d9d6 + uncommitted changes
|
||||||
|
# command: python.exe student-tests/test_api.py
|
||||||
|
# exit code: 0, 2.8 s
|
||||||
|
|
||||||
|
PASS: duplicate username returns 409
|
||||||
|
PASS: invalid JSON returns 400
|
||||||
|
PASS: wrong and extra fields return 400
|
||||||
|
PASS: foreign chat returns 404
|
||||||
|
PASS: empty and too long messages return 400, 4000 chars is ok
|
||||||
|
PASS: pagination works across multiple pages, foreign cursor returns 400
|
||||||
|
PASS: empty cursor (?cursor= and ?cursor) returns 400, no cursor returns the first page
|
||||||
|
PASS: chat list pagination
|
||||||
|
PASS: unicode text is stored as is
|
||||||
|
PASS: strings with U+0000 are stored and returned as is
|
||||||
|
PASS: user response has no password or hash
|
||||||
|
PASS: password length 12..128, blank password is rejected
|
||||||
|
PASS: Basic login returns opaque Bearer token with expiry
|
||||||
|
PASS: wrong password and unknown user both return the same 401
|
||||||
|
PASS: malformed Basic header returns 401
|
||||||
|
PASS: password with colon and unicode works
|
||||||
|
PASS: business endpoints need Bearer; Basic and X-User-Id give 401
|
||||||
|
PASS: X-User-Id does not change the current user
|
||||||
|
PASS: 401 comes before 400 and 404
|
||||||
|
PASS: logout revokes only the current session
|
||||||
|
20 passed, 0 failed
|
||||||
@@ -0,0 +1,108 @@
|
|||||||
|
# Сводка проверок (Docker-стенд, 2026-11-01)
|
||||||
|
|
||||||
|
Шаги запускались группами (`collect_evidence.py check stack smoke configs telemetry dashboard telemetry-history`, затем каждый инцидент и `load` отдельно, затем `--from jobs-variants`); `load` выполнен дважды — в группе lab6 и в регрессии, evidence/load.txt и load.json — от второго прогона.
|
||||||
|
|
||||||
|
| Шаг | Что проверяет | Результат | Время, с | Файл |
|
||||||
|
| --- | --- | --- | --- | --- |
|
||||||
|
| check | offline contract check (make check) | PASS | 0 | evidence/check.txt |
|
||||||
|
| stack | stand layout: replicas, ports, limits, networks, secrets, S3, telemetry services | PASS | 3 | evidence/stack.txt |
|
||||||
|
| smoke | public smoke via the balancer (make test) with telemetry ON, with attachment | PASS | 1 | evidence/smoke.txt |
|
||||||
|
| configs | lab6: collector/prometheus/loki/tempo configs by their own tools, alert unit tests, Grafana provisioning | PASS | 6 | evidence/configs.txt |
|
||||||
|
| telemetry | lab6: request<->log<->trace, upload->publish->consume->S3/DB chain, retry, replay link, no secrets | PASS | 79 | evidence/telemetry.txt |
|
||||||
|
| dashboard | lab6: every dashboard/rule query runs on live data, key signals present, no forbidden labels | PASS | 1 | evidence/dashboard.txt |
|
||||||
|
| telemetry-history | lab6: traces/logs/metrics survive restart of prometheus/loki/tempo (volumes) | PASS | 39 | evidence/telemetry-history.txt |
|
||||||
|
| incident-worker | lab6 incident 1: worker stopped -> queue lag + no consumers firing -> resolved | PASS | 162 | evidence/incident-worker.txt |
|
||||||
|
| incident-db | lab6 incident 2: slow PostgreSQL (table lock) -> high error rate firing, cause from traces -> resolved | PASS | 286 | evidence/incident-db.txt |
|
||||||
|
| incident-telemetry | lab6: Tempo+Loki down -> API unaffected, collector memory bounded, losses counted | PASS | 380 | evidence/incident-telemetry.txt |
|
||||||
|
| incident-collector | lab6: Collector down -> API unaffected, SDK queue bounded, drops counted | PASS | 156 | evidence/incident-collector.txt |
|
||||||
|
| load | public load generator, 30 s, concurrency 4 | PASS | 30 | evidence/load.txt |
|
||||||
|
| jobs-variants | lab5: thumbnail/crop/optimized, EXIF orientation, alpha, no upscale, metadata stripped | PASS | 20 | evidence/jobs-variants.txt |
|
||||||
|
| jobs-limits | lab5: worker limits - bytes, pixels, memory (RLIMIT_AS), time (JOB_TIMEOUT_SECONDS) | PASS | 73 | evidence/jobs-limits.txt |
|
||||||
|
| jobs-recovery | lab5: worker down -> 202 -> restart api+broker -> ready; broker down -> outbox waits | PASS | 55 | evidence/jobs-recovery.txt |
|
||||||
|
| jobs-outbox | lab5: dispatcher crash between COMMIT and publish / after confirm | PASS | 17 | evidence/jobs-outbox.txt |
|
||||||
|
| jobs-crash | lab5: worker crash after S3 PUT / after COMMIT before ack, staging cleanup, frozen worker lease | PASS | 157 | evidence/jobs-crash.txt |
|
||||||
|
| jobs-retry | lab5: backoff+jitter, retry_exhausted, replay, permanent errors, DLQ | PASS | 78 | evidence/jobs-retry.txt |
|
||||||
|
| jobs-janitor-replay | lab5: janitor does not delete objects re-written by replay (review fix) | PASS | 187 | evidence/jobs-janitor-replay.txt |
|
||||||
|
| jobs-concurrency | lab5: WORKER_CONCURRENCY consumers, prefetch 1, bounded processing | PASS | 16 | evidence/jobs-concurrency.txt |
|
||||||
|
| attachments | lab4: files, limits, damaged images, MIME, filenames, access, attachment_ids | PASS | 10 | evidence/attachments.txt |
|
||||||
|
| s3-security | lab4: private bucket outside API, limited API key, no console/ports, idempotent bootstrap | PASS | 10 | evidence/s3-security.txt |
|
||||||
|
| consistency | lab4: aborted upload, crash between S3 and DB, janitor, orphans, S3 outage | PASS | 160 | evidence/consistency.txt |
|
||||||
|
| upload-memory | lab4: >=10 concurrent uploads, peak memory of api replicas | PASS | 53 | evidence/upload-memory.txt |
|
||||||
|
| balancing | balancing, shared sessions/data/files, X-Instance-Id | PASS | 2 | evidence/balancing.txt |
|
||||||
|
| idempotency | Idempotency-Key, parallel same key via replicas | PASS | 2 | evidence/idempotency.txt |
|
||||||
|
| student-tests | lab1/lab2 API tests via the balancer | PASS | 3 | evidence/student-tests.txt |
|
||||||
|
| race | lab2 races via the balancer | PASS | 1 | evidence/race.txt |
|
||||||
|
| db | PostgreSQL: rollback, FK, Argon2id, unique keys, migrations | PASS | 12 | evidence/db.txt |
|
||||||
|
| redis | Redis as the session store | PASS | 3 | evidence/redis.txt |
|
||||||
|
| explain | EXPLAIN of the history page (full plans: explain-history.txt) | PASS | 1 | evidence/explain.txt |
|
||||||
|
| failover | public failover test for every replica, file downloaded via the other one (make test-failover) | PASS | 42 | evidence/failover.txt |
|
||||||
|
| load | public load generator, 30 s, concurrency 4 | PASS | 30 | evidence/load.txt |
|
||||||
|
| failures | Redis and PostgreSQL outages via the balancer | PASS | 56 | evidence/failures.txt |
|
||||||
|
| persistence | public persistence test incl. S3 + proof of real restarts (make test-persistence) | PASS | 62 | evidence/persistence.txt |
|
||||||
|
|
||||||
|
# Замеры
|
||||||
|
|
||||||
|
Машина: Windows 10, AMD64 Family 26 Model 68 Stepping 0, AuthenticAMD; Docker VM: 16 CPU, 15.2 GiB RAM; клиент Python 3.11.1; дата прогона 2026-11-01, commit 470d9d6 + uncommitted changes. Лимиты контейнеров: api 1 CPU / 512 MiB каждая, postgres, s3 (SeaweedFS) и rabbitmq по 1 CPU / 512 MiB, redis и lb по 0,5 CPU / 128 MiB, janitor 0,25 CPU / 192 MiB, dispatcher 0,25 CPU / 128 MiB, worker 2 CPU / 1536 MiB; lab6: otel-collector 0,5 CPU / 384 MiB, prometheus, loki, tempo по 1 CPU / 512 MiB, grafana 0,5 CPU / 384 MiB.
|
||||||
|
|
||||||
|
Алерты на контролируемых инцидентах (`student-tests/incidents.py`, состояние из Prometheus /api/v1/alerts):
|
||||||
|
|
||||||
|
| Инцидент | Воздействие | Алерт | firing через, с | resolved через, с после устранения |
|
||||||
|
| --- | --- | --- | --- | --- |
|
||||||
|
| 1. worker | `docker compose stop worker` + 3 загрузки | MessengerWorkerNoConsumers | 101.7 (от stop) | 29.7 (от start) |
|
||||||
|
| 1. worker | `docker compose stop worker` + 3 загрузки | MessengerQueueLagGrowing | 131.8 (от stop) | 29.7 (от start) |
|
||||||
|
| 2. PostgreSQL | `LOCK TABLE messages IN ACCESS EXCLUSIVE MODE` на 210 с под нагрузкой 8 потоков | MessengerHighErrorRate | 140.8 (от LOCK) | 50.1 (от COMMIT блокировки) |
|
||||||
|
| 3. телеметрия | `docker compose stop tempo loki` на 150 с | TelemetryExportFailing | 73.1 (от stop) | 165.6 (от start) |
|
||||||
|
|
||||||
|
Инцидент 1, наблюдения в момент firing: consumers очереди = 0.0, сообщений в RabbitMQ = 6.0, queued заданий = 3.0, возраст старейшего = 127.3 с, возраст outbox = 0.0 с (публикация работает), метрики worker не обновлялись 131.0 с, последний лог worker: event=`service.stopped`; трасса загрузки содержит `publish attachments`, но не содержит `process attachments.process.v1`. После start: вложения ready через 4.7 с, backlog 0.0.
|
||||||
|
|
||||||
|
Инцидент 2, наблюдения: доля 5xx (1m) = 1.0, коды POST /messages за 1m = {'201': 0.0, '200': 0.0, '404': 0.0, '400': 0.0, '409': 0.0, '503': 1.6}, p95 POST /messages = 7.375 с, чтения других таблиц (GET /users/me, /chats) во время блокировки {'GET /users/me + GET /chats, avg per request': 17.2, 'max': 23.5} мс, занято соединений пула = 8.0, p95 ожидания пула = 0.001 с, логи API event=db.unavailable по error_type: {'QueryCanceled': 216}. Error-трасса 237e2d6b1c7db22adbb7d4752fa47f2: server-спан 5013.3 мс, из них `INSERT` 5001.3 мс (доля 0.998, статус ERROR). Контраст «медленный обработчик» (трасса 4a222e478c2d51006538a7bfeb485eb1, POST /users): server 21.2 мс, `argon2id hash` 16.4 мс, все спаны БД 0.7 мс. Нагрузка: до 115.5 успешных записей/с, во время блокировки {'503': 320, '201': 8}, после 116.8/с без ошибок.
|
||||||
|
|
||||||
|
| Отказ телеметрии | API во время отказа | Память | Потери телеметрии |
|
||||||
|
| --- | --- | --- | --- |
|
||||||
|
| `stop tempo loki` на 150 с (окно повторов Collector 120 с) | 9555 запросов, коды {'201': 9555}, доля успеха 1.0 | heap Collector 35.5 -> max 86.9 MiB (limit 200), контейнер max 107.0 MiB | send/enqueue failed в Collector: {'spans': 4096, 'log_records': 235}; пробные трассы дошли 13/13, потеряны отправленные на [] с |
|
||||||
|
| `stop otel-collector` на 90 с | 5852 запросов, коды {'201': 5852}, доля успеха 1.0 | api 295.2 -> max 313.5 MiB (2 реплики) | SDK api потерял спанов: {'processor_queue_full': 8252, 'exporter_failed': 10752} (queue_full — очередь 2048 полна, exporter_failed — Collector не принял пакет); после start экспорт возобновился (10287 спанов за минуту) |
|
||||||
|
|
||||||
|
Публичный тест отказа (`make test-failover`, 15 с записи во время остановки реплики):
|
||||||
|
|
||||||
|
| Прогон | Остановлена реплика | Реплик до отказа | Успешных записей | Неудачных попыток (повторены с тем же ключом) | Первая запись после hook, с |
|
||||||
|
| --- | --- | --- | --- | --- | --- |
|
||||||
|
| 1 | 171a473b0d28 | 2 | 127 | 0 | 4.75 |
|
||||||
|
| 2 | 1db2fedff78f | 2 | 126 | 0 | 4.578 |
|
||||||
|
|
||||||
|
Одновременные загрузки (`student-tests/upload_memory.py`, перед каждым сценарием реплики перезапускаются, лимит api 512 MiB / 1 CPU). Пик RSS — VmHWM процесса API; cgroup — память контейнера, включая page cache временных файлов:
|
||||||
|
|
||||||
|
| Сценарий | Одновременно | Объём, MiB | Время, с (p50 / max) | Реплика: загрузок | RSS до, MiB | Пик RSS, MiB | Рост, MiB | Пик cgroup, MiB |
|
||||||
|
| --- | --- | --- | --- | --- | --- | --- | --- | --- |
|
||||||
|
| 10 x ~9.5 MiB PNG (noise) | 10 | 94.9 | 1.77 / 1.95 | 171a473b0d28: 5 | 109.6 | 124.5 | 14.9 | 162.8 |
|
||||||
|
| | | | | 1db2fedff78f: 5 | 110.5 | 125.6 | 15.1 | 163.3 |
|
||||||
|
| 10 x 40 Mpx PNG (8000x5000 RGB) | 10 | 1.1 | 0.98 / 1.3 | 171a473b0d28: 5 | 107.6 | 112.4 | 4.8 | 101.5 |
|
||||||
|
| | | | | 1db2fedff78f: 5 | 107.1 | 109.8 | 2.7 | 100.0 |
|
||||||
|
| 20 mixed: large + 40 Mpx + JPEG | 20 | 133.2 | 1.45 / 1.95 | 171a473b0d28: 10 | 109.0 | 136.3 | 27.3 | 191.4 |
|
||||||
|
| | | | | 1db2fedff78f: 10 | 110.6 | 135.0 | 24.3 | 172.8 |
|
||||||
|
|
||||||
|
Один PNG 8000x5000 RGB внутри контейнера api, дополнительный пик RSS: наивно (тело целиком в памяти + полное декодирование Pillow) 161 MiB, потоковая проверка 0 MiB. 10 таких наивных загрузок одновременно не поместились бы в 512 MiB.
|
||||||
|
|
||||||
|
Публичный генератор нагрузки (`tests/load.py`, closed loop, concurrency 4, 30 с, 2 реплики): 239.02 успешных POST/с, ошибок 0 (доля 0.0), задержка успешных p50 16.0 мс, p95 32.0 мс, p99 47.0 мс.
|
||||||
|
|
||||||
|
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
|
||||||
|
| --- | --- | --- | --- |
|
||||||
|
| публичный `make test-failover` для каждой реплики | запись продолжается не позже 10 с, подтверждённые сообщения на месте, дублей нет | PASS 2 раз(а), неудачных попыток: 0, 0 | первая запись после hook через 4.75, 4.578 с (hook ждёт остановки ~4 с); потерь нет |
|
||||||
|
| перезапуск всего стенда (`restart` hook: postgres, redis, s3, api, janitor, lb) | данные, файл и прежний токен живы | PASS, те же контейнеры с новым временем старта | 60.9 с до готовности; потерь нет |
|
||||||
|
| `docker compose stop s3` (SeaweedFS остановлен) | загрузка и скачивание -> 503, метаданные, сообщения и /health/ready работают | upload 503 за 20234 мс, download 503, остальное 200/201 | после start файлы на месте; uploading-строку убрал janitor; потерь нет |
|
||||||
|
| `docker compose pause s3` (SeaweedFS завис) | то же, но через таймауты S3-клиента | upload 503 за 14703 мс (2 попытки x 5 с + откат 2 с) | после unpause файлы на месте; потерь нет |
|
||||||
|
| падение между PUT в S3 и COMMIT в БД (код API в контейнере, finish_upload падает) | вложение не видно, janitor не трогает его до конца grace period, потом удаляет объект и строку | GET/прикрепление -> 404; janitor: свежая строка сохранена, dry-run только отчёт, затем удалено | мусора не осталось; повторный проход ничего не делает |
|
||||||
|
| обрыв загрузки клиентом (9 МБ заявлено, отправлено 0,2 и 3 МБ) | ничего не сохраняется | строк и объектов не прибавилось, временных файлов 0 | мгновенно |
|
||||||
|
| worker остановлен, upload, рестарт rabbitmq и api с volumes, start worker | 202 queued, сообщение с вложением 201, content 409; после рестартов job в очереди; тот же job_id -> ready | worker started -> same job 6a45774d-0d88-46ec-87b9-70fb4c122bbd ready, thumbnail downloadable; staging object deleted | потерь нет; staging удалён после COMMIT |
|
||||||
|
| rabbitmq остановлен во время upload | 202 и /health/ready 200 без брокера; строка outbox ждёт; после start публикуется | rabbitmq started -> dispatcher published the pending row -> ready | потерь нет |
|
||||||
|
| crash dispatcher между COMMIT задания и publish | строка outbox не отмечена -> публикуется после рестарта процесса | dispatcher killed between COMMIT and publish; restarted by policy, outbox re-published -> job d1eb476a-fbe1-4271-861f-74f3e943b8cc ready, attempt 1, no manual action | без ручного действия |
|
||||||
|
| crash dispatcher после confirm, до отметки outbox | дубль сообщения, worker распознаёт по состоянию job | dispatcher killed after publisher confirm, before marking outbox -> message published twice; worker: ['created', 'claimed', 'ready', 'duplicate_ignored'] -> one result set | один набор вариантов |
|
||||||
|
| crash worker после PUT в S3, до COMMIT и ack | повтор после истечения аренды перезаписывает те же ключи | worker killed after S3 PUT (before COMMIT and ack): redelivery saw a live lease (no-op), lease expired after 15 s, attempt 2 rewrote the same keys -> one set; events ['created', 'claimed', 'duplicate_ignored', 'lease_expired', 'claimed', 'ready'] | один набор, 4 объекта |
|
||||||
|
| crash worker после COMMIT, до ack | redelivery ready-задачи -> no-op ack | worker killed after COMMIT, before ack: broker redelivered, worker acked as no-op (1:duplicate_ignored:status=ready); queue empty (0 ready, 0 unacked) | очередь пуста, дублей нет |
|
||||||
|
| worker завис (не продлевает аренду) | dispatcher возвращает job, другой поток завершает; поздний результат отклонён fencing | frozen attempt 1 stopped renewing; lease expired, recovered by dispatcher, attempt 2 ready after 17 s; frozen attempt woke up -> stale_result_rejected; events ['created', 'claimed', 'lease_expired', 'claimed', 'ready', 'stale_result_rejected'] | ready не откатан |
|
||||||
|
| зависание обработки изображения | процесс убит по JOB_TIMEOUT_SECONDS, временная ошибка, аренда продлевалась | imaging hang -> killed after 60 s (processing_timeout, temporary), lease renewed meanwhile (no lease_expired), attempt 2 ready after 62 s | повтор -> ready |
|
||||||
|
| нехватка памяти в процессе обработки (RLIMIT_AS) | failed/resource_limit, worker жив | memory bomb in the imaging process -> failed/resource_limit after attempt 1; worker itself survived | постоянная ошибка, без повторов |
|
||||||
|
| 2 временные ошибки S3 | backoff с jitter, затем ready | 2 x storage_unavailable -> retries after 1.90 s and 3.20 s (equal jitter: 1..2 s, 2..4 s), attempt 3 ready | - |
|
||||||
|
| 5 временных ошибок подряд | failed/retry_exhausted, реестр failed, content 409 | 5 temporary errors -> failed/retry_exhausted (public error_code), content 409, attach 409, listed by ops.py failed (replayable) | replay оператора |
|
||||||
|
| replay failed и повторный replay ready | тот же job_id -> ready; повтор отклонён | ops replay -> same job_id d8ab860c-eb40-4df2-8d98-67aade24f2f2 attempt 6 ready; second replay refused: "job is ready, replay is only for failed jobs" | дубля нет |
|
||||||
|
| невалидное сообщение и неизвестный job_id | reject -> DLQ, dlq-replay не исполняет | schema-invalid message and unknown job_id -> reject -> DLQ (2 message(s)); dlq-replay kept them: ['invalid message is kept for analysis', 'unknown job'] | сообщения сохранены |
|
||||||
|
| 8 заданий в очереди, старт worker | WORKER_CONCURRENCY=2, prefetch 1 | 2 consumers, prefetch_count 1, manual ack; 8 queued 3000x2000 JPEG drained in 2.4 s after worker start, peak processing 2 of 8 (6 samples) | - |
|
||||||
@@ -0,0 +1,9 @@
|
|||||||
|
# telemetry-history: lab6: traces/logs/metrics survive restart of prometheus/loki/tempo (volumes)
|
||||||
|
# date: 2026-11-01 20:09:51, commit: 470d9d6 + uncommitted changes
|
||||||
|
# command: python.exe student-tests/test_telemetry.py history
|
||||||
|
# exit code: 0, 38.6 s
|
||||||
|
|
||||||
|
before restart: trace 0b9106c1057e9753186aec51b529946b found, its log found, sum(http_server_request_duration_seconds_count{job="messenger-api"}) at t-30s = 434
|
||||||
|
after restart of prometheus, loki, tempo: trace 0b9106c1057e9753186aec51b529946b (4 spans), 1 log record(s), metric at t-30s = 434 (same value)
|
||||||
|
PASS: persistent volumes keep traces, logs and metrics across restarts of the stores
|
||||||
|
1 passed, 0 failed
|
||||||
@@ -0,0 +1,145 @@
|
|||||||
|
# telemetry: lab6: request<->log<->trace, upload->publish->consume->S3/DB chain, retry, replay link, no secrets
|
||||||
|
# date: 2026-11-01 20:09:11, commit: 470d9d6 + uncommitted changes
|
||||||
|
# command: python.exe student-tests/test_telemetry.py request upload-chain retry-chain replay-link
|
||||||
|
# exit code: 0, 78.6 s
|
||||||
|
|
||||||
|
POST message: 201, X-Request-Id=df174aef-1d0d-4340-b2d7-907241684f5e, trace_id=6be6bfe68dce05dd2bbbf0048a53c7ce (client traceparent)
|
||||||
|
- [api] POST /api/v1/chats/{chat_id}/messages (SERVER, 4.6 ms) http.response.status_code=201
|
||||||
|
- [api] redis GET (CLIENT, 0.2 ms)
|
||||||
|
- [api] SELECT (CLIENT, 0.3 ms)
|
||||||
|
- [api] INSERT (CLIENT, 0.4 ms)
|
||||||
|
Loki: 'POST /api/v1/chats/{chat_id}/messages -> 201 (4.6 ms)' event=http.request severity=INFO service=messenger-api trace_id=6be6bfe68dce05dd2bbbf0048a53c7ce span_id=78cd8621fce4f3b8 request_id=df174aef-1d0d-4340-b2d7-907241684f5e
|
||||||
|
PASS: request -> server span with DB/Redis children -> structured log with the same trace_id/span_id and request_id; no message text, password or token in logs/span attributes
|
||||||
|
- [api] POST /api/v1/chats/{chat_id}/attachments (SERVER, 13.5 ms) http.response.status_code=202
|
||||||
|
- [api] redis GET (CLIENT, 0.2 ms)
|
||||||
|
- [api] SELECT (CLIENT, 0.4 ms)
|
||||||
|
- [api] inspect image (INTERNAL, 0.5 ms)
|
||||||
|
- [api] INSERT (CLIENT, 0.6 ms)
|
||||||
|
- [api] S3 PutObject (CLIENT, 2.9 ms) aws.s3.key=staging/1b42fa0c1dd84589a58369be9f034238
|
||||||
|
- [api] UPDATE (CLIENT, 0.7 ms)
|
||||||
|
- [api] create attachments (PRODUCER, 2.4 ms) messenger.job.attempt=1
|
||||||
|
- [api] INSERT (CLIENT, 0.7 ms)
|
||||||
|
- [api] INSERT (CLIENT, 0.6 ms)
|
||||||
|
- [api] INSERT (CLIENT, 0.6 ms)
|
||||||
|
- [api] NOTIFY (CLIENT, 0.1 ms)
|
||||||
|
- [dispatcher] publish attachments (PRODUCER, 4.0 ms) messenger.job.attempt=1
|
||||||
|
- [dispatcher] UPDATE (CLIENT, 0.4 ms)
|
||||||
|
- [worker] process attachments.process.v1 (CONSUMER, 161.4 ms) messenger.job.attempt=1 messenger.job.outcome=ready
|
||||||
|
- [worker] SELECT (CLIENT, 0.4 ms)
|
||||||
|
- [worker] UPDATE (CLIENT, 0.3 ms)
|
||||||
|
- [worker] UPDATE (CLIENT, 0.4 ms)
|
||||||
|
- [worker] INSERT (CLIENT, 0.3 ms)
|
||||||
|
- [worker] S3 GetObject (CLIENT, 2.0 ms) aws.s3.key=staging/1b42fa0c1dd84589a58369be9f034238
|
||||||
|
- [worker] verify source sha256 (INTERNAL, 0.0 ms)
|
||||||
|
- [worker] imaging (INTERNAL, 139.8 ms)
|
||||||
|
- [worker] S3 PutObject (CLIENT, 2.8 ms) aws.s3.key=attachments/98cd3d9b-6ee6-4d65-8c63-9586acb34da1/v1/original
|
||||||
|
- [worker] S3 PutObject (CLIENT, 1.9 ms) aws.s3.key=attachments/98cd3d9b-6ee6-4d65-8c63-9586acb34da1/v1/optimized.webp
|
||||||
|
- [worker] S3 PutObject (CLIENT, 2.3 ms) aws.s3.key=attachments/98cd3d9b-6ee6-4d65-8c63-9586acb34da1/v1/thumbnail.jpg
|
||||||
|
- [worker] S3 PutObject (CLIENT, 1.8 ms) aws.s3.key=attachments/98cd3d9b-6ee6-4d65-8c63-9586acb34da1/v1/crop.jpg
|
||||||
|
- [worker] commit result (INTERNAL, 4.9 ms)
|
||||||
|
- [worker] SELECT (CLIENT, 0.5 ms)
|
||||||
|
- [worker] INSERT (CLIENT, 0.5 ms)
|
||||||
|
- [worker] INSERT (CLIENT, 0.3 ms)
|
||||||
|
- [worker] INSERT (CLIENT, 0.4 ms)
|
||||||
|
- [worker] UPDATE (CLIENT, 0.4 ms)
|
||||||
|
- [worker] UPDATE (CLIENT, 0.3 ms)
|
||||||
|
- [worker] INSERT (CLIENT, 0.3 ms)
|
||||||
|
- [worker] cleanup staging (INTERNAL, 4.7 ms)
|
||||||
|
- [worker] S3 DeleteObject (CLIENT, 2.2 ms) aws.s3.key=staging/1b42fa0c1dd84589a58369be9f034238
|
||||||
|
- [worker] UPDATE (CLIENT, 2.2 ms)
|
||||||
|
worker logs with trace_id f5d1897588244cad5fc6ded83b930725: ['job.claimed', 'job.ready']
|
||||||
|
PASS: one trace f5d1897588244cad5fc6ded83b930725: API 202 -> outbox create -> dispatcher publish -> worker consume -> 4 S3 objects + COMMIT ready (37 spans, services: messenger-api, messenger-dispatcher, messenger-worker)
|
||||||
|
- [api] POST /api/v1/chats/{chat_id}/attachments (SERVER, 12.6 ms) http.response.status_code=202
|
||||||
|
- [api] redis GET (CLIENT, 0.2 ms)
|
||||||
|
- [api] SELECT (CLIENT, 0.5 ms)
|
||||||
|
- [api] inspect image (INTERNAL, 0.3 ms)
|
||||||
|
- [api] INSERT (CLIENT, 0.6 ms)
|
||||||
|
- [api] S3 PutObject (CLIENT, 2.5 ms) aws.s3.key=staging/d44ab652329d41458bd477c6cf3ba29c
|
||||||
|
- [api] UPDATE (CLIENT, 0.6 ms)
|
||||||
|
- [api] create attachments (PRODUCER, 1.4 ms) messenger.job.attempt=1
|
||||||
|
- [api] INSERT (CLIENT, 0.4 ms)
|
||||||
|
- [api] INSERT (CLIENT, 0.3 ms)
|
||||||
|
- [api] INSERT (CLIENT, 0.3 ms)
|
||||||
|
- [api] NOTIFY (CLIENT, 0.1 ms)
|
||||||
|
- [dispatcher] publish attachments (PRODUCER, 4.2 ms) messenger.job.attempt=1
|
||||||
|
- [dispatcher] UPDATE (CLIENT, 0.4 ms)
|
||||||
|
- [worker] process attachments.process.v1 (CONSUMER, 135.6 ms) ERROR messenger.job.attempt=1 messenger.job.outcome=retry error.type=storage_unavailable
|
||||||
|
- [worker] SELECT (CLIENT, 0.4 ms)
|
||||||
|
- [worker] UPDATE (CLIENT, 0.4 ms)
|
||||||
|
- [worker] UPDATE (CLIENT, 0.4 ms)
|
||||||
|
- [worker] INSERT (CLIENT, 0.3 ms)
|
||||||
|
- [worker] S3 GetObject (CLIENT, 2.6 ms) aws.s3.key=staging/d44ab652329d41458bd477c6cf3ba29c
|
||||||
|
- [worker] verify source sha256 (INTERNAL, 0.0 ms)
|
||||||
|
- [worker] imaging (INTERNAL, 123.0 ms)
|
||||||
|
- [worker] SELECT (CLIENT, 0.4 ms)
|
||||||
|
- [worker] UPDATE (CLIENT, 0.5 ms)
|
||||||
|
- [worker] UPDATE (CLIENT, 0.3 ms)
|
||||||
|
- [worker] create attachments (PRODUCER, 1.0 ms) messenger.job.attempt=2
|
||||||
|
- [worker] INSERT (CLIENT, 0.6 ms)
|
||||||
|
- [worker] NOTIFY (CLIENT, 0.2 ms)
|
||||||
|
- [dispatcher] publish attachments (PRODUCER, 5.2 ms) messenger.job.attempt=2
|
||||||
|
- [dispatcher] UPDATE (CLIENT, 0.5 ms)
|
||||||
|
- [worker] process attachments.process.v1 (CONSUMER, 144.9 ms) messenger.job.attempt=2 messenger.job.outcome=ready
|
||||||
|
- [worker] SELECT (CLIENT, 0.4 ms)
|
||||||
|
- [worker] UPDATE (CLIENT, 0.4 ms)
|
||||||
|
- [worker] UPDATE (CLIENT, 0.4 ms)
|
||||||
|
- [worker] INSERT (CLIENT, 0.3 ms)
|
||||||
|
- [worker] S3 GetObject (CLIENT, 1.7 ms) aws.s3.key=staging/d44ab652329d41458bd477c6cf3ba29c
|
||||||
|
- [worker] verify source sha256 (INTERNAL, 0.0 ms)
|
||||||
|
- [worker] imaging (INTERNAL, 125.7 ms)
|
||||||
|
- [worker] S3 PutObject (CLIENT, 2.5 ms) aws.s3.key=attachments/214e2e86-8b1d-488b-99d8-5c60fd849c0c/v1/original
|
||||||
|
- [worker] S3 PutObject (CLIENT, 1.8 ms) aws.s3.key=attachments/214e2e86-8b1d-488b-99d8-5c60fd849c0c/v1/optimized.webp
|
||||||
|
- [worker] S3 PutObject (CLIENT, 1.7 ms) aws.s3.key=attachments/214e2e86-8b1d-488b-99d8-5c60fd849c0c/v1/thumbnail.jpg
|
||||||
|
- [worker] S3 PutObject (CLIENT, 1.6 ms) aws.s3.key=attachments/214e2e86-8b1d-488b-99d8-5c60fd849c0c/v1/crop.jpg
|
||||||
|
- [worker] commit result (INTERNAL, 4.4 ms)
|
||||||
|
- [worker] SELECT (CLIENT, 0.4 ms)
|
||||||
|
- [worker] INSERT (CLIENT, 0.4 ms)
|
||||||
|
- [worker] INSERT (CLIENT, 0.3 ms)
|
||||||
|
- [worker] INSERT (CLIENT, 0.2 ms)
|
||||||
|
- [worker] UPDATE (CLIENT, 0.4 ms)
|
||||||
|
- [worker] UPDATE (CLIENT, 0.3 ms)
|
||||||
|
- [worker] INSERT (CLIENT, 0.4 ms)
|
||||||
|
- [worker] cleanup staging (INTERNAL, 4.5 ms)
|
||||||
|
- [worker] S3 DeleteObject (CLIENT, 1.7 ms) aws.s3.key=staging/d44ab652329d41458bd477c6cf3ba29c
|
||||||
|
- [worker] UPDATE (CLIENT, 2.5 ms)
|
||||||
|
- [worker] INSERT (CLIENT, 0.4 ms)
|
||||||
|
worker log: 'job 7bd96513-9f85-471e-b553-ec49f9fe4560 attempt 1: storage_unavailable (temporary) -> retry' error_type=storage_unavailable
|
||||||
|
messenger_job_duration_seconds_count{outcome="retry"}: 0 -> 1
|
||||||
|
PASS: retry in one trace 5d34bb1eecbc90702d2f5e7de5b9bca6: attempt 1 ERROR storage_unavailable -> create -> publish -> attempt 2 ready; retry counted in metrics and logged
|
||||||
|
original trace 38319800cdce5da47cb2b821dc03bf6e: 5 attempts, last outcome failed
|
||||||
|
- [ops] ops replay (INTERNAL, 12.3 ms)
|
||||||
|
- [ops] SELECT (CLIENT, 1.9 ms)
|
||||||
|
- [ops] UPDATE (CLIENT, 0.6 ms)
|
||||||
|
- [ops] UPDATE (CLIENT, 0.4 ms)
|
||||||
|
- [ops] INSERT (CLIENT, 0.5 ms)
|
||||||
|
- [ops] create attachments (PRODUCER, 0.8 ms) links=1 messenger.job.attempt=6
|
||||||
|
- [ops] INSERT (CLIENT, 0.5 ms)
|
||||||
|
- [ops] NOTIFY (CLIENT, 0.2 ms)
|
||||||
|
- [dispatcher] publish attachments (PRODUCER, 70.0 ms) messenger.job.attempt=6
|
||||||
|
- [dispatcher] UPDATE (CLIENT, 66.1 ms)
|
||||||
|
- [worker] process attachments.process.v1 (CONSUMER, 145.2 ms) messenger.job.attempt=6 messenger.job.outcome=ready
|
||||||
|
- [worker] SELECT (CLIENT, 0.5 ms)
|
||||||
|
- [worker] UPDATE (CLIENT, 0.5 ms)
|
||||||
|
- [worker] UPDATE (CLIENT, 0.5 ms)
|
||||||
|
- [worker] INSERT (CLIENT, 0.2 ms)
|
||||||
|
- [worker] S3 GetObject (CLIENT, 1.7 ms) aws.s3.key=staging/31f520a1f9d2445689ed0344a495998f
|
||||||
|
- [worker] verify source sha256 (INTERNAL, 0.0 ms)
|
||||||
|
- [worker] imaging (INTERNAL, 125.7 ms)
|
||||||
|
- [worker] S3 PutObject (CLIENT, 2.5 ms) aws.s3.key=attachments/ebdeb0f4-a519-42b0-808b-3e6cf118b861/v1/original
|
||||||
|
- [worker] S3 PutObject (CLIENT, 2.0 ms) aws.s3.key=attachments/ebdeb0f4-a519-42b0-808b-3e6cf118b861/v1/optimized.webp
|
||||||
|
- [worker] S3 PutObject (CLIENT, 1.6 ms) aws.s3.key=attachments/ebdeb0f4-a519-42b0-808b-3e6cf118b861/v1/thumbnail.jpg
|
||||||
|
- [worker] S3 PutObject (CLIENT, 1.6 ms) aws.s3.key=attachments/ebdeb0f4-a519-42b0-808b-3e6cf118b861/v1/crop.jpg
|
||||||
|
- [worker] commit result (INTERNAL, 4.4 ms)
|
||||||
|
- [worker] SELECT (CLIENT, 0.5 ms)
|
||||||
|
- [worker] INSERT (CLIENT, 0.3 ms)
|
||||||
|
- [worker] INSERT (CLIENT, 0.2 ms)
|
||||||
|
- [worker] INSERT (CLIENT, 0.2 ms)
|
||||||
|
- [worker] UPDATE (CLIENT, 0.4 ms)
|
||||||
|
- [worker] UPDATE (CLIENT, 0.3 ms)
|
||||||
|
- [worker] INSERT (CLIENT, 0.3 ms)
|
||||||
|
- [worker] cleanup staging (INTERNAL, 4.1 ms)
|
||||||
|
- [worker] S3 DeleteObject (CLIENT, 2.1 ms) aws.s3.key=staging/31f520a1f9d2445689ed0344a495998f
|
||||||
|
- [worker] UPDATE (CLIENT, 1.8 ms)
|
||||||
|
replay trace 9417040b8f913c26894caaaf0996e442: create span links -> ['38319800cdce5da47cb2b821dc03bf6e'] (original upload trace 38319800cdce5da47cb2b821dc03bf6e)
|
||||||
|
PASS: replay = new trace (root 'ops replay') with a link to the original upload; worker finished it
|
||||||
|
4 passed, 0 failed
|
||||||
@@ -0,0 +1,124 @@
|
|||||||
|
{
|
||||||
|
"date": "2026-11-01 20:42",
|
||||||
|
"api_limit_mib": 512,
|
||||||
|
"api_limit_cpus": 1.0,
|
||||||
|
"runs": [
|
||||||
|
{
|
||||||
|
"scenario": "10 x ~9.5 MiB PNG (noise)",
|
||||||
|
"concurrency": 10,
|
||||||
|
"file_bytes": [
|
||||||
|
9953047
|
||||||
|
],
|
||||||
|
"total_mib": 94.9,
|
||||||
|
"all_202": true,
|
||||||
|
"wall_seconds": 1.95,
|
||||||
|
"latency_s": {
|
||||||
|
"p50": 1.77,
|
||||||
|
"max": 1.95
|
||||||
|
},
|
||||||
|
"sampler_rounds": 16,
|
||||||
|
"replicas": [
|
||||||
|
{
|
||||||
|
"replica": "171a473b0d28",
|
||||||
|
"uploads_served": 5,
|
||||||
|
"rss_before_mib": 109.6,
|
||||||
|
"rss_peak_mib": 124.5,
|
||||||
|
"rss_growth_mib": 14.9,
|
||||||
|
"rss_peak_sampled_mib": 124.5,
|
||||||
|
"cgroup_peak_mib": 162.8,
|
||||||
|
"cgroup_peak_sampled_mib": 160.2
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"replica": "1db2fedff78f",
|
||||||
|
"uploads_served": 5,
|
||||||
|
"rss_before_mib": 110.5,
|
||||||
|
"rss_peak_mib": 125.6,
|
||||||
|
"rss_growth_mib": 15.1,
|
||||||
|
"rss_peak_sampled_mib": 125.6,
|
||||||
|
"cgroup_peak_mib": 163.3,
|
||||||
|
"cgroup_peak_sampled_mib": 160.8
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"scenario": "10 x 40 Mpx PNG (8000x5000 RGB)",
|
||||||
|
"concurrency": 10,
|
||||||
|
"file_bytes": [
|
||||||
|
116714
|
||||||
|
],
|
||||||
|
"total_mib": 1.1,
|
||||||
|
"all_202": true,
|
||||||
|
"wall_seconds": 1.3,
|
||||||
|
"latency_s": {
|
||||||
|
"p50": 0.98,
|
||||||
|
"max": 1.3
|
||||||
|
},
|
||||||
|
"sampler_rounds": 9,
|
||||||
|
"replicas": [
|
||||||
|
{
|
||||||
|
"replica": "171a473b0d28",
|
||||||
|
"uploads_served": 5,
|
||||||
|
"rss_before_mib": 107.6,
|
||||||
|
"rss_peak_mib": 112.4,
|
||||||
|
"rss_growth_mib": 4.8,
|
||||||
|
"rss_peak_sampled_mib": 112.4,
|
||||||
|
"cgroup_peak_mib": 101.5,
|
||||||
|
"cgroup_peak_sampled_mib": 101.4
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"replica": "1db2fedff78f",
|
||||||
|
"uploads_served": 5,
|
||||||
|
"rss_before_mib": 107.1,
|
||||||
|
"rss_peak_mib": 109.8,
|
||||||
|
"rss_growth_mib": 2.7,
|
||||||
|
"rss_peak_sampled_mib": 109.8,
|
||||||
|
"cgroup_peak_mib": 100.0,
|
||||||
|
"cgroup_peak_sampled_mib": 100.0
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"scenario": "20 mixed: large + 40 Mpx + JPEG",
|
||||||
|
"concurrency": 20,
|
||||||
|
"file_bytes": [
|
||||||
|
1492,
|
||||||
|
116714,
|
||||||
|
9953047
|
||||||
|
],
|
||||||
|
"total_mib": 133.2,
|
||||||
|
"all_202": true,
|
||||||
|
"wall_seconds": 1.95,
|
||||||
|
"latency_s": {
|
||||||
|
"p50": 1.45,
|
||||||
|
"max": 1.95
|
||||||
|
},
|
||||||
|
"sampler_rounds": 11,
|
||||||
|
"replicas": [
|
||||||
|
{
|
||||||
|
"replica": "171a473b0d28",
|
||||||
|
"uploads_served": 10,
|
||||||
|
"rss_before_mib": 109.0,
|
||||||
|
"rss_peak_mib": 136.3,
|
||||||
|
"rss_growth_mib": 27.3,
|
||||||
|
"rss_peak_sampled_mib": 136.3,
|
||||||
|
"cgroup_peak_mib": 191.4,
|
||||||
|
"cgroup_peak_sampled_mib": 191.4
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"replica": "1db2fedff78f",
|
||||||
|
"uploads_served": 10,
|
||||||
|
"rss_before_mib": 110.6,
|
||||||
|
"rss_peak_mib": 135.0,
|
||||||
|
"rss_growth_mib": 24.3,
|
||||||
|
"rss_peak_sampled_mib": 135.0,
|
||||||
|
"cgroup_peak_mib": 172.8,
|
||||||
|
"cgroup_peak_sampled_mib": 171.4
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"single_file_extra_rss_mib": {
|
||||||
|
"naive": 161,
|
||||||
|
"streaming": 0
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,13 @@
|
|||||||
|
# upload-memory: lab4: >=10 concurrent uploads, peak memory of api replicas
|
||||||
|
# date: 2026-11-01 20:42:33, commit: 470d9d6 + uncommitted changes
|
||||||
|
# command: python.exe student-tests/upload_memory.py --output evidence/upload-memory.json
|
||||||
|
# exit code: 0, 53.4 s
|
||||||
|
|
||||||
|
client: Python 3.11.1 on Windows 10
|
||||||
|
preparing synthetic images (random-noise PNG ~9.5 MiB each, 40 Mpx PNG)...
|
||||||
|
large: 9953047 bytes; 40 Mpx: 116714 bytes
|
||||||
|
{"scenario": "10 x ~9.5 MiB PNG (noise)", "concurrency": 10, "file_bytes": [9953047], "total_mib": 94.9, "all_202": true, "wall_seconds": 1.95, "latency_s": {"p50": 1.77, "max": 1.95}, "sampler_rounds": 16, "replicas": [{"replica": "171a473b0d28", "uploads_served": 5, "rss_before_mib": 109.6, "rss_peak_mib": 124.5, "rss_growth_mib": 14.9, "rss_peak_sampled_mib": 124.5, "cgroup_peak_mib": 162.8, "cgroup_peak_sampled_mib": 160.2}, {"replica": "1db2fedff78f", "uploads_served": 5, "rss_before_mib": 110.5, "rss_peak_mib": 125.6, "rss_growth_mib": 15.1, "rss_peak_sampled_mib": 125.6, "cgroup_peak_mib": 163.3, "cgroup_peak_sampled_mib": 160.8}]}
|
||||||
|
{"scenario": "10 x 40 Mpx PNG (8000x5000 RGB)", "concurrency": 10, "file_bytes": [116714], "total_mib": 1.1, "all_202": true, "wall_seconds": 1.3, "latency_s": {"p50": 0.98, "max": 1.3}, "sampler_rounds": 9, "replicas": [{"replica": "171a473b0d28", "uploads_served": 5, "rss_before_mib": 107.6, "rss_peak_mib": 112.4, "rss_growth_mib": 4.8, "rss_peak_sampled_mib": 112.4, "cgroup_peak_mib": 101.5, "cgroup_peak_sampled_mib": 101.4}, {"replica": "1db2fedff78f", "uploads_served": 5, "rss_before_mib": 107.1, "rss_peak_mib": 109.8, "rss_growth_mib": 2.7, "rss_peak_sampled_mib": 109.8, "cgroup_peak_mib": 100.0, "cgroup_peak_sampled_mib": 100.0}]}
|
||||||
|
{"scenario": "20 mixed: large + 40 Mpx + JPEG", "concurrency": 20, "file_bytes": [1492, 116714, 9953047], "total_mib": 133.2, "all_202": true, "wall_seconds": 1.95, "latency_s": {"p50": 1.45, "max": 1.95}, "sampler_rounds": 11, "replicas": [{"replica": "171a473b0d28", "uploads_served": 10, "rss_before_mib": 109.0, "rss_peak_mib": 136.3, "rss_growth_mib": 27.3, "rss_peak_sampled_mib": 136.3, "cgroup_peak_mib": 191.4, "cgroup_peak_sampled_mib": 191.4}, {"replica": "1db2fedff78f", "uploads_served": 10, "rss_before_mib": 110.6, "rss_peak_mib": 135.0, "rss_growth_mib": 24.3, "rss_peak_sampled_mib": 135.0, "cgroup_peak_mib": 172.8, "cgroup_peak_sampled_mib": 171.4}]}
|
||||||
|
one 8000x5000 RGB PNG inside the api container, extra peak RSS: naive (body in RAM + full decode) 161 MiB, streaming check 0 MiB
|
||||||
|
PASS: all uploads succeeded; largest RSS growth of one replica: 27.3 MiB (limit 512 MiB); results in evidence/upload-memory.json
|
||||||
@@ -0,0 +1,51 @@
|
|||||||
|
"""Учебное внедрение сбоев для тестов отказов (только при FAULT_INJECTION=1).
|
||||||
|
|
||||||
|
Тест создаёт в общем volume /faults файлы <точка>.<n>, по одному на срабатывание:
|
||||||
|
|
||||||
|
docker compose exec -T worker sh -c 'touch /faults/worker_crash_after_put.1'
|
||||||
|
|
||||||
|
Процесс, дошедший до точки, атомарно удаляет один файл (os.remove успешен ровно у одного процесса)
|
||||||
|
и выполняет сбой. Так сбой случается заданное число раз и не зацикливает перезапуски.
|
||||||
|
|
||||||
|
Точки:
|
||||||
|
dispatcher_crash_before_publish — COMMIT задания уже был, publish ещё нет (окно COMMIT -> publish)
|
||||||
|
dispatcher_crash_after_confirm — брокер подтвердил публикацию, отметка в outbox ещё не записана
|
||||||
|
worker_crash_after_put — объекты записаны в S3, COMMIT результата ещё нет
|
||||||
|
worker_crash_after_commit — COMMIT результата был, ack ещё нет
|
||||||
|
worker_freeze — worker «завис»: не продлевает аренду и спит 3 x lease
|
||||||
|
worker_s3_error — временная ошибка S3 при записи результата
|
||||||
|
imaging_hang / imaging_invalid / imaging_oom — сбой внутри процесса обработки изображения
|
||||||
|
"""
|
||||||
|
import logging
|
||||||
|
import os
|
||||||
|
import sys
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
import settings
|
||||||
|
|
||||||
|
log = logging.getLogger("messenger.faults")
|
||||||
|
|
||||||
|
|
||||||
|
def hit(point: str) -> bool:
|
||||||
|
if not settings.FAULT_INJECTION:
|
||||||
|
return False
|
||||||
|
directory = Path(settings.FAULTS_DIR)
|
||||||
|
try:
|
||||||
|
candidates = sorted(directory.glob(point + ".*"))
|
||||||
|
except OSError:
|
||||||
|
return False
|
||||||
|
for path in candidates:
|
||||||
|
try:
|
||||||
|
path.unlink()
|
||||||
|
except FileNotFoundError:
|
||||||
|
continue # этот файл забрал другой процесс
|
||||||
|
log.warning("FAULT INJECTION: %s", point)
|
||||||
|
return True
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
|
def crash(point: str) -> None:
|
||||||
|
"""Аварийное завершение всего процесса без очистки (как SIGKILL/OOM): без ack, без закрытия соединений."""
|
||||||
|
if hit(point):
|
||||||
|
sys.stderr.flush()
|
||||||
|
os._exit(137)
|
||||||
+209
@@ -0,0 +1,209 @@
|
|||||||
|
"""Преобразование изображения, pipeline_version=1. Запускается worker'ом ОТДЕЛЬНЫМ процессом с лимитами.
|
||||||
|
|
||||||
|
python imaging.py <source> <out_dir> <content_type> <max_pixels>
|
||||||
|
|
||||||
|
Почему отдельный процесс: декодер Pillow работает в C, его нельзя прервать изнутри Python и нельзя
|
||||||
|
ограничить по памяти отдельно от остального worker. Процесс сам ставит себе RLIMIT_AS (память,
|
||||||
|
IMAGING_MEMORY_MB), RLIMIT_CPU (процессорное время) и RLIMIT_FSIZE до импорта Pillow, а worker
|
||||||
|
убивает его по таймауту JOB_TIMEOUT_SECONDS.
|
||||||
|
Падение, зависание или нехватка памяти здесь не роняют worker и не ломают подключение к брокеру.
|
||||||
|
|
||||||
|
Результат: файлы thumbnail.jpg, crop.jpg, optimized.webp в out_dir и одна строка JSON в stdout.
|
||||||
|
Код выхода: 0 — успех; 3 — невалидное изображение; 4 — слишком много пикселей; 5 — не хватило памяти;
|
||||||
|
прочее (в том числе убит сигналом) — неожиданная ошибка, worker считает её временной.
|
||||||
|
|
||||||
|
Правила вариантов (contracts/README.md):
|
||||||
|
- thumbnail: JPEG, вписан в 256x256 с сохранением пропорций, без увеличения, размеры округляются вниз
|
||||||
|
(минимум 1 px), EXIF orientation применяется, прозрачность накладывается на белый фон. 320x200 -> 256x160.
|
||||||
|
- crop: JPEG 128x128, центрированный квадрат (маленькие исходники увеличиваются).
|
||||||
|
- optimized: WebP, вписан в 1280x1280 без увеличения, quality=80, method=4; прозрачность сохраняется.
|
||||||
|
Во всех вариантах нет EXIF/XMP/комментариев: в save() передаётся только ICC-профиль (цвет), остальное
|
||||||
|
отбрасывается. Pillow при одинаковой версии даёт побайтно одинаковый результат, поэтому повтор job
|
||||||
|
пишет в те же ключи S3 те же байты.
|
||||||
|
"""
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import resource
|
||||||
|
import sys
|
||||||
|
import time
|
||||||
|
|
||||||
|
|
||||||
|
def apply_limits() -> None:
|
||||||
|
"""Лимиты на себя, ДО импорта Pillow: память (RLIMIT_AS), CPU, размер записываемых файлов."""
|
||||||
|
memory = int(os.environ.get("IMAGING_MEMORY_MB", "640")) * 1024 * 1024
|
||||||
|
cpu = int(os.environ.get("IMAGING_CPU_SECONDS", "60"))
|
||||||
|
resource.setrlimit(resource.RLIMIT_AS, (memory, memory))
|
||||||
|
resource.setrlimit(resource.RLIMIT_CPU, (cpu, cpu + 5))
|
||||||
|
resource.setrlimit(resource.RLIMIT_FSIZE, (64 * 1024 * 1024, 64 * 1024 * 1024))
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
apply_limits()
|
||||||
|
|
||||||
|
from PIL import Image # noqa: E402 — после apply_limits: память декодера тоже под лимитом
|
||||||
|
|
||||||
|
PIPELINE_VERSION = 1
|
||||||
|
THUMB_BOX = 256
|
||||||
|
CROP_SIZE = 128
|
||||||
|
OPTIMIZED_BOX = 1280
|
||||||
|
THUMB_QUALITY = 85
|
||||||
|
CROP_QUALITY = 85
|
||||||
|
WEBP_QUALITY = 80
|
||||||
|
WEBP_METHOD = 4
|
||||||
|
|
||||||
|
EXIT_INVALID = 3
|
||||||
|
EXIT_TOO_LARGE = 4
|
||||||
|
EXIT_MEMORY = 5
|
||||||
|
|
||||||
|
FORMATS = {"image/png": "PNG", "image/jpeg": "JPEG"}
|
||||||
|
# EXIF orientation -> преобразование (как в PIL.ImageOps.exif_transpose)
|
||||||
|
ORIENTATION = {
|
||||||
|
2: [Image.Transpose.FLIP_LEFT_RIGHT],
|
||||||
|
3: [Image.Transpose.ROTATE_180],
|
||||||
|
4: [Image.Transpose.FLIP_TOP_BOTTOM],
|
||||||
|
5: [Image.Transpose.TRANSPOSE],
|
||||||
|
6: [Image.Transpose.ROTATE_270],
|
||||||
|
7: [Image.Transpose.TRANSVERSE],
|
||||||
|
8: [Image.Transpose.ROTATE_90],
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
class TooLarge(Exception):
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
def fit(width: int, height: int, box: int) -> tuple[int, int]:
|
||||||
|
"""Вписать в box x box без увеличения; размеры вниз, минимум 1 px. Целочисленно, без float."""
|
||||||
|
if width <= box and height <= box:
|
||||||
|
return width, height
|
||||||
|
if width >= height:
|
||||||
|
return box, max(1, height * box // width)
|
||||||
|
return max(1, width * box // height), box
|
||||||
|
|
||||||
|
|
||||||
|
def orientation_of(image: Image.Image) -> int:
|
||||||
|
try:
|
||||||
|
value = image.getexif().get(0x0112, 1)
|
||||||
|
except Exception: # noqa: BLE001 — битый EXIF не делает картинку невалидной, ориентацию не трогаем
|
||||||
|
return 1
|
||||||
|
return value if value in ORIENTATION else 1
|
||||||
|
|
||||||
|
|
||||||
|
def normalize(image: Image.Image) -> Image.Image:
|
||||||
|
"""Любой режим PNG/JPEG -> L, RGB или RGBA (RGBA только если есть прозрачность)."""
|
||||||
|
mode = image.mode
|
||||||
|
transparent = "transparency" in image.info
|
||||||
|
if mode in ("I;16", "I;16B", "I;16L", "I"):
|
||||||
|
image = image.convert("I").point(lambda value: value * (1 / 256)).convert("L")
|
||||||
|
mode = "L"
|
||||||
|
if mode == "1":
|
||||||
|
return image.convert("L")
|
||||||
|
if mode in ("P", "PA"):
|
||||||
|
return image.convert("RGBA" if transparent or mode == "PA" else "RGB")
|
||||||
|
if mode in ("LA", "La", "RGBa"):
|
||||||
|
return image.convert("RGBA")
|
||||||
|
if mode in ("L", "RGB") and transparent:
|
||||||
|
return image.convert("RGBA")
|
||||||
|
if mode in ("L", "RGB", "RGBA"):
|
||||||
|
return image
|
||||||
|
return image.convert("RGB") # CMYK, YCbCr, LAB, ...
|
||||||
|
|
||||||
|
|
||||||
|
def on_white(image: Image.Image) -> Image.Image:
|
||||||
|
"""Прозрачность -> белый фон (для JPEG)."""
|
||||||
|
if image.mode != "RGBA":
|
||||||
|
return image
|
||||||
|
background = Image.new("RGB", image.size, (255, 255, 255))
|
||||||
|
background.paste(image, mask=image.getchannel("A"))
|
||||||
|
return background
|
||||||
|
|
||||||
|
|
||||||
|
def process(source: str, out_dir: str, content_type: str, max_pixels: int) -> dict:
|
||||||
|
Image.MAX_IMAGE_PIXELS = max_pixels
|
||||||
|
with Image.open(source) as image:
|
||||||
|
if image.format != FORMATS.get(content_type):
|
||||||
|
raise ValueError("format does not match stored content type")
|
||||||
|
width, height = image.size
|
||||||
|
if width * height > max_pixels:
|
||||||
|
raise TooLarge()
|
||||||
|
orientation = orientation_of(image)
|
||||||
|
icc = image.info.get("icc_profile")
|
||||||
|
swap = orientation in (5, 6, 7, 8)
|
||||||
|
oriented = (height, width) if swap else (width, height)
|
||||||
|
|
||||||
|
# Сначала уменьшаем до optimized (<= 1280), потом поворачиваем: квадратные рамки 1280/256/128
|
||||||
|
# не зависят от поворота, а поворот маленькой картинки дешевле, чем копия 40-мегапиксельной.
|
||||||
|
target = fit(*oriented, OPTIMIZED_BOX)
|
||||||
|
stored_target = (target[1], target[0]) if swap else target
|
||||||
|
if image.format == "JPEG":
|
||||||
|
image.draft(None, stored_target) # декодирование сразу в масштабе 1/2..1/8, если хватает
|
||||||
|
image.load()
|
||||||
|
work = normalize(image)
|
||||||
|
if work.size != stored_target:
|
||||||
|
work = work.resize(stored_target, Image.Resampling.LANCZOS, reducing_gap=3.0)
|
||||||
|
elif work is image:
|
||||||
|
work = work.copy()
|
||||||
|
for step in ORIENTATION.get(orientation, []):
|
||||||
|
work = work.transpose(step)
|
||||||
|
|
||||||
|
extra = {"icc_profile": icc} if icc else {}
|
||||||
|
results = []
|
||||||
|
|
||||||
|
optimized = work if work.mode != "L" else work.convert("RGB")
|
||||||
|
path = os.path.join(out_dir, "optimized.webp")
|
||||||
|
optimized.save(path, format="WEBP", quality=WEBP_QUALITY, method=WEBP_METHOD, **extra)
|
||||||
|
results.append({"name": "optimized", "content_type": "image/webp", "width": optimized.width,
|
||||||
|
"height": optimized.height, "file": "optimized.webp"})
|
||||||
|
|
||||||
|
thumb_size = fit(*oriented, THUMB_BOX) # от размеров ИСХОДНИКА, а не промежуточной копии
|
||||||
|
thumb = work if work.size == thumb_size else work.resize(thumb_size, Image.Resampling.LANCZOS)
|
||||||
|
thumb = on_white(thumb)
|
||||||
|
path = os.path.join(out_dir, "thumbnail.jpg")
|
||||||
|
thumb.save(path, format="JPEG", quality=THUMB_QUALITY, optimize=True, **extra)
|
||||||
|
results.append({"name": "thumbnail", "content_type": "image/jpeg", "width": thumb.width,
|
||||||
|
"height": thumb.height, "file": "thumbnail.jpg"})
|
||||||
|
|
||||||
|
side = min(work.size)
|
||||||
|
left, top = (work.width - side) // 2, (work.height - side) // 2
|
||||||
|
crop = work.crop((left, top, left + side, top + side))
|
||||||
|
crop = on_white(crop.resize((CROP_SIZE, CROP_SIZE), Image.Resampling.LANCZOS))
|
||||||
|
path = os.path.join(out_dir, "crop.jpg")
|
||||||
|
crop.save(path, format="JPEG", quality=CROP_QUALITY, optimize=True, **extra)
|
||||||
|
results.append({"name": "crop", "content_type": "image/jpeg", "width": CROP_SIZE, "height": CROP_SIZE,
|
||||||
|
"file": "crop.jpg"})
|
||||||
|
|
||||||
|
return {"pipeline_version": PIPELINE_VERSION, "source": {"width": width, "height": height,
|
||||||
|
"orientation": orientation}, "variants": results}
|
||||||
|
|
||||||
|
|
||||||
|
def injected_fault() -> None:
|
||||||
|
"""Учебные сбои (FAULT_IMAGING задаёт worker, только при FAULT_INJECTION=1)."""
|
||||||
|
fault = os.environ.get("FAULT_IMAGING", "")
|
||||||
|
if fault == "hang":
|
||||||
|
time.sleep(3600)
|
||||||
|
elif fault == "invalid":
|
||||||
|
raise ValueError("injected invalid image")
|
||||||
|
elif fault == "oom":
|
||||||
|
blocks = []
|
||||||
|
while True:
|
||||||
|
blocks.append(bytearray(64 * 1024 * 1024))
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> int:
|
||||||
|
source, out_dir, content_type, max_pixels = sys.argv[1], sys.argv[2], sys.argv[3], int(sys.argv[4])
|
||||||
|
try:
|
||||||
|
injected_fault()
|
||||||
|
result = process(source, out_dir, content_type, max_pixels)
|
||||||
|
except (TooLarge, Image.DecompressionBombError):
|
||||||
|
return EXIT_TOO_LARGE
|
||||||
|
except MemoryError:
|
||||||
|
return EXIT_MEMORY
|
||||||
|
except (OSError, SyntaxError, ValueError, IndexError, TypeError, EOFError, Image.UnidentifiedImageError):
|
||||||
|
# Pillow сообщает о битом файле разными исключениями; подробности не нужны ни клиенту, ни логу.
|
||||||
|
return EXIT_INVALID
|
||||||
|
print(json.dumps(result), flush=True)
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
sys.exit(main())
|
||||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,13 @@
|
|||||||
|
# Grafana provisioning: dashboard из файлов (infra/grafana/dashboards/*.json монтируется только для чтения).
|
||||||
|
apiVersion: 1
|
||||||
|
|
||||||
|
providers:
|
||||||
|
- name: messenger
|
||||||
|
folder: Messenger
|
||||||
|
type: file
|
||||||
|
disableDeletion: true
|
||||||
|
allowUiUpdates: false
|
||||||
|
updateIntervalSeconds: 30
|
||||||
|
options:
|
||||||
|
path: /etc/grafana/dashboards
|
||||||
|
foldersFromFilesStructure: false
|
||||||
@@ -0,0 +1,67 @@
|
|||||||
|
# Grafana provisioning: источники данных (импортируются при старте, ручной настройки нет).
|
||||||
|
# Grafana сама ничего не хранит из трёх сигналов — только интерфейс и запросы к хранилищам.
|
||||||
|
# "$$" — буквальный "$" (иначе Grafana подставит переменную окружения).
|
||||||
|
apiVersion: 1
|
||||||
|
|
||||||
|
datasources:
|
||||||
|
- name: Prometheus
|
||||||
|
uid: prometheus
|
||||||
|
type: prometheus
|
||||||
|
access: proxy
|
||||||
|
url: http://prometheus:9090
|
||||||
|
isDefault: true
|
||||||
|
editable: false
|
||||||
|
jsonData:
|
||||||
|
httpMethod: POST
|
||||||
|
prometheusType: Prometheus
|
||||||
|
prometheusVersion: 3.15.1
|
||||||
|
timeInterval: 15s
|
||||||
|
manageAlerts: false # правила алертов в Git (infra/prometheus/rules), Grafana их только показывает
|
||||||
|
|
||||||
|
- name: Loki
|
||||||
|
uid: loki
|
||||||
|
type: loki
|
||||||
|
access: proxy
|
||||||
|
url: http://loki:3100
|
||||||
|
editable: false
|
||||||
|
jsonData:
|
||||||
|
maxLines: 1000
|
||||||
|
# trace_id — structured metadata записи (OTLP): ссылка «лог -> трасса» в Tempo.
|
||||||
|
derivedFields:
|
||||||
|
- name: TraceID
|
||||||
|
matcherType: label
|
||||||
|
matcherRegex: trace_id
|
||||||
|
datasourceUid: tempo
|
||||||
|
url: "$${__value.raw}"
|
||||||
|
urlDisplayLabel: Открыть трассу
|
||||||
|
|
||||||
|
- name: Tempo
|
||||||
|
uid: tempo
|
||||||
|
type: tempo
|
||||||
|
access: proxy
|
||||||
|
url: http://tempo:3200
|
||||||
|
editable: false
|
||||||
|
jsonData:
|
||||||
|
# «спан -> логи»: те же trace_id в Loki, окно ±5 минут вокруг спана.
|
||||||
|
tracesToLogsV2:
|
||||||
|
datasourceUid: loki
|
||||||
|
spanStartTimeShift: -5m
|
||||||
|
spanEndTimeShift: 5m
|
||||||
|
filterByTraceID: false
|
||||||
|
filterBySpanID: false
|
||||||
|
customQuery: true
|
||||||
|
query: '{service_name="$${__span.tags["service.name"]}"} | trace_id="$${__trace.traceId}"'
|
||||||
|
# «спан -> метрики»: задержка и ошибки сервиса за время спана.
|
||||||
|
tracesToMetrics:
|
||||||
|
datasourceUid: prometheus
|
||||||
|
spanStartTimeShift: -5m
|
||||||
|
spanEndTimeShift: 5m
|
||||||
|
queries:
|
||||||
|
- name: API p95
|
||||||
|
query: 'histogram_quantile(0.95, sum by (le) (rate(http_server_request_duration_seconds_bucket{job="messenger-api", http_route!~"/health/.*"}[1m])))'
|
||||||
|
nodeGraph:
|
||||||
|
enabled: true
|
||||||
|
search:
|
||||||
|
hide: false
|
||||||
|
lokiSearch:
|
||||||
|
datasourceUid: loki
|
||||||
@@ -0,0 +1,77 @@
|
|||||||
|
# HAProxy — единственная точка входа: клиент -> lb:8080 -> api (2+ реплики).
|
||||||
|
# Сессии и данные общие (Redis, PostgreSQL), поэтому любой запрос может обслужить
|
||||||
|
# любая реплика: балансировка round robin, без привязки клиента к реплике (no sticky).
|
||||||
|
|
||||||
|
global
|
||||||
|
log stdout format raw local0 info
|
||||||
|
maxconn 4096
|
||||||
|
# При остановке контейнера (SIGUSR1, soft stop) дождаться начатых запросов, но не дольше 10 с.
|
||||||
|
hard-stop-after 10s
|
||||||
|
|
||||||
|
# Список реплик берётся из DNS Docker: имя сервиса "api" -> IP всех запущенных реплик.
|
||||||
|
# Остановили реплику или добавили новую — HAProxy увидит это без перезапуска.
|
||||||
|
resolvers docker
|
||||||
|
nameserver dns 127.0.0.11:53
|
||||||
|
resolve_retries 3
|
||||||
|
timeout resolve 1s
|
||||||
|
timeout retry 1s
|
||||||
|
hold valid 2s # перечитывать список реплик раз в 2 с
|
||||||
|
hold obsolete 2s # IP, пропавший из DNS (контейнер остановлен), убрать через 2 с
|
||||||
|
hold nx 2s
|
||||||
|
hold other 2s
|
||||||
|
hold refused 2s
|
||||||
|
hold timeout 2s
|
||||||
|
accepted_payload_size 8192
|
||||||
|
|
||||||
|
defaults
|
||||||
|
mode http
|
||||||
|
log global
|
||||||
|
option httplog
|
||||||
|
option dontlog-normal # в лог только ошибки, а не каждый запрос
|
||||||
|
timeout connect 1s # сеть внутри Docker быстрая: 1 с хватает, мёртвый адрес не держит запрос
|
||||||
|
timeout client 30s
|
||||||
|
# Ожидание ответа API после того, как запрос (включая тело файла) передан. Самый долгий ответ — загрузка
|
||||||
|
# при зависшем S3: PUT 2 попытки x (1 + 5 с) + откат 3 с + БД до 8 с ~ 24 с. Пока тело файла идёт,
|
||||||
|
# сервер «активен», этот таймаут не срабатывает; приём тела ограничивает сам API (UPLOAD_*_TIMEOUT).
|
||||||
|
timeout server 30s
|
||||||
|
timeout http-request 10s
|
||||||
|
timeout http-keep-alive 5s
|
||||||
|
timeout queue 5s
|
||||||
|
timeout check 1s
|
||||||
|
# Ошибка СОЕДИНЕНИЯ (запрос ещё не отправлен) — повторить на другой реплике, это безопасно для любого метода.
|
||||||
|
# Запрос, который уже ушёл в API, HAProxy сам не повторяет: POST повторяет клиент с тем же Idempotency-Key.
|
||||||
|
retries 2
|
||||||
|
option redispatch 1
|
||||||
|
|
||||||
|
frontend api_in
|
||||||
|
bind :8080
|
||||||
|
option forwardfor
|
||||||
|
default_backend api
|
||||||
|
|
||||||
|
backend api
|
||||||
|
balance roundrobin
|
||||||
|
# Проверка готовности: /health/ready = API отвечает и видит PostgreSQL и Redis.
|
||||||
|
option httpchk
|
||||||
|
http-check send meth GET uri /health/ready ver HTTP/1.1 hdr Host lb
|
||||||
|
http-check expect status 200
|
||||||
|
# До 6 реплик (2 обычно, 3 во время rolling update); пустые слоты стоят в MAINT.
|
||||||
|
# Проверка раз в 1 с, при смене состояния раз в 0,5 с; 2 неудачи подряд -> DOWN, 2 удачи -> UP.
|
||||||
|
# observe layer4: первая же ошибка соединения у живого запроса сразу выводит реплику из балансировки.
|
||||||
|
server-template api 6 api:8080 check inter 1s fastinter 500ms downinter 2s fall 2 rise 2 observe layer4 error-limit 1 on-error mark-down resolvers docker resolve-prefer ipv4 init-addr none
|
||||||
|
# Аварийный режим (как panic mode у Envoy): если НИ ОДНА реплика не готова (например, упал Redis
|
||||||
|
# или PostgreSQL), запросы всё равно идут в API. Клиент получает ответ самого API — 503 в формате
|
||||||
|
# контракта с X-Request-Id, /health/live — 200, а регистрация, которой Redis не нужен, работает.
|
||||||
|
# Пока готова хоть одна реплика, эти записи не используются.
|
||||||
|
# allow-dup-ip: те же IP, что у основных слотов (по умолчанию HAProxy запрещает повтор адреса в backend).
|
||||||
|
server-template fallback 6 api:8080 backup no-check resolvers docker resolve-prefer ipv4 resolve-opts allow-dup-ip init-addr none
|
||||||
|
option allbackups
|
||||||
|
|
||||||
|
# Статистика HAProxy: http://127.0.0.1:8404/stats (опубликовано только на 127.0.0.1).
|
||||||
|
# /lb-health — healthcheck контейнера lb: 503, если направить запрос некуда.
|
||||||
|
frontend stats
|
||||||
|
bind :8404
|
||||||
|
monitor-uri /lb-health
|
||||||
|
monitor fail if { nbsrv(api) lt 1 }
|
||||||
|
stats enable
|
||||||
|
stats uri /stats
|
||||||
|
stats refresh 2s
|
||||||
@@ -0,0 +1,52 @@
|
|||||||
|
# Loki 3.7.9 — хранилище ЛОГОВ (single binary, файловая система в volume lokidata).
|
||||||
|
# Логи приходят от Collector по OTLP (/otlp/v1/logs). Ресурсные атрибуты service.name,
|
||||||
|
# service.instance.id, deployment.environment.name становятся индексными метками (service_name, ...);
|
||||||
|
# атрибуты записи (event, request_id, trace_id, span_id, severity) — structured metadata, по ним
|
||||||
|
# фильтруют без индекса: {service_name="messenger-api"} | trace_id="<id>".
|
||||||
|
# Проверка: loki -config.file=loki.yaml -verify-config
|
||||||
|
auth_enabled: false
|
||||||
|
|
||||||
|
server:
|
||||||
|
http_listen_port: 3100
|
||||||
|
grpc_listen_port: 9095
|
||||||
|
log_level: warn
|
||||||
|
|
||||||
|
common:
|
||||||
|
instance_addr: 127.0.0.1
|
||||||
|
path_prefix: /loki
|
||||||
|
replication_factor: 1
|
||||||
|
ring:
|
||||||
|
kvstore:
|
||||||
|
store: inmemory
|
||||||
|
storage:
|
||||||
|
filesystem:
|
||||||
|
chunks_directory: /loki/chunks
|
||||||
|
rules_directory: /loki/rules
|
||||||
|
|
||||||
|
schema_config:
|
||||||
|
configs:
|
||||||
|
- from: "2024-01-01"
|
||||||
|
store: tsdb
|
||||||
|
object_store: filesystem
|
||||||
|
schema: v13
|
||||||
|
index:
|
||||||
|
prefix: index_
|
||||||
|
period: 24h
|
||||||
|
|
||||||
|
limits_config:
|
||||||
|
allow_structured_metadata: true # нужно для OTLP: trace_id/span_id и атрибуты записи
|
||||||
|
retention_period: 168h # 7 дней учебной истории
|
||||||
|
ingestion_rate_mb: 8
|
||||||
|
ingestion_burst_size_mb: 16
|
||||||
|
max_query_lookback: 168h
|
||||||
|
|
||||||
|
compactor:
|
||||||
|
working_directory: /loki/compactor
|
||||||
|
retention_enabled: true
|
||||||
|
delete_request_store: filesystem
|
||||||
|
|
||||||
|
pattern_ingester:
|
||||||
|
enabled: false
|
||||||
|
|
||||||
|
analytics:
|
||||||
|
reporting_enabled: false
|
||||||
@@ -0,0 +1,129 @@
|
|||||||
|
# OpenTelemetry Collector (otel/opentelemetry-collector-contrib:0.164.0) — единственный приёмник
|
||||||
|
# телеметрии приложения. Не хранит данные: принимает OTLP, ограничивает память, пакетирует и
|
||||||
|
# отправляет в три хранилища. Проверка конфигурации: otelcol-contrib validate --config=collector.yaml
|
||||||
|
#
|
||||||
|
# api/worker/dispatcher/janitor --OTLP/HTTP :4318--> receiver otlp
|
||||||
|
# traces -> memory_limiter -> attributes/scrub -> batch -> otlp_grpc/tempo (Tempo :4317)
|
||||||
|
# logs -> memory_limiter -> attributes/scrub -> batch -> otlp_http/loki (Loki /otlp)
|
||||||
|
# metrics -> memory_limiter -> batch -> otlp_http/prometheus (Prometheus /api/v1/otlp)
|
||||||
|
#
|
||||||
|
# Порты 4317/4318/8888/13133 слушаются только во внутренней сети telemetry (в compose не публикуются).
|
||||||
|
#
|
||||||
|
# Поведение при недоступном хранилище (ограниченная буферизация, RAM не растёт бесконечно):
|
||||||
|
# - у каждого exporter своя очередь в памяти (sending_queue) фиксированного размера: заполнилась ->
|
||||||
|
# новые пакеты отбрасываются и считаются (otelcol_exporter_enqueue_failed_*);
|
||||||
|
# - повтор с экспоненциальной задержкой не дольше max_elapsed_time, потом пакет отбрасывается
|
||||||
|
# (otelcol_exporter_send_failed_*);
|
||||||
|
# - memory_limiter: при приближении к лимиту receiver отказывает клиентам (SDK получает ошибку и
|
||||||
|
# сам отбрасывает данные в своей ограниченной очереди), процесс не уходит в OOM;
|
||||||
|
# - остальные pipeline работают независимо: упал Tempo — логи и метрики продолжают доходить.
|
||||||
|
# Очередь только в памяти: при рестарте Collector неотправленное теряется (осознанный компромисс,
|
||||||
|
# file_storage не нужен для учебного стенда).
|
||||||
|
|
||||||
|
extensions:
|
||||||
|
health_check:
|
||||||
|
endpoint: 0.0.0.0:13133
|
||||||
|
|
||||||
|
receivers:
|
||||||
|
otlp:
|
||||||
|
protocols:
|
||||||
|
grpc:
|
||||||
|
endpoint: 0.0.0.0:4317
|
||||||
|
http:
|
||||||
|
endpoint: 0.0.0.0:4318
|
||||||
|
|
||||||
|
processors:
|
||||||
|
# Первым в каждом pipeline: проверка heap Go раз в секунду. Мягкий предел = limit - spike = 160 MiB
|
||||||
|
# (receiver начинает отказывать), жёсткий 200 MiB (принудительный GC). Лимит контейнера 384 MiB:
|
||||||
|
# RSS включает ещё runtime Go и отображённый в память бинарник contrib (~150 MiB, вытесняемые страницы).
|
||||||
|
memory_limiter:
|
||||||
|
check_interval: 1s
|
||||||
|
limit_mib: 200
|
||||||
|
spike_limit_mib: 40
|
||||||
|
|
||||||
|
# Защита в глубину: приложение не пишет секреты в атрибуты, но если бы инструментирование ошиблось,
|
||||||
|
# такие ключи не дойдут до хранилищ. Удаляются по шаблону имени ключа (спаны и логи).
|
||||||
|
attributes/scrub:
|
||||||
|
actions:
|
||||||
|
- pattern: (?i).*(password|passwd|authorization|cookie|secret|token|api[_.-]?key|signature).*
|
||||||
|
action: delete
|
||||||
|
- key: db.statement.parameters
|
||||||
|
action: delete
|
||||||
|
- key: db.query.parameter
|
||||||
|
action: delete
|
||||||
|
|
||||||
|
batch:
|
||||||
|
timeout: 2s
|
||||||
|
send_batch_size: 1024
|
||||||
|
send_batch_max_size: 2048
|
||||||
|
|
||||||
|
exporters:
|
||||||
|
otlp_grpc/tempo:
|
||||||
|
endpoint: tempo:4317
|
||||||
|
tls:
|
||||||
|
insecure: true # lab6: внутренняя сеть; TLS/mTLS по trust matrix — в lab7
|
||||||
|
timeout: 5s
|
||||||
|
sending_queue:
|
||||||
|
enabled: true
|
||||||
|
num_consumers: 2
|
||||||
|
queue_size: 200 # пакетов (до 2048 спанов каждый)
|
||||||
|
retry_on_failure:
|
||||||
|
enabled: true
|
||||||
|
initial_interval: 1s
|
||||||
|
max_interval: 10s
|
||||||
|
max_elapsed_time: 120s
|
||||||
|
|
||||||
|
otlp_http/loki:
|
||||||
|
endpoint: http://loki:3100/otlp # -> /otlp/v1/logs
|
||||||
|
timeout: 5s
|
||||||
|
sending_queue:
|
||||||
|
enabled: true
|
||||||
|
num_consumers: 2
|
||||||
|
queue_size: 200
|
||||||
|
retry_on_failure:
|
||||||
|
enabled: true
|
||||||
|
initial_interval: 1s
|
||||||
|
max_interval: 10s
|
||||||
|
max_elapsed_time: 120s
|
||||||
|
|
||||||
|
otlp_http/prometheus:
|
||||||
|
endpoint: http://prometheus:9090/api/v1/otlp # -> /api/v1/otlp/v1/metrics
|
||||||
|
timeout: 5s
|
||||||
|
sending_queue:
|
||||||
|
enabled: true
|
||||||
|
num_consumers: 1 # один поток: точки одной серии приходят по порядку
|
||||||
|
queue_size: 100
|
||||||
|
retry_on_failure:
|
||||||
|
enabled: true
|
||||||
|
initial_interval: 1s
|
||||||
|
max_interval: 10s
|
||||||
|
max_elapsed_time: 120s
|
||||||
|
|
||||||
|
service:
|
||||||
|
extensions: [health_check]
|
||||||
|
telemetry:
|
||||||
|
logs:
|
||||||
|
level: info
|
||||||
|
# Собственные метрики Collector (принято/отправлено/отброшено, размер очередей, RSS) —
|
||||||
|
# Prometheus забирает их pull-ом с otel-collector:8888. Так потери телеметрии видны и считаются.
|
||||||
|
metrics:
|
||||||
|
level: detailed
|
||||||
|
readers:
|
||||||
|
- pull:
|
||||||
|
exporter:
|
||||||
|
prometheus:
|
||||||
|
host: 0.0.0.0
|
||||||
|
port: 8888
|
||||||
|
pipelines:
|
||||||
|
traces:
|
||||||
|
receivers: [otlp]
|
||||||
|
processors: [memory_limiter, attributes/scrub, batch]
|
||||||
|
exporters: [otlp_grpc/tempo]
|
||||||
|
logs:
|
||||||
|
receivers: [otlp]
|
||||||
|
processors: [memory_limiter, attributes/scrub, batch]
|
||||||
|
exporters: [otlp_http/loki]
|
||||||
|
metrics:
|
||||||
|
receivers: [otlp]
|
||||||
|
processors: [memory_limiter, batch]
|
||||||
|
exporters: [otlp_http/prometheus]
|
||||||
@@ -0,0 +1,42 @@
|
|||||||
|
# Prometheus 3.15.1 — хранилище МЕТРИК. Принимает метрики приложения push-ом по OTLP от Collector
|
||||||
|
# (флаг --web.enable-otlp-receiver, путь /api/v1/otlp/v1/metrics) и сам забирает (pull) служебные
|
||||||
|
# метрики Collector, Loki и Tempo. Вычисляет recording rules (SLI) и alert rules; состояние алертов
|
||||||
|
# (pending/firing/resolved) видно на /alerts и через /api/v1/alerts — внешняя доставка не нужна.
|
||||||
|
# Проверка: promtool check config prometheus.yml
|
||||||
|
global:
|
||||||
|
scrape_interval: 15s
|
||||||
|
scrape_timeout: 5s
|
||||||
|
evaluation_interval: 15s
|
||||||
|
|
||||||
|
# Как имена OTLP переводятся в имена Prometheus (фиксируем явно, чтобы dashboard не сломался при
|
||||||
|
# смене значения по умолчанию): точки -> подчёркивания, суффиксы единиц и _total.
|
||||||
|
# http.server.request.duration (s, histogram) -> http_server_request_duration_seconds_{bucket,sum,count}
|
||||||
|
# атрибут http.route -> метка http_route; service.name -> job; service.instance.id -> instance.
|
||||||
|
otlp:
|
||||||
|
translation_strategy: UnderscoreEscapingWithSuffixes
|
||||||
|
|
||||||
|
storage:
|
||||||
|
tsdb:
|
||||||
|
# Collector после недоступности Prometheus дошлёт пакеты из очереди с прошлыми временными метками.
|
||||||
|
out_of_order_time_window: 10m
|
||||||
|
|
||||||
|
rule_files:
|
||||||
|
- /etc/prometheus/rules/*.yml
|
||||||
|
|
||||||
|
scrape_configs:
|
||||||
|
- job_name: prometheus
|
||||||
|
static_configs:
|
||||||
|
- targets: ["localhost:9090"]
|
||||||
|
|
||||||
|
# Собственные метрики Collector: принято/отправлено/отброшено по сигналам, очереди, RSS.
|
||||||
|
- job_name: otel-collector
|
||||||
|
static_configs:
|
||||||
|
- targets: ["otel-collector:8888"]
|
||||||
|
|
||||||
|
- job_name: loki
|
||||||
|
static_configs:
|
||||||
|
- targets: ["loki:3100"]
|
||||||
|
|
||||||
|
- job_name: tempo
|
||||||
|
static_configs:
|
||||||
|
- targets: ["tempo:3200"]
|
||||||
@@ -0,0 +1,113 @@
|
|||||||
|
# Alert rules (Prometheus). Внешней доставки нет: состояние inactive/pending/firing видно на
|
||||||
|
# http://127.0.0.1:9090/alerts, через GET /api/v1/alerts и в Grafana (Alerting -> data source rules);
|
||||||
|
# метрика ALERTS{alertstate="firing"} хранится в TSDB, поэтому видно и когда алерт стал resolved.
|
||||||
|
# Каждое правило: условие, окно rate, порог, for, описание и ссылка на локальный runbook.
|
||||||
|
# Проверка: promtool check rules alerts.yml; promtool test rules ../tests/alerts_test.yml
|
||||||
|
groups:
|
||||||
|
- name: messenger-alerts
|
||||||
|
interval: 15s
|
||||||
|
rules:
|
||||||
|
# (1) Высокая доля ошибок API. Окно 1m, порог 5% (в 5 раз выше бюджета SLO-1 = 1%), for 1m,
|
||||||
|
# и не меньше 0,2 запроса/с, чтобы единичный 503 при пустом трафике не будил дежурного.
|
||||||
|
- alert: MessengerHighErrorRate
|
||||||
|
expr: |
|
||||||
|
(
|
||||||
|
(sum(rate(http_server_request_duration_seconds_count{job="messenger-api", http_route!~"/health/.*", http_response_status_class="5xx"}[1m])) or on() vector(0))
|
||||||
|
/ sum(rate(http_server_request_duration_seconds_count{job="messenger-api", http_route!~"/health/.*"}[1m]))
|
||||||
|
) > 0.05
|
||||||
|
and on()
|
||||||
|
sum(rate(http_server_request_duration_seconds_count{job="messenger-api", http_route!~"/health/.*"}[1m])) > 0.2
|
||||||
|
for: 1m
|
||||||
|
labels:
|
||||||
|
severity: page
|
||||||
|
service: messenger-api
|
||||||
|
slo: availability
|
||||||
|
annotations:
|
||||||
|
summary: "API: {{ $value | humanizePercentage }} бизнес-запросов завершаются 5xx (порог 5%)"
|
||||||
|
description: >-
|
||||||
|
Доля ответов 5xx среди бизнес-запросов API (без /health) за последнюю минуту выше 5% дольше
|
||||||
|
минуты. Обычно это недоступность или медленная работа PostgreSQL/Redis/S3 (503) либо ошибка
|
||||||
|
кода (500). Сравните p95 HTTP и p95 ожидания соединения из пула, найдите error-трассу.
|
||||||
|
runbook_url: docs/runbooks.md#messengerhigherrorrate
|
||||||
|
|
||||||
|
# (2) Растущий lag очереди: возраст старейшего ГОТОВОГО к обработке задания (available_at прошёл,
|
||||||
|
# статус queued) больше 30 с дольше минуты. Пока задания не забирают, возраст растёт линейно,
|
||||||
|
# так что правило ловит именно рост, а не единичный всплеск глубины очереди.
|
||||||
|
- alert: MessengerQueueLagGrowing
|
||||||
|
expr: max(messenger_queue_oldest_age_seconds{messaging_destination_name="attachments.process.v1"}) > 30
|
||||||
|
for: 1m
|
||||||
|
labels:
|
||||||
|
severity: page
|
||||||
|
service: messenger-worker
|
||||||
|
slo: time_to_ready
|
||||||
|
annotations:
|
||||||
|
summary: "Очередь attachments.process.v1: старейшее задание ждёт {{ $value | humanizeDuration }}"
|
||||||
|
description: >-
|
||||||
|
Задания обработки вложений не забираются: возраст старейшего ожидающего задания выше 30 с
|
||||||
|
дольше минуты (SLO-3: 95% вложений ready за 30 с). Проверьте число consumers очереди,
|
||||||
|
занятость worker (busy/capacity) и ошибки worker в логах.
|
||||||
|
runbook_url: docs/runbooks.md#messengerqueuelaggrowing
|
||||||
|
|
||||||
|
# (3) Ни одного consumer у очереди: worker остановлен/упал. Дополняет (2): срабатывает даже без нагрузки.
|
||||||
|
- alert: MessengerWorkerNoConsumers
|
||||||
|
expr: max(messenger_queue_consumers{messaging_destination_name="attachments.process.v1"}) == 0
|
||||||
|
for: 1m
|
||||||
|
labels:
|
||||||
|
severity: warning
|
||||||
|
service: messenger-worker
|
||||||
|
annotations:
|
||||||
|
summary: "У очереди attachments.process.v1 нет consumers"
|
||||||
|
description: "RabbitMQ не видит ни одного подписчика основной очереди дольше минуты: worker не запущен или не может подключиться к брокеру."
|
||||||
|
runbook_url: docs/runbooks.md#messengerworkernoconsumers
|
||||||
|
|
||||||
|
# (4) Насыщение пула соединений API: запросы ждут соединение дольше минуты.
|
||||||
|
- alert: MessengerDbPoolSaturated
|
||||||
|
expr: max by (instance) (db_client_connection_pending_requests{job="messenger-api"}) > 0
|
||||||
|
for: 1m
|
||||||
|
labels:
|
||||||
|
severity: warning
|
||||||
|
service: messenger-api
|
||||||
|
annotations:
|
||||||
|
summary: "Пул PostgreSQL реплики {{ $labels.instance }}: {{ $value }} запросов ждут соединение"
|
||||||
|
description: "Все DB_POOL_MAX соединений заняты. Причина обычно — медленные запросы или блокировки в PostgreSQL, а не нехватка соединений."
|
||||||
|
runbook_url: docs/runbooks.md#messengerdbpoolsaturated
|
||||||
|
|
||||||
|
# (5) Collector не может доставить телеметрию в Tempo/Loki/Prometheus: либо пакеты уже теряются
|
||||||
|
# (send_failed — истёк повтор 120 с у пакета, enqueue_failed — очередь полна), либо очередь exporter
|
||||||
|
# дольше минуты заполнена больше чем на 5% (хранилище не принимает, пакеты копятся). Во время пакет
|
||||||
|
# ждёт в очереди его окно повторов не идёт, поэтому без второго условия алерт сработал бы только
|
||||||
|
# после первых потерь, то есть через 2+ минуты недоступности.
|
||||||
|
- alert: TelemetryExportFailing
|
||||||
|
expr: |
|
||||||
|
sum by (exporter) (
|
||||||
|
rate(otelcol_exporter_send_failed_spans[2m])
|
||||||
|
or rate(otelcol_exporter_send_failed_log_records[2m])
|
||||||
|
or rate(otelcol_exporter_send_failed_metric_points[2m])
|
||||||
|
or rate(otelcol_exporter_enqueue_failed_spans[2m])
|
||||||
|
or rate(otelcol_exporter_enqueue_failed_log_records[2m])
|
||||||
|
or rate(otelcol_exporter_enqueue_failed_metric_points[2m])
|
||||||
|
) > 0
|
||||||
|
or
|
||||||
|
max by (exporter) (otelcol_exporter_queue_size / otelcol_exporter_queue_capacity) > 0.05
|
||||||
|
for: 1m
|
||||||
|
labels:
|
||||||
|
severity: warning
|
||||||
|
service: otel-collector
|
||||||
|
annotations:
|
||||||
|
summary: "Collector не доставляет телеметрию: exporter {{ $labels.exporter }}"
|
||||||
|
description: "Хранилище не принимает данные: очередь exporter растёт или пакеты уже отброшены (окно повторов 120 с истекло / очередь полна). Работа API не затронута; потери считаются в otelcol_exporter_send_failed_* и otelcol_exporter_enqueue_failed_*."
|
||||||
|
runbook_url: docs/runbooks.md#telemetryexportfailing
|
||||||
|
|
||||||
|
# (6) Collector недоступен: Prometheus не может забрать его служебные метрики. Метрики приложения
|
||||||
|
# в этот период не доходят (их везёт Collector), поэтому алерт смотрит на up самого Collector.
|
||||||
|
- alert: TelemetryCollectorDown
|
||||||
|
expr: up{job="otel-collector"} == 0
|
||||||
|
for: 1m
|
||||||
|
labels:
|
||||||
|
severity: warning
|
||||||
|
service: otel-collector
|
||||||
|
annotations:
|
||||||
|
summary: "OTel Collector недоступен"
|
||||||
|
description: "Приложение продолжает работать, SDK отбрасывает телеметрию сверх своей очереди (2048 спанов/логов на процесс). Метрики приложения за этот период будут с пропуском."
|
||||||
|
runbook_url: docs/runbooks.md#telemetrycollectordown
|
||||||
|
|
||||||
@@ -0,0 +1,50 @@
|
|||||||
|
# Recording rules: SLI и агрегаты для dashboard. Проверка: promtool check rules recording.yml
|
||||||
|
#
|
||||||
|
# Числитель/знаменатель ошибок API (contracts/observability.md):
|
||||||
|
# знаменатель — все бизнес-запросы API (job="messenger-api", маршрут НЕ /health/*), любой код;
|
||||||
|
# числитель — из них ответы 5xx (отказ сервера: БД/Redis/S3 недоступны, внутренняя ошибка).
|
||||||
|
# 4xx (неверный пароль, 404 чужого чата, 409) — поведение клиента, в ошибки SLO не входят.
|
||||||
|
# Percentile — только из buckets гистограммы, просуммированных по всем репликам ДО histogram_quantile
|
||||||
|
# (sum by (le)); p95 отдельных реплик не усредняются.
|
||||||
|
groups:
|
||||||
|
- name: messenger-api-sli
|
||||||
|
interval: 15s
|
||||||
|
rules:
|
||||||
|
- record: messenger:http_requests:rate1m
|
||||||
|
expr: sum(rate(http_server_request_duration_seconds_count{job="messenger-api", http_route!~"/health/.*"}[1m]))
|
||||||
|
- record: messenger:http_errors_5xx:rate1m
|
||||||
|
expr: |
|
||||||
|
sum(rate(http_server_request_duration_seconds_count{job="messenger-api", http_route!~"/health/.*", http_response_status_class="5xx"}[1m]))
|
||||||
|
or on() vector(0)
|
||||||
|
- record: messenger:http_error_ratio:rate1m
|
||||||
|
expr: messenger:http_errors_5xx:rate1m / messenger:http_requests:rate1m
|
||||||
|
- record: messenger:http_request_duration_seconds:p50_5m
|
||||||
|
expr: histogram_quantile(0.50, sum by (le) (rate(http_server_request_duration_seconds_bucket{job="messenger-api", http_route!~"/health/.*"}[5m])))
|
||||||
|
- record: messenger:http_request_duration_seconds:p95_5m
|
||||||
|
expr: histogram_quantile(0.95, sum by (le) (rate(http_server_request_duration_seconds_bucket{job="messenger-api", http_route!~"/health/.*"}[5m])))
|
||||||
|
- record: messenger:http_request_duration_seconds:p99_5m
|
||||||
|
expr: histogram_quantile(0.99, sum by (le) (rate(http_server_request_duration_seconds_bucket{job="messenger-api", http_route!~"/health/.*"}[5m])))
|
||||||
|
|
||||||
|
# SLO (учебное скользящее окно 1 ч; в эксплуатации то же выражение с [28d]):
|
||||||
|
# SLO-1 доступность: >= 99% бизнес-запросов без 5xx;
|
||||||
|
# SLO-2 задержка: >= 95% POST /api/v1/chats/{chat_id}/messages быстрее 250 мс;
|
||||||
|
# SLO-3 фоновая обработка: >= 95% вложений ready не позже 30 с после приёма (202).
|
||||||
|
- name: messenger-slo
|
||||||
|
interval: 30s
|
||||||
|
rules:
|
||||||
|
- record: messenger:slo_availability:ratio_1h
|
||||||
|
expr: |
|
||||||
|
1 - (
|
||||||
|
(sum(increase(http_server_request_duration_seconds_count{job="messenger-api", http_route!~"/health/.*", http_response_status_class="5xx"}[1h])) or on() vector(0))
|
||||||
|
/ sum(increase(http_server_request_duration_seconds_count{job="messenger-api", http_route!~"/health/.*"}[1h]))
|
||||||
|
)
|
||||||
|
- record: messenger:slo_availability:error_budget_remaining_1h
|
||||||
|
expr: 1 - (1 - messenger:slo_availability:ratio_1h) / (1 - 0.99)
|
||||||
|
- record: messenger:slo_latency_messages:ratio_1h
|
||||||
|
expr: |
|
||||||
|
sum(increase(http_server_request_duration_seconds_bucket{job="messenger-api", http_request_method="POST", http_route="/api/v1/chats/{chat_id}/messages", http_response_status_class="2xx", le="0.25"}[1h]))
|
||||||
|
/ sum(increase(http_server_request_duration_seconds_count{job="messenger-api", http_request_method="POST", http_route="/api/v1/chats/{chat_id}/messages", http_response_status_class="2xx"}[1h]))
|
||||||
|
- record: messenger:slo_time_to_ready:ratio_1h
|
||||||
|
expr: |
|
||||||
|
sum(increase(messenger_attachment_time_to_ready_seconds_bucket{le="30"}[1h]))
|
||||||
|
/ sum(increase(messenger_attachment_time_to_ready_seconds_count[1h]))
|
||||||
@@ -0,0 +1,121 @@
|
|||||||
|
# Unit-тесты alert rules на синтетических рядах (без стенда):
|
||||||
|
# promtool test rules infra/prometheus/tests/alerts_test.yml
|
||||||
|
# docker compose exec prometheus promtool test rules /etc/prometheus/tests/alerts_test.yml
|
||||||
|
# Проверяют порог, окно и for: правило не срабатывает раньше времени и при пустом трафике.
|
||||||
|
rule_files:
|
||||||
|
- ../rules/alerts.yml
|
||||||
|
evaluation_interval: 15s
|
||||||
|
|
||||||
|
tests:
|
||||||
|
# MessengerHighErrorRate: 2 rps успешных, с 3-й минуты +1 rps 5xx (доля 33%).
|
||||||
|
- interval: 15s
|
||||||
|
input_series:
|
||||||
|
- series: 'http_server_request_duration_seconds_count{job="messenger-api", http_route="/api/v1/chats/{chat_id}/messages", http_response_status_class="2xx"}'
|
||||||
|
values: '0+30x60'
|
||||||
|
- series: 'http_server_request_duration_seconds_count{job="messenger-api", http_route="/api/v1/chats/{chat_id}/messages", http_response_status_class="5xx"}'
|
||||||
|
values: '0x12 0+15x48'
|
||||||
|
# health-проверки с ошибками не считаются (знаменатель и числитель без /health)
|
||||||
|
- series: 'http_server_request_duration_seconds_count{job="messenger-api", http_route="/health/ready", http_response_status_class="5xx"}'
|
||||||
|
values: '0+100x60'
|
||||||
|
alert_rule_test:
|
||||||
|
- eval_time: 3m
|
||||||
|
alertname: MessengerHighErrorRate
|
||||||
|
exp_alerts: []
|
||||||
|
- eval_time: 3m45s # условие выполняется, но for: 1m ещё не прошёл
|
||||||
|
alertname: MessengerHighErrorRate
|
||||||
|
exp_alerts: []
|
||||||
|
- eval_time: 5m
|
||||||
|
alertname: MessengerHighErrorRate
|
||||||
|
exp_alerts:
|
||||||
|
- exp_labels: {severity: page, service: messenger-api, slo: availability}
|
||||||
|
exp_annotations:
|
||||||
|
summary: 'API: 33.33% бизнес-запросов завершаются 5xx (порог 5%)'
|
||||||
|
description: Доля ответов 5xx среди бизнес-запросов API (без /health) за последнюю минуту выше 5% дольше минуты. Обычно это недоступность или медленная работа PostgreSQL/Redis/S3 (503) либо ошибка кода (500). Сравните p95 HTTP и p95 ожидания соединения из пула, найдите error-трассу.
|
||||||
|
runbook_url: docs/runbooks.md#messengerhigherrorrate
|
||||||
|
|
||||||
|
# Пустой трафик: единичные 5xx при 0,05 rps не будят дежурного (порог 0,2 запроса/с).
|
||||||
|
- interval: 15s
|
||||||
|
input_series:
|
||||||
|
- series: 'http_server_request_duration_seconds_count{job="messenger-api", http_route="/api/v1/users/me", http_response_status_class="5xx"}'
|
||||||
|
values: '0+1x40'
|
||||||
|
alert_rule_test:
|
||||||
|
- eval_time: 9m
|
||||||
|
alertname: MessengerHighErrorRate
|
||||||
|
exp_alerts: []
|
||||||
|
|
||||||
|
# MessengerQueueLagGrowing: возраст старейшего задания растёт со 2-й минуты, затем worker вернулся.
|
||||||
|
- interval: 15s
|
||||||
|
input_series:
|
||||||
|
- series: 'messenger_queue_oldest_age_seconds{job="messenger-dispatcher", messaging_destination_name="attachments.process.v1"}'
|
||||||
|
values: '0x8 0+15x20 0x20'
|
||||||
|
- series: 'messenger_queue_consumers{job="messenger-dispatcher", messaging_destination_name="attachments.process.v1"}'
|
||||||
|
values: '2x8 0x20 2x20'
|
||||||
|
alert_rule_test:
|
||||||
|
- eval_time: 2m30s # возраст 30 с — ещё не больше порога
|
||||||
|
alertname: MessengerQueueLagGrowing
|
||||||
|
exp_alerts: []
|
||||||
|
- eval_time: 4m
|
||||||
|
alertname: MessengerQueueLagGrowing
|
||||||
|
exp_alerts:
|
||||||
|
- exp_labels: {severity: page, service: messenger-worker, slo: time_to_ready}
|
||||||
|
exp_annotations:
|
||||||
|
summary: 'Очередь attachments.process.v1: старейшее задание ждёт 1m 45s'
|
||||||
|
description: 'Задания обработки вложений не забираются: возраст старейшего ожидающего задания выше 30 с дольше минуты (SLO-3: 95% вложений ready за 30 с). Проверьте число consumers очереди, занятость worker (busy/capacity) и ошибки worker в логах.'
|
||||||
|
runbook_url: docs/runbooks.md#messengerqueuelaggrowing
|
||||||
|
- eval_time: 4m
|
||||||
|
alertname: MessengerWorkerNoConsumers
|
||||||
|
exp_alerts:
|
||||||
|
- exp_labels: {severity: warning, service: messenger-worker}
|
||||||
|
exp_annotations:
|
||||||
|
summary: У очереди attachments.process.v1 нет consumers
|
||||||
|
description: 'RabbitMQ не видит ни одного подписчика основной очереди дольше минуты: worker не запущен или не может подключиться к брокеру.'
|
||||||
|
runbook_url: docs/runbooks.md#messengerworkernoconsumers
|
||||||
|
- eval_time: 9m # worker вернулся, очередь разобрана -> resolved
|
||||||
|
alertname: MessengerQueueLagGrowing
|
||||||
|
exp_alerts: []
|
||||||
|
|
||||||
|
# TelemetryExportFailing и TelemetryCollectorDown
|
||||||
|
- interval: 15s
|
||||||
|
input_series:
|
||||||
|
- series: 'otelcol_exporter_send_failed_spans{exporter="otlp_grpc/tempo"}'
|
||||||
|
values: '0x8 0+50x20'
|
||||||
|
- series: 'up{job="otel-collector"}'
|
||||||
|
values: '1x8 0x20'
|
||||||
|
alert_rule_test:
|
||||||
|
- eval_time: 5m
|
||||||
|
alertname: TelemetryExportFailing
|
||||||
|
exp_alerts:
|
||||||
|
- exp_labels: {severity: warning, service: otel-collector, exporter: otlp_grpc/tempo}
|
||||||
|
exp_annotations:
|
||||||
|
summary: 'Collector не доставляет телеметрию: exporter otlp_grpc/tempo'
|
||||||
|
description: 'Хранилище не принимает данные: очередь exporter растёт или пакеты уже отброшены (окно повторов 120 с истекло / очередь полна). Работа API не затронута; потери считаются в otelcol_exporter_send_failed_* и otelcol_exporter_enqueue_failed_*.'
|
||||||
|
runbook_url: docs/runbooks.md#telemetryexportfailing
|
||||||
|
- eval_time: 5m
|
||||||
|
alertname: TelemetryCollectorDown
|
||||||
|
exp_alerts:
|
||||||
|
- exp_labels: {severity: warning, service: otel-collector, job: otel-collector}
|
||||||
|
exp_annotations:
|
||||||
|
summary: OTel Collector недоступен
|
||||||
|
description: Приложение продолжает работать, SDK отбрасывает телеметрию сверх своей очереди (2048 спанов/логов на процесс). Метрики приложения за этот период будут с пропуском.
|
||||||
|
runbook_url: docs/runbooks.md#telemetrycollectordown
|
||||||
|
|
||||||
|
# TelemetryExportFailing по заполнению очереди: Loki недоступен, потерь ещё нет, но очередь растёт.
|
||||||
|
- interval: 15s
|
||||||
|
input_series:
|
||||||
|
- series: 'otelcol_exporter_queue_size{exporter="otlp_http/loki"}'
|
||||||
|
values: '0x8 0+2x30'
|
||||||
|
- series: 'otelcol_exporter_queue_capacity{exporter="otlp_http/loki"}'
|
||||||
|
values: '200x38'
|
||||||
|
alert_rule_test:
|
||||||
|
- eval_time: 3m # 6 пакетов из 200 (3%) — ниже порога 5%
|
||||||
|
alertname: TelemetryExportFailing
|
||||||
|
exp_alerts: []
|
||||||
|
- eval_time: 5m
|
||||||
|
alertname: TelemetryExportFailing
|
||||||
|
exp_alerts:
|
||||||
|
- exp_labels: {severity: warning, service: otel-collector, exporter: otlp_http/loki}
|
||||||
|
exp_annotations:
|
||||||
|
summary: 'Collector не доставляет телеметрию: exporter otlp_http/loki'
|
||||||
|
description: 'Хранилище не принимает данные: очередь exporter растёт или пакеты уже отброшены (окно повторов 120 с истекло / очередь полна). Работа API не затронута; потери считаются в otelcol_exporter_send_failed_* и otelcol_exporter_enqueue_failed_*.'
|
||||||
|
runbook_url: docs/runbooks.md#telemetryexportfailing
|
||||||
|
|
||||||
@@ -0,0 +1,22 @@
|
|||||||
|
# RabbitMQ 4.1 (сервис rabbitmq). Топология, пользователи и права — в secrets/rabbitmq_definitions.json
|
||||||
|
# (генерирует scripts/init_secrets.py), файл импортируется при каждом старте узла. Пользователя guest нет:
|
||||||
|
# при импорте определений на старте пользователь по умолчанию не создаётся.
|
||||||
|
definitions.import_backend = local_filesystem
|
||||||
|
definitions.local.path = /run/secrets/rabbitmq_definitions
|
||||||
|
definitions.skip_if_unchanged = true
|
||||||
|
|
||||||
|
listeners.tcp.default = 5672
|
||||||
|
loopback_users = none
|
||||||
|
|
||||||
|
# Ресурсы: при нехватке памяти/диска брокер блокирует издателей (dispatcher ждёт и повторяет),
|
||||||
|
# сообщения не теряются. Абсолютное значение: ниже лимита контейнера 512M (доля считалась бы от RAM хоста).
|
||||||
|
vm_memory_high_watermark.absolute = 320MB
|
||||||
|
disk_free_limit.absolute = 256MB
|
||||||
|
|
||||||
|
# Consumer, не приславший ack за это время, считается зависшим: канал закрывается, сообщение
|
||||||
|
# возвращается в очередь. Должно быть больше JOB_TIMEOUT_SECONDS + времени записи в S3.
|
||||||
|
consumer_timeout = 300000
|
||||||
|
|
||||||
|
log.console = true
|
||||||
|
log.console.level = info
|
||||||
|
log.file = false
|
||||||
@@ -0,0 +1,13 @@
|
|||||||
|
bind 0.0.0.0
|
||||||
|
port 6379
|
||||||
|
protected-mode yes
|
||||||
|
dir /data
|
||||||
|
|
||||||
|
appendonly yes
|
||||||
|
appendfsync always
|
||||||
|
save ""
|
||||||
|
|
||||||
|
maxmemory 64mb
|
||||||
|
maxmemory-policy noeviction
|
||||||
|
|
||||||
|
include /run/secrets/redis_auth
|
||||||
@@ -0,0 +1,39 @@
|
|||||||
|
# Tempo 3.1.1 — хранилище ТРАСС (single binary: distributor -> live-store -> блоки в volume tempodata,
|
||||||
|
# без Kafka). Трассы приходят от Collector по OTLP gRPC :4317 (только внутренняя сеть telemetry),
|
||||||
|
# запросы Grafana и тестов — HTTP :3200 (/api/traces/<trace_id>, /api/search, TraceQL).
|
||||||
|
# Проверка: tempo -config.file=tempo.yaml -config.verify=true
|
||||||
|
stream_over_http_enabled: true
|
||||||
|
|
||||||
|
server:
|
||||||
|
http_listen_port: 3200
|
||||||
|
grpc_listen_port: 9095
|
||||||
|
log_level: warn
|
||||||
|
|
||||||
|
distributor:
|
||||||
|
receivers:
|
||||||
|
otlp:
|
||||||
|
protocols:
|
||||||
|
grpc:
|
||||||
|
endpoint: 0.0.0.0:4317
|
||||||
|
|
||||||
|
storage:
|
||||||
|
trace:
|
||||||
|
backend: local
|
||||||
|
wal:
|
||||||
|
path: /var/tempo/wal
|
||||||
|
local:
|
||||||
|
path: /var/tempo/traces
|
||||||
|
|
||||||
|
# Хранение 7 дней (как у Loki); удаление старых блоков делают backend-scheduler/worker.
|
||||||
|
backend_scheduler:
|
||||||
|
local_work_path: /var/tempo/scheduler
|
||||||
|
provider:
|
||||||
|
compaction:
|
||||||
|
compaction:
|
||||||
|
block_retention: 168h
|
||||||
|
backend_worker:
|
||||||
|
compaction:
|
||||||
|
block_retention: 168h
|
||||||
|
|
||||||
|
usage_report:
|
||||||
|
reporting_enabled: false
|
||||||
+322
@@ -0,0 +1,322 @@
|
|||||||
|
"""Очистка и сверка PostgreSQL <-> S3 (сервис janitor).
|
||||||
|
|
||||||
|
python janitor.py --once [--dry-run] [--grace-seconds N]
|
||||||
|
|
||||||
|
Проход (раз в JANITOR_INTERVAL_SECONDS, один janitor за счёт pg_try_advisory_lock):
|
||||||
|
|
||||||
|
1. Брошенные намерения (как в lab4): state=uploading старше grace — исходник так и не был поставлен
|
||||||
|
в очередь. Удалить объект staging (идемпотентно), затем строку.
|
||||||
|
2. Staging после безопасной фиксации:
|
||||||
|
- ready: исходник уже опубликован в attachments/<id>/v<N>/original -> удалить staging-объект
|
||||||
|
(обычно это делает worker сразу после ack; здесь — если он не успел);
|
||||||
|
- failed: исходник хранится STAGING_RETENTION_SECONDS после ошибки (для replay), потом удаляется.
|
||||||
|
queued/processing не трогаются никогда: это текущая работа.
|
||||||
|
3. Outbox: опубликованные строки старше OUTBOX_RETENTION_SECONDS удаляются.
|
||||||
|
4. Обход bucket по всем prefix (объекты старше grace):
|
||||||
|
- staging/, originals/: сирота, если ни одна строка не ссылается на ключ как source_key;
|
||||||
|
- attachments/<id>/...: сирота, только если вложение в КОНЕЧНОМ состоянии и ключ ему не нужен:
|
||||||
|
вложения нет; оно failed и replay уже невозможен (source_key IS NULL — исходник удалён по retention);
|
||||||
|
оно ready, но ключ не входит в опубликованный набор (оригинал + варианты). У queued/processing
|
||||||
|
и у failed с исходником в staging объекты не трогаются: replay/следующая попытка перезапишет их
|
||||||
|
теми же ключами, и удаление устаревшим проходом сломало бы новый результат.
|
||||||
|
Повторная проверка и DELETE выполняются в одной транзакции под FOR SHARE на строке вложения:
|
||||||
|
replay (FOR UPDATE) и захват worker'ом (UPDATE) ждут конца DELETE, а не идут параллельно с ним.
|
||||||
|
Отчёт о ready-вложениях без объекта в bucket (missing) — только отчёт, это потеря данных.
|
||||||
|
|
||||||
|
Grace (ATTACHMENT_GRACE_SECONDS, по умолчанию 1 ч) больше любой живой загрузки/попытки, поэтому
|
||||||
|
janitor не удаляет то, что ещё пишется.
|
||||||
|
"""
|
||||||
|
import argparse
|
||||||
|
import json
|
||||||
|
import logging
|
||||||
|
import re
|
||||||
|
import signal
|
||||||
|
import threading
|
||||||
|
import time
|
||||||
|
from datetime import datetime, timedelta, timezone
|
||||||
|
from pathlib import Path
|
||||||
|
from uuid import UUID
|
||||||
|
|
||||||
|
import psycopg
|
||||||
|
from psycopg.rows import dict_row
|
||||||
|
|
||||||
|
import settings
|
||||||
|
import telemetry
|
||||||
|
|
||||||
|
telemetry.setup("messenger-janitor")
|
||||||
|
|
||||||
|
import pipeline # noqa: E402
|
||||||
|
import storage # noqa: E402
|
||||||
|
|
||||||
|
log = logging.getLogger("messenger.janitor")
|
||||||
|
|
||||||
|
JANITOR_LOCK = 20260404
|
||||||
|
BATCH = 100
|
||||||
|
STOP = threading.Event()
|
||||||
|
ACTOR = "janitor"
|
||||||
|
FINAL_KEY_RE = re.compile(r"^attachments/([0-9a-f-]{36})/")
|
||||||
|
|
||||||
|
|
||||||
|
def touch_alive() -> None:
|
||||||
|
try:
|
||||||
|
Path(settings.ALIVE_FILE).touch()
|
||||||
|
except OSError:
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
def idle(seconds: float) -> bool:
|
||||||
|
"""Пауза между проходами с обновлением файла живости (healthcheck). True — пришёл сигнал остановки."""
|
||||||
|
deadline = time.monotonic() + seconds
|
||||||
|
while (left := deadline - time.monotonic()) > 0:
|
||||||
|
touch_alive()
|
||||||
|
if STOP.wait(min(left, 10)):
|
||||||
|
return True
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
|
def connect():
|
||||||
|
return psycopg.connect(settings.DATABASE_URL, autocommit=True, row_factory=dict_row,
|
||||||
|
connect_timeout=settings.DB_CONNECT_TIMEOUT_SECONDS,
|
||||||
|
options=f"-c statement_timeout={settings.DB_STATEMENT_TIMEOUT_MS} -c TimeZone=UTC")
|
||||||
|
|
||||||
|
|
||||||
|
def reclaim_stale_intents(conn, grace: int, dry_run: bool) -> tuple[int, int]:
|
||||||
|
"""Шаг 1. Возвращает (найдено, удалено)."""
|
||||||
|
found = deleted = 0
|
||||||
|
while not STOP.is_set():
|
||||||
|
with conn.transaction():
|
||||||
|
rows = conn.execute(
|
||||||
|
"""SELECT id, source_key FROM attachments
|
||||||
|
WHERE state = 'uploading' AND created_at < now() - make_interval(secs => %s)
|
||||||
|
ORDER BY created_at LIMIT %s
|
||||||
|
FOR UPDATE SKIP LOCKED""",
|
||||||
|
(grace, BATCH),
|
||||||
|
).fetchall()
|
||||||
|
found += len(rows)
|
||||||
|
if dry_run or not rows:
|
||||||
|
return found, deleted
|
||||||
|
for row in rows:
|
||||||
|
storage.delete(row["source_key"]) # недоступен S3 -> исключение, транзакция откатится
|
||||||
|
conn.execute("DELETE FROM attachments WHERE id = %s AND state = 'uploading'", (row["id"],))
|
||||||
|
deleted += 1
|
||||||
|
log.info("reclaimed abandoned upload %s (object and row deleted)", row["id"])
|
||||||
|
if len(rows) < BATCH:
|
||||||
|
return found, deleted
|
||||||
|
return found, deleted
|
||||||
|
|
||||||
|
|
||||||
|
def cleanup_staging(conn, dry_run: bool) -> dict:
|
||||||
|
"""Шаг 2. Исходники ready (сразу) и failed (после retention)."""
|
||||||
|
report = {"staging_ready_deleted": 0, "staging_failed_expired": 0, "staging_kept_for_replay": 0}
|
||||||
|
while not STOP.is_set():
|
||||||
|
with conn.transaction():
|
||||||
|
rows = conn.execute(
|
||||||
|
"""SELECT a.id, a.state, a.source_key, j.job_id, j.attempt FROM attachments a
|
||||||
|
LEFT JOIN attachment_jobs j ON j.attachment_id = a.id AND j.pipeline_version = a.pipeline_version
|
||||||
|
WHERE a.source_key IS NOT NULL
|
||||||
|
AND ((a.state = 'ready' AND a.updated_at < now() - interval '60 seconds')
|
||||||
|
OR (a.state = 'failed' AND a.updated_at < now() - make_interval(secs => %s)))
|
||||||
|
ORDER BY a.updated_at LIMIT %s FOR UPDATE OF a SKIP LOCKED""",
|
||||||
|
(settings.STAGING_RETENTION_SECONDS, BATCH),
|
||||||
|
).fetchall()
|
||||||
|
if dry_run or not rows:
|
||||||
|
if dry_run:
|
||||||
|
report["staging_ready_deleted"] = sum(r["state"] == "ready" for r in rows)
|
||||||
|
report["staging_failed_expired"] = sum(r["state"] == "failed" for r in rows)
|
||||||
|
break
|
||||||
|
for row in rows:
|
||||||
|
storage.delete(row["source_key"])
|
||||||
|
conn.execute("UPDATE attachments SET source_key = NULL WHERE id = %s", (row["id"],))
|
||||||
|
if row["state"] == "ready":
|
||||||
|
report["staging_ready_deleted"] += 1
|
||||||
|
else:
|
||||||
|
report["staging_failed_expired"] += 1
|
||||||
|
if row["job_id"] is not None:
|
||||||
|
pipeline.event(conn, row["job_id"], row["attempt"], "staging_expired", ACTOR,
|
||||||
|
"replay no longer possible")
|
||||||
|
if len(rows) < BATCH:
|
||||||
|
break
|
||||||
|
report["staging_kept_for_replay"] = conn.execute(
|
||||||
|
"SELECT count(*) AS n FROM attachments WHERE state = 'failed' AND source_key IS NOT NULL").fetchone()["n"]
|
||||||
|
return report
|
||||||
|
|
||||||
|
|
||||||
|
def cleanup_outbox(conn, dry_run: bool) -> int:
|
||||||
|
if dry_run:
|
||||||
|
return conn.execute(
|
||||||
|
"""SELECT count(*) AS n FROM outbox WHERE published_at < now() - make_interval(secs => %s)""",
|
||||||
|
(settings.OUTBOX_RETENTION_SECONDS,)).fetchone()["n"]
|
||||||
|
return conn.execute(
|
||||||
|
"""DELETE FROM outbox WHERE published_at < now() - make_interval(secs => %s)""",
|
||||||
|
(settings.OUTBOX_RETENTION_SECONDS,)).rowcount
|
||||||
|
|
||||||
|
|
||||||
|
def classify_final(conn, keys: list[str]) -> set[str]:
|
||||||
|
"""Какие ключи attachments/<id>/... не принадлежат ни одному живому вложению (кандидаты в сироты)."""
|
||||||
|
ids = {}
|
||||||
|
for key in keys:
|
||||||
|
match = FINAL_KEY_RE.match(key)
|
||||||
|
if match:
|
||||||
|
try:
|
||||||
|
ids[key] = UUID(match.group(1))
|
||||||
|
except ValueError:
|
||||||
|
pass
|
||||||
|
unique = list(set(ids.values()))
|
||||||
|
rows = {r["id"]: r for r in conn.execute(
|
||||||
|
"""SELECT a.id, a.state, a.object_key, a.source_key,
|
||||||
|
ARRAY(SELECT v.object_key FROM attachment_variants v
|
||||||
|
WHERE v.attachment_id = a.id AND v.pipeline_version = a.pipeline_version) AS variant_keys
|
||||||
|
FROM attachments a WHERE a.id = ANY(%s)""", (unique,)).fetchall()}
|
||||||
|
orphans = set()
|
||||||
|
for key in keys:
|
||||||
|
attachment_id = ids.get(key)
|
||||||
|
row = rows.get(attachment_id) if attachment_id else None
|
||||||
|
if row is None:
|
||||||
|
orphans.add(key)
|
||||||
|
elif row["state"] == "failed" and row["source_key"] is None:
|
||||||
|
# failed без исходника — конечное состояние: replay отказывает («source file is no longer in
|
||||||
|
# staging»), других переходов из failed нет, поэтому никто больше не запишет эти ключи.
|
||||||
|
orphans.add(key)
|
||||||
|
elif row["state"] == "ready" and key != row["object_key"] and key not in row["variant_keys"]:
|
||||||
|
orphans.add(key)
|
||||||
|
return orphans
|
||||||
|
|
||||||
|
|
||||||
|
def sweep_objects(conn, grace: int, dry_run: bool) -> dict:
|
||||||
|
"""Шаг 4. Обход bucket постранично; сверка ключей с БД пачками по 1000."""
|
||||||
|
started = datetime.now(timezone.utc)
|
||||||
|
cutoff = started - timedelta(seconds=grace)
|
||||||
|
scanned = orphans = orphans_deleted = 0
|
||||||
|
listed = set()
|
||||||
|
|
||||||
|
def orphan_keys(prefix, items):
|
||||||
|
keys = [key for key, _ in items]
|
||||||
|
if prefix == storage.FINAL_PREFIX:
|
||||||
|
return classify_final(conn, keys)
|
||||||
|
known = {r["source_key"] for r in conn.execute(
|
||||||
|
"SELECT source_key FROM attachments WHERE source_key = ANY(%s)", (keys,)).fetchall()}
|
||||||
|
return {key for key in keys if key not in known}
|
||||||
|
|
||||||
|
def delete_if_orphan(prefix, key) -> bool:
|
||||||
|
"""Повторная проверка прямо перед удалением и DELETE — атомарно относительно replay/worker."""
|
||||||
|
if prefix != storage.FINAL_PREFIX:
|
||||||
|
# staging/originals: строка-указатель создаётся ДО PUT, ключ случайный и не переиспользуется,
|
||||||
|
# поэтому «нет строки сейчас» означает «не будет никогда».
|
||||||
|
if conn.execute("SELECT 1 FROM attachments WHERE source_key = %s", (key,)).fetchone() is not None:
|
||||||
|
return False
|
||||||
|
storage.delete(key)
|
||||||
|
return True
|
||||||
|
match = FINAL_KEY_RE.match(key)
|
||||||
|
try:
|
||||||
|
attachment_id = UUID(match.group(1)) if match else None
|
||||||
|
except ValueError:
|
||||||
|
attachment_id = None
|
||||||
|
with conn.transaction():
|
||||||
|
if attachment_id is not None:
|
||||||
|
# FOR SHARE совместим с FOR KEY SHARE (вставка сообщения со ссылкой на вложение), но
|
||||||
|
# конфликтует с FOR UPDATE replay и с UPDATE worker'а: состояние не изменится до COMMIT.
|
||||||
|
conn.execute("SELECT 1 FROM attachments WHERE id = %s FOR SHARE", (attachment_id,))
|
||||||
|
if key not in classify_final(conn, [key]):
|
||||||
|
return False
|
||||||
|
storage.delete(key) # S3 недоступен -> исключение, транзакция откатится, повтор на след. проходе
|
||||||
|
return True
|
||||||
|
|
||||||
|
def check(prefix, items):
|
||||||
|
nonlocal orphans, orphans_deleted
|
||||||
|
candidates = orphan_keys(prefix, items)
|
||||||
|
for key, modified in items:
|
||||||
|
if key not in candidates or modified >= cutoff:
|
||||||
|
continue
|
||||||
|
orphans += 1
|
||||||
|
if not dry_run and delete_if_orphan(prefix, key):
|
||||||
|
orphans_deleted += 1
|
||||||
|
log.info("deleted orphan object %s", key)
|
||||||
|
|
||||||
|
for prefix in storage.ALL_PREFIXES:
|
||||||
|
page = []
|
||||||
|
for key, modified, _size in storage.list_objects(prefix):
|
||||||
|
scanned += 1
|
||||||
|
listed.add(key)
|
||||||
|
page.append((key, modified))
|
||||||
|
if len(page) >= 1000:
|
||||||
|
check(prefix, page)
|
||||||
|
page = []
|
||||||
|
if page:
|
||||||
|
check(prefix, page)
|
||||||
|
|
||||||
|
missing = []
|
||||||
|
for r in conn.execute(
|
||||||
|
"""SELECT a.id, a.object_key, ARRAY(SELECT v.object_key FROM attachment_variants v
|
||||||
|
WHERE v.attachment_id = a.id AND v.pipeline_version = a.pipeline_version) AS variant_keys
|
||||||
|
FROM attachments a WHERE a.state = 'ready' AND a.updated_at < %s""", (started,)).fetchall():
|
||||||
|
if r["object_key"] not in listed or any(k not in listed for k in r["variant_keys"]):
|
||||||
|
missing.append(str(r["id"]))
|
||||||
|
if missing:
|
||||||
|
log.error("ready attachments with objects missing in S3 (not deleted, needs restore): %s",
|
||||||
|
", ".join(missing[:20]))
|
||||||
|
return {"objects_scanned": scanned, "orphan_objects": orphans, "orphan_objects_deleted": orphans_deleted,
|
||||||
|
"missing_objects": len(missing)}
|
||||||
|
|
||||||
|
|
||||||
|
def run_once(grace: int, dry_run: bool) -> dict | None:
|
||||||
|
# Один проход = одна трасса (раз в JANITOR_INTERVAL_SECONDS): запросы к БД и S3 — её дети.
|
||||||
|
with telemetry.internal_span("janitor pass", {"messenger.janitor.dry_run": dry_run}) as span:
|
||||||
|
report = _run_once(grace, dry_run)
|
||||||
|
if report:
|
||||||
|
span.set_attributes({f"messenger.janitor.{k}": v for k, v in report.items()
|
||||||
|
if isinstance(v, (int, float, bool))})
|
||||||
|
return report
|
||||||
|
|
||||||
|
|
||||||
|
def _run_once(grace: int, dry_run: bool) -> dict | None:
|
||||||
|
started = time.monotonic()
|
||||||
|
with connect() as conn:
|
||||||
|
if not conn.execute("SELECT pg_try_advisory_lock(%s) AS ok", (JANITOR_LOCK,)).fetchone()["ok"]:
|
||||||
|
log.info("another janitor is running, skip")
|
||||||
|
return None
|
||||||
|
try:
|
||||||
|
found, deleted = reclaim_stale_intents(conn, grace, dry_run)
|
||||||
|
report = {"dry_run": dry_run, "grace_seconds": grace, "stale_uploads": found,
|
||||||
|
"stale_uploads_deleted": deleted, **cleanup_staging(conn, dry_run),
|
||||||
|
"outbox_rows_deleted": cleanup_outbox(conn, dry_run), **sweep_objects(conn, grace, dry_run)}
|
||||||
|
finally:
|
||||||
|
conn.execute("SELECT pg_advisory_unlock(%s)", (JANITOR_LOCK,))
|
||||||
|
report["seconds"] = round(time.monotonic() - started, 3)
|
||||||
|
return report
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
parser = argparse.ArgumentParser(description="Reclaim abandoned uploads, clean staging, sweep orphan objects")
|
||||||
|
parser.add_argument("--once", action="store_true", help="one pass and exit")
|
||||||
|
parser.add_argument("--dry-run", action="store_true", help="only report, delete nothing")
|
||||||
|
parser.add_argument("--grace-seconds", type=int, default=settings.ATTACHMENT_GRACE_SECONDS)
|
||||||
|
args = parser.parse_args()
|
||||||
|
if args.grace_seconds < 2 * settings.UPLOAD_TIMEOUT_SECONDS:
|
||||||
|
parser.error(f"--grace-seconds must be >= {2 * settings.UPLOAD_TIMEOUT_SECONDS} "
|
||||||
|
"(2 x UPLOAD_TIMEOUT_SECONDS), otherwise a running upload could be deleted")
|
||||||
|
|
||||||
|
for sig in (signal.SIGTERM, signal.SIGINT):
|
||||||
|
signal.signal(sig, lambda *_: STOP.set())
|
||||||
|
while True:
|
||||||
|
touch_alive()
|
||||||
|
try:
|
||||||
|
report = run_once(args.grace_seconds, args.dry_run)
|
||||||
|
if report is not None:
|
||||||
|
print(json.dumps(report), flush=True)
|
||||||
|
log.info("janitor pass done", extra={"event": "janitor.pass", **{
|
||||||
|
k: v for k, v in report.items() if isinstance(v, (int, float, bool))}})
|
||||||
|
failed = False
|
||||||
|
except (psycopg.Error, storage.StorageUnavailable) as exc:
|
||||||
|
log.warning("pass failed, will retry: %s", type(exc).__name__,
|
||||||
|
extra={"event": "janitor.pass_failed", "error.type": type(exc).__name__})
|
||||||
|
failed = True
|
||||||
|
if args.once:
|
||||||
|
telemetry.shutdown()
|
||||||
|
raise SystemExit(1 if failed else 0)
|
||||||
|
if idle(settings.JANITOR_INTERVAL_SECONDS):
|
||||||
|
telemetry.shutdown()
|
||||||
|
return
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
@@ -0,0 +1,666 @@
|
|||||||
|
import base64
|
||||||
|
import binascii
|
||||||
|
import contextlib
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
import logging
|
||||||
|
import re
|
||||||
|
import threading
|
||||||
|
import time
|
||||||
|
import uuid
|
||||||
|
from contextlib import asynccontextmanager
|
||||||
|
from datetime import datetime
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Literal
|
||||||
|
from urllib.parse import quote
|
||||||
|
from uuid import UUID
|
||||||
|
|
||||||
|
import uvicorn
|
||||||
|
from fastapi import FastAPI, Header, Query, Request, Response
|
||||||
|
from fastapi.exceptions import RequestValidationError
|
||||||
|
from fastapi.responses import JSONResponse, StreamingResponse
|
||||||
|
from opentelemetry import trace
|
||||||
|
from opentelemetry.trace import SpanKind
|
||||||
|
from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator
|
||||||
|
from starlette.concurrency import run_in_threadpool
|
||||||
|
from starlette.exceptions import HTTPException as StarletteHTTPException
|
||||||
|
from starlette.routing import Match
|
||||||
|
|
||||||
|
import settings
|
||||||
|
import telemetry
|
||||||
|
|
||||||
|
# lab6: телеметрия настраивается до первого соединения с БД (pool.open в lifespan).
|
||||||
|
telemetry.setup("messenger-api")
|
||||||
|
|
||||||
|
import attachments # noqa: E402
|
||||||
|
import auth # noqa: E402
|
||||||
|
import db # noqa: E402
|
||||||
|
import storage # noqa: E402
|
||||||
|
import uploads # noqa: E402
|
||||||
|
from errors import ApiError, bad_request, conflict, not_found, unauthorized, unavailable # noqa: E402
|
||||||
|
|
||||||
|
log = logging.getLogger("messenger")
|
||||||
|
|
||||||
|
# HTTP-метрики (contracts/observability.md). Метки: метод, ШАБЛОН маршрута, код и класс статуса —
|
||||||
|
# никаких реальных UUID, raw URL, request_id. Длительность — до отправки заголовков ответа
|
||||||
|
# (для скачивания файла тело потом стримится из S3 и в длительность не входит).
|
||||||
|
HTTP_DURATION = telemetry.meter.create_histogram(
|
||||||
|
"http.server.request.duration", unit="s", description="Duration of HTTP server requests")
|
||||||
|
HTTP_ACTIVE = telemetry.meter.create_up_down_counter(
|
||||||
|
"http.server.active_requests", unit="{request}", description="HTTP requests being processed now")
|
||||||
|
KNOWN_METHODS = {"GET", "POST", "PUT", "PATCH", "DELETE", "HEAD", "OPTIONS"}
|
||||||
|
|
||||||
|
DRAINING = threading.Event()
|
||||||
|
|
||||||
|
|
||||||
|
class Server(uvicorn.Server):
|
||||||
|
"""Остановка по SIGTERM в два шага.
|
||||||
|
|
||||||
|
1. /health/ready сразу отвечает 503, а запросы ещё обслуживаются. За SHUTDOWN_DRAIN_SECONDS
|
||||||
|
HAProxy видит 503 (проверка раз в 0,5-1 с, 2 неудачи) и перестаёт слать сюда новые запросы.
|
||||||
|
2. Обычная остановка uvicorn: закрыть приём соединений и доделать начатые запросы
|
||||||
|
(не дольше SHUTDOWN_TIMEOUT_SECONDS). Без шага 1 запрос, пришедший в момент SIGTERM,
|
||||||
|
получал бы 502 от балансировщика.
|
||||||
|
Повторный сигнал останавливает сразу.
|
||||||
|
"""
|
||||||
|
|
||||||
|
def handle_exit(self, sig, frame):
|
||||||
|
if DRAINING.is_set():
|
||||||
|
return super().handle_exit(sig, frame)
|
||||||
|
DRAINING.set()
|
||||||
|
log.info("Signal %s: readiness -> 503, leaving the balancer, stop in %s s",
|
||||||
|
sig, settings.SHUTDOWN_DRAIN_SECONDS)
|
||||||
|
threading.Timer(settings.SHUTDOWN_DRAIN_SECONDS, super().handle_exit, (sig, frame)).start()
|
||||||
|
|
||||||
|
|
||||||
|
@asynccontextmanager
|
||||||
|
async def lifespan(app: FastAPI):
|
||||||
|
settings.check()
|
||||||
|
# Миграции применяет сервис migrate (migrate.py); реплика только ждёт готовую схему.
|
||||||
|
await run_in_threadpool(db.wait_for_schema, settings.SCHEMA_WAIT_SECONDS)
|
||||||
|
# Только для временных файлов текущих загрузок (удаляются в конце запроса); оригиналы живут в S3.
|
||||||
|
Path(settings.UPLOAD_TMP_DIR).mkdir(parents=True, exist_ok=True)
|
||||||
|
db.pool.open()
|
||||||
|
log.info("Started: instance=%s, version=%s, env=%s, session TTL=%s s, sessions in Redis, files in S3 bucket %s, "
|
||||||
|
"processing via outbox (pipeline v%s)", settings.INSTANCE_ID, settings.APP_VERSION, settings.APP_ENV,
|
||||||
|
settings.SESSION_TTL_SECONDS, settings.S3_BUCKET, settings.PIPELINE_VERSION,
|
||||||
|
extra={"event": "service.started"})
|
||||||
|
yield
|
||||||
|
db.pool.close()
|
||||||
|
auth.close()
|
||||||
|
log.info("Stopped: connections closed, data stays in PostgreSQL and Redis")
|
||||||
|
telemetry.shutdown()
|
||||||
|
|
||||||
|
|
||||||
|
app = FastAPI(lifespan=lifespan)
|
||||||
|
|
||||||
|
|
||||||
|
USERNAME_RE = re.compile(r"[a-z0-9_]{3,32}")
|
||||||
|
IDEMPOTENCY_KEY_RE = re.compile(r"[A-Za-z0-9._:-]{1,128}")
|
||||||
|
|
||||||
|
|
||||||
|
class CreateUser(BaseModel):
|
||||||
|
model_config = ConfigDict(extra="forbid")
|
||||||
|
|
||||||
|
username: str = Field(pattern=r"^[a-z0-9_]{3,32}$")
|
||||||
|
display_name: str = Field(min_length=1, max_length=100)
|
||||||
|
password: str = Field(min_length=auth.PASSWORD_MIN_LENGTH, max_length=auth.PASSWORD_MAX_LENGTH)
|
||||||
|
|
||||||
|
@field_validator("display_name", "password")
|
||||||
|
@classmethod
|
||||||
|
def validate_not_blank(cls, value: str) -> str:
|
||||||
|
if not value.strip():
|
||||||
|
raise ValueError("value cannot be blank")
|
||||||
|
return value
|
||||||
|
|
||||||
|
|
||||||
|
class User(BaseModel):
|
||||||
|
id: UUID
|
||||||
|
username: str
|
||||||
|
display_name: str
|
||||||
|
created_at: datetime
|
||||||
|
|
||||||
|
|
||||||
|
class Session(BaseModel):
|
||||||
|
token: str
|
||||||
|
token_type: str
|
||||||
|
expires_at: datetime
|
||||||
|
user: User
|
||||||
|
|
||||||
|
|
||||||
|
class CreateChat(BaseModel):
|
||||||
|
model_config = ConfigDict(extra="forbid")
|
||||||
|
|
||||||
|
title: str = Field(min_length=1, max_length=100)
|
||||||
|
member_ids: list[UUID] = Field(min_length=1, max_length=100)
|
||||||
|
|
||||||
|
@field_validator("title")
|
||||||
|
@classmethod
|
||||||
|
def validate_title(cls, value: str) -> str:
|
||||||
|
if not value.strip():
|
||||||
|
raise ValueError("title cannot be empty")
|
||||||
|
return value
|
||||||
|
|
||||||
|
|
||||||
|
class Chat(BaseModel):
|
||||||
|
id: UUID
|
||||||
|
title: str
|
||||||
|
member_ids: list[UUID]
|
||||||
|
created_at: datetime
|
||||||
|
|
||||||
|
|
||||||
|
class ChatPage(BaseModel):
|
||||||
|
items: list[Chat]
|
||||||
|
next_cursor: str | None
|
||||||
|
|
||||||
|
|
||||||
|
class CreateMessage(BaseModel):
|
||||||
|
model_config = ConfigDict(extra="forbid")
|
||||||
|
|
||||||
|
text: str = Field(max_length=4000)
|
||||||
|
# С lab4: 0-10 уникальных вложений этого чата; пустой текст допустим только с вложением.
|
||||||
|
attachment_ids: list[UUID] = Field(default_factory=list, max_length=10)
|
||||||
|
|
||||||
|
@model_validator(mode="after")
|
||||||
|
def validate_content(self):
|
||||||
|
if len(set(self.attachment_ids)) != len(self.attachment_ids):
|
||||||
|
raise ValueError("attachment_ids must be unique")
|
||||||
|
if not self.attachment_ids and not self.text.strip():
|
||||||
|
raise ValueError("text cannot be empty without attachments")
|
||||||
|
return self
|
||||||
|
|
||||||
|
|
||||||
|
class Message(BaseModel):
|
||||||
|
id: UUID
|
||||||
|
chat_id: UUID
|
||||||
|
sender_id: UUID
|
||||||
|
text: str
|
||||||
|
created_at: datetime
|
||||||
|
attachment_ids: list[UUID]
|
||||||
|
|
||||||
|
|
||||||
|
class Variant(BaseModel):
|
||||||
|
name: Literal["thumbnail", "crop", "optimized"]
|
||||||
|
content_type: Literal["image/jpeg", "image/png", "image/webp"]
|
||||||
|
size_bytes: int
|
||||||
|
width: int
|
||||||
|
height: int
|
||||||
|
sha256: str
|
||||||
|
|
||||||
|
|
||||||
|
class Attachment(BaseModel):
|
||||||
|
id: UUID
|
||||||
|
chat_id: UUID
|
||||||
|
owner_id: UUID
|
||||||
|
filename: str
|
||||||
|
content_type: str
|
||||||
|
size_bytes: int
|
||||||
|
sha256: str
|
||||||
|
status: Literal["queued", "processing", "ready", "failed"]
|
||||||
|
variants: list[Variant]
|
||||||
|
error_code: str | None
|
||||||
|
created_at: datetime
|
||||||
|
updated_at: datetime
|
||||||
|
|
||||||
|
|
||||||
|
class MessagePage(BaseModel):
|
||||||
|
items: list[Message]
|
||||||
|
next_cursor: str | None
|
||||||
|
|
||||||
|
|
||||||
|
def error_response(
|
||||||
|
request: Request,
|
||||||
|
code: str,
|
||||||
|
message: str,
|
||||||
|
status_code: int,
|
||||||
|
headers: dict | None = None,
|
||||||
|
):
|
||||||
|
request_id = request.state.request_id
|
||||||
|
|
||||||
|
return JSONResponse(
|
||||||
|
status_code=status_code,
|
||||||
|
content={
|
||||||
|
"error": {
|
||||||
|
"code": code,
|
||||||
|
"message": message,
|
||||||
|
"request_id": request_id,
|
||||||
|
}
|
||||||
|
},
|
||||||
|
headers={"X-Request-Id": request_id, **(headers or {})},
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@app.exception_handler(ApiError)
|
||||||
|
async def api_error_handler(request: Request, exc: ApiError):
|
||||||
|
return error_response(request, exc.code, exc.message, exc.status_code, exc.headers)
|
||||||
|
|
||||||
|
|
||||||
|
@app.exception_handler(RequestValidationError)
|
||||||
|
async def validation_exception_handler(
|
||||||
|
request: Request,
|
||||||
|
exc: RequestValidationError,
|
||||||
|
):
|
||||||
|
return error_response(
|
||||||
|
request,
|
||||||
|
"invalid_request",
|
||||||
|
"Invalid request data",
|
||||||
|
400,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@app.exception_handler(StarletteHTTPException)
|
||||||
|
async def http_exception_handler(request: Request, exc: StarletteHTTPException):
|
||||||
|
if exc.status_code == 404:
|
||||||
|
return error_response(request, "not_found", "Not found", 404)
|
||||||
|
return error_response(request, "invalid_request", "Request cannot be processed", exc.status_code, exc.headers)
|
||||||
|
|
||||||
|
|
||||||
|
@app.exception_handler(Exception)
|
||||||
|
async def general_exception_handler(
|
||||||
|
request: Request,
|
||||||
|
exc: Exception,
|
||||||
|
):
|
||||||
|
return error_response(
|
||||||
|
request,
|
||||||
|
"internal_error",
|
||||||
|
"Internal server error",
|
||||||
|
500,
|
||||||
|
)
|
||||||
|
|
||||||
|
PUBLIC_ENDPOINTS = {
|
||||||
|
("POST", "/api/v1/users"),
|
||||||
|
("POST", "/api/v1/auth/sessions"),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def route_template(request: Request) -> str:
|
||||||
|
"""Шаблон маршрута (/api/v1/chats/{chat_id}/messages) — метка метрик и имя спана вместо raw URL."""
|
||||||
|
for route in app.router.routes:
|
||||||
|
match, _ = route.matches(request.scope)
|
||||||
|
if match != Match.NONE:
|
||||||
|
return getattr(route, "path", "unmatched")
|
||||||
|
return "unmatched"
|
||||||
|
|
||||||
|
|
||||||
|
@app.middleware("http")
|
||||||
|
async def request_context(request: Request, call_next):
|
||||||
|
request_id = str(uuid.uuid4())
|
||||||
|
request.state.request_id = request_id
|
||||||
|
started = time.perf_counter()
|
||||||
|
path = request.url.path
|
||||||
|
method = request.method if request.method in KNOWN_METHODS else "_OTHER"
|
||||||
|
route = route_template(request)
|
||||||
|
# Проверки здоровья (HAProxy раз в секунду, Docker) — только в метриках, без трасс: иначе это
|
||||||
|
# большая часть всех трасс. Входящий traceparent клиента продолжается (W3C Trace Context).
|
||||||
|
traced = not path.startswith("/health/")
|
||||||
|
span_scope = (telemetry.tracer.start_as_current_span(
|
||||||
|
f"{method} {route}", context=telemetry.extract(request.headers), kind=SpanKind.SERVER,
|
||||||
|
attributes={"http.request.method": method, "http.route": route, "url.scheme": "http",
|
||||||
|
"request.id": request_id},
|
||||||
|
record_exception=False, set_status_on_exception=False)
|
||||||
|
if traced else contextlib.nullcontext(trace.INVALID_SPAN))
|
||||||
|
HTTP_ACTIVE.add(1, {"http.request.method": method})
|
||||||
|
status = 500
|
||||||
|
try:
|
||||||
|
with span_scope as span:
|
||||||
|
try:
|
||||||
|
if path.startswith("/api/") and (request.method, path) not in PUBLIC_ENDPOINTS:
|
||||||
|
request.state.session = await run_in_threadpool(
|
||||||
|
auth.check_bearer, request.headers.get("Authorization")
|
||||||
|
)
|
||||||
|
response = await call_next(request)
|
||||||
|
except ApiError as exc:
|
||||||
|
response = error_response(request, exc.code, exc.message, exc.status_code, exc.headers)
|
||||||
|
except Exception as exc:
|
||||||
|
log.exception("Unhandled error", extra={"event": "http.unhandled_error", "request_id": request_id})
|
||||||
|
telemetry.mark_error(span, type(exc).__name__)
|
||||||
|
response = error_response(request, "internal_error", "Internal server error", 500)
|
||||||
|
|
||||||
|
status = response.status_code
|
||||||
|
span.set_attribute("http.response.status_code", status)
|
||||||
|
if status >= 500:
|
||||||
|
telemetry.mark_error(span, str(status))
|
||||||
|
response.headers["X-Request-Id"] = request_id
|
||||||
|
response.headers["X-Instance-Id"] = settings.INSTANCE_ID # какая реплика ответила
|
||||||
|
trace_id = telemetry.trace_id_hex() if traced else None
|
||||||
|
if trace_id:
|
||||||
|
response.headers["X-Trace-Id"] = trace_id # расширение: найти трассу по ответу
|
||||||
|
elapsed = time.perf_counter() - started
|
||||||
|
# Проверки здоровья приходят каждую секунду от HAProxy и Docker, в лог пишем только ошибки.
|
||||||
|
level = logging.DEBUG if not traced and status == 200 else logging.INFO
|
||||||
|
log.log(level, "%s %s -> %s (%.1f ms)", method, route, status, elapsed * 1000,
|
||||||
|
extra={"event": "http.request", "http.request.method": method, "http.route": route,
|
||||||
|
"http.response.status_code": status, "duration_ms": round(elapsed * 1000, 1),
|
||||||
|
"request_id": request_id})
|
||||||
|
finally:
|
||||||
|
HTTP_ACTIVE.add(-1, {"http.request.method": method})
|
||||||
|
elapsed = time.perf_counter() - started
|
||||||
|
HTTP_DURATION.record(elapsed, {"http.request.method": method, "http.route": route,
|
||||||
|
"http.response.status_code": status,
|
||||||
|
"http.response.status_class": f"{status // 100}xx"})
|
||||||
|
return response
|
||||||
|
|
||||||
|
|
||||||
|
def current_user_id(request: Request) -> UUID:
|
||||||
|
session = getattr(request.state, "session", None)
|
||||||
|
if session is None:
|
||||||
|
raise unauthorized("Bearer token is required")
|
||||||
|
return session.user_id
|
||||||
|
|
||||||
|
|
||||||
|
def encode_cursor(
|
||||||
|
created_at: datetime,
|
||||||
|
object_id: UUID,
|
||||||
|
*,
|
||||||
|
chat_id: UUID | None = None,
|
||||||
|
user_id: UUID | None = None,
|
||||||
|
) -> str:
|
||||||
|
data = {
|
||||||
|
"created_at": created_at.isoformat(),
|
||||||
|
"id": str(object_id),
|
||||||
|
}
|
||||||
|
|
||||||
|
if chat_id is not None:
|
||||||
|
data["chat_id"] = str(chat_id)
|
||||||
|
|
||||||
|
if user_id is not None:
|
||||||
|
data["user_id"] = str(user_id)
|
||||||
|
|
||||||
|
raw = json.dumps(data).encode("utf-8")
|
||||||
|
return base64.urlsafe_b64encode(raw).decode("ascii")
|
||||||
|
|
||||||
|
|
||||||
|
def decode_cursor(cursor: str):
|
||||||
|
try:
|
||||||
|
raw = base64.urlsafe_b64decode(cursor.encode("ascii"))
|
||||||
|
data = json.loads(raw.decode("utf-8"))
|
||||||
|
|
||||||
|
created_at = datetime.fromisoformat(data["created_at"])
|
||||||
|
|
||||||
|
if created_at.tzinfo is None:
|
||||||
|
raise ValueError("Invalid cursor")
|
||||||
|
|
||||||
|
if not isinstance(data["id"], str):
|
||||||
|
raise ValueError("Invalid cursor")
|
||||||
|
|
||||||
|
object_id = UUID(data["id"])
|
||||||
|
|
||||||
|
chat_id = UUID(data["chat_id"]) if "chat_id" in data else None
|
||||||
|
user_id = UUID(data["user_id"]) if "user_id" in data else None
|
||||||
|
|
||||||
|
return created_at, object_id, chat_id, user_id
|
||||||
|
|
||||||
|
except (
|
||||||
|
ValueError,
|
||||||
|
KeyError,
|
||||||
|
TypeError,
|
||||||
|
AttributeError,
|
||||||
|
json.JSONDecodeError,
|
||||||
|
binascii.Error,
|
||||||
|
):
|
||||||
|
raise ValueError("Invalid cursor")
|
||||||
|
|
||||||
|
|
||||||
|
def page_after(cursor: str | None, *, chat_id: UUID | None = None, user_id: UUID | None = None):
|
||||||
|
if cursor is None:
|
||||||
|
return None
|
||||||
|
try:
|
||||||
|
created_at, object_id, cursor_chat_id, cursor_user_id = decode_cursor(cursor)
|
||||||
|
except ValueError:
|
||||||
|
raise bad_request("Invalid cursor") from None
|
||||||
|
if cursor_chat_id != chat_id or cursor_user_id != user_id:
|
||||||
|
raise bad_request("Invalid cursor")
|
||||||
|
return created_at, object_id
|
||||||
|
|
||||||
|
|
||||||
|
@app.get("/health/live")
|
||||||
|
def health_live():
|
||||||
|
return {"status": "ok"}
|
||||||
|
|
||||||
|
|
||||||
|
@app.get("/health/ready")
|
||||||
|
def health_ready():
|
||||||
|
if DRAINING.is_set():
|
||||||
|
raise unavailable("Instance is shutting down")
|
||||||
|
db.ping()
|
||||||
|
auth.ping_redis()
|
||||||
|
return {"status": "ok"}
|
||||||
|
|
||||||
|
|
||||||
|
@app.post("/api/v1/users", response_model=User, status_code=201)
|
||||||
|
def create_user(user_data: CreateUser):
|
||||||
|
password_hash = auth.hash_password(user_data.password) # сам пароль нигде не сохраняется
|
||||||
|
return db.create_user(user_data.username, user_data.display_name, password_hash)
|
||||||
|
|
||||||
|
|
||||||
|
@app.post("/api/v1/auth/sessions", response_model=Session, status_code=201)
|
||||||
|
def create_session(
|
||||||
|
response: Response,
|
||||||
|
authorization: str | None = Header(default=None),
|
||||||
|
):
|
||||||
|
credentials = auth.parse_basic(authorization)
|
||||||
|
if credentials is None:
|
||||||
|
raise auth.login_failed()
|
||||||
|
username, password = credentials
|
||||||
|
|
||||||
|
found = None
|
||||||
|
if USERNAME_RE.fullmatch(username) and len(password) <= auth.PASSWORD_MAX_LENGTH:
|
||||||
|
found = db.get_user_with_password(username)
|
||||||
|
if not auth.password_matches(found["password_hash"] if found else None, password):
|
||||||
|
raise auth.login_failed()
|
||||||
|
|
||||||
|
token, expires_at = auth.create_session(found["id"])
|
||||||
|
response.headers["Cache-Control"] = "no-store"
|
||||||
|
user = {key: found[key] for key in ("id", "username", "display_name", "created_at")}
|
||||||
|
return {"token": token, "token_type": "Bearer", "expires_at": expires_at, "user": user}
|
||||||
|
|
||||||
|
|
||||||
|
@app.delete("/api/v1/auth/sessions/current", status_code=204)
|
||||||
|
def delete_session(request: Request):
|
||||||
|
current_user_id(request)
|
||||||
|
auth.delete_session(request.state.session)
|
||||||
|
return Response(status_code=204)
|
||||||
|
|
||||||
|
|
||||||
|
@app.get("/api/v1/users/me", response_model=User)
|
||||||
|
def get_current_user(request: Request):
|
||||||
|
user = db.get_user(current_user_id(request))
|
||||||
|
if user is None:
|
||||||
|
raise unauthorized("Session is invalid or expired")
|
||||||
|
return user
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
@app.post("/api/v1/chats", response_model=Chat, status_code=201)
|
||||||
|
def create_chat(request: Request, chat_data: CreateChat):
|
||||||
|
user_id = current_user_id(request)
|
||||||
|
|
||||||
|
if user_id not in chat_data.member_ids:
|
||||||
|
raise bad_request("Creator must be a chat member")
|
||||||
|
|
||||||
|
if len(chat_data.member_ids) != len(set(chat_data.member_ids)):
|
||||||
|
raise bad_request("Duplicate member IDs")
|
||||||
|
return db.create_chat(chat_data.title, chat_data.member_ids)
|
||||||
|
|
||||||
|
|
||||||
|
@app.get("/api/v1/chats", response_model=ChatPage)
|
||||||
|
def list_chats(
|
||||||
|
request: Request,
|
||||||
|
limit: int = Query(default=50, ge=1, le=100),
|
||||||
|
cursor: str | None = Query(default=None),
|
||||||
|
):
|
||||||
|
user_id = current_user_id(request)
|
||||||
|
after = page_after(cursor, user_id=user_id)
|
||||||
|
|
||||||
|
rows = db.list_chats(user_id, after, limit)
|
||||||
|
items = rows[:limit]
|
||||||
|
|
||||||
|
next_cursor = None
|
||||||
|
if len(rows) > limit:
|
||||||
|
last = items[-1]
|
||||||
|
next_cursor = encode_cursor(last["created_at"], last["id"], user_id=user_id)
|
||||||
|
|
||||||
|
return ChatPage(items=items, next_cursor=next_cursor)
|
||||||
|
|
||||||
|
|
||||||
|
@app.get("/api/v1/chats/{chat_id}", response_model=Chat)
|
||||||
|
def get_chat(request: Request, chat_id: UUID):
|
||||||
|
return db.get_chat(chat_id, current_user_id(request))
|
||||||
|
|
||||||
|
|
||||||
|
def request_fingerprint(data: BaseModel) -> bytes:
|
||||||
|
"""SHA-256 тела после разбора: порядок ключей и пробелы в JSON на него не влияют.
|
||||||
|
|
||||||
|
Пустой attachment_ids не входит в отпечаток: {"text": "x"} и {"text": "x", "attachment_ids": []}
|
||||||
|
равны (контракт), и хеши сообщений, сохранённых до lab4, остаются прежними.
|
||||||
|
"""
|
||||||
|
body = data.model_dump(mode="json")
|
||||||
|
if not body.get("attachment_ids"):
|
||||||
|
body.pop("attachment_ids", None)
|
||||||
|
canonical = json.dumps(body, sort_keys=True, ensure_ascii=False, separators=(",", ":"))
|
||||||
|
return hashlib.sha256(canonical.encode("utf-8")).digest()
|
||||||
|
|
||||||
|
|
||||||
|
@app.post(
|
||||||
|
"/api/v1/chats/{chat_id}/messages",
|
||||||
|
response_model=Message,
|
||||||
|
status_code=201,
|
||||||
|
)
|
||||||
|
def create_message(
|
||||||
|
request: Request,
|
||||||
|
message_data: CreateMessage,
|
||||||
|
chat_id: UUID,
|
||||||
|
idempotency_key: str | None = Header(default=None),
|
||||||
|
):
|
||||||
|
# С lab3 ключ обязателен: повтор того же запроса (после таймаута, через другую реплику)
|
||||||
|
# вернёт то же сообщение, а не создаст второе. См. db.add_message.
|
||||||
|
if idempotency_key is None or not IDEMPOTENCY_KEY_RE.fullmatch(idempotency_key):
|
||||||
|
raise bad_request("Idempotency-Key header is required: 1-128 characters A-Z a-z 0-9 . _ : -")
|
||||||
|
return db.add_message(chat_id, current_user_id(request), message_data.text, message_data.attachment_ids,
|
||||||
|
idempotency_key, request_fingerprint(message_data))
|
||||||
|
|
||||||
|
|
||||||
|
@app.get(
|
||||||
|
"/api/v1/chats/{chat_id}/messages",
|
||||||
|
response_model=MessagePage,
|
||||||
|
)
|
||||||
|
def list_messages(
|
||||||
|
request: Request,
|
||||||
|
chat_id: UUID,
|
||||||
|
limit: int = Query(default=50, ge=1, le=100),
|
||||||
|
cursor: str | None = Query(default=None),
|
||||||
|
):
|
||||||
|
user_id = current_user_id(request)
|
||||||
|
after = page_after(cursor, chat_id=chat_id)
|
||||||
|
|
||||||
|
rows = db.list_messages(chat_id, user_id, after, limit)
|
||||||
|
items = rows[:limit]
|
||||||
|
|
||||||
|
next_cursor = None
|
||||||
|
if len(rows) > limit:
|
||||||
|
last = items[-1]
|
||||||
|
next_cursor = encode_cursor(last["created_at"], last["id"], chat_id=chat_id)
|
||||||
|
|
||||||
|
return MessagePage(items=items, next_cursor=next_cursor)
|
||||||
|
|
||||||
|
|
||||||
|
@app.post(
|
||||||
|
"/api/v1/chats/{chat_id}/attachments",
|
||||||
|
response_model=Attachment,
|
||||||
|
status_code=202,
|
||||||
|
)
|
||||||
|
async def upload_attachment(
|
||||||
|
request: Request,
|
||||||
|
chat_id: UUID,
|
||||||
|
filename: str | None = Query(default=None),
|
||||||
|
):
|
||||||
|
"""Приём PNG/JPEG сырыми байтами. 202 queued — когда исходник в staging (S3), а задание и сообщение
|
||||||
|
для брокера зафиксированы в PostgreSQL одной транзакцией (attachments.store). Обработку делает worker.
|
||||||
|
|
||||||
|
Порядок проверок: сессия (middleware) -> участник чата (404) -> filename (400) -> Content-Type (415)
|
||||||
|
-> Content-Length (413) — всё до чтения тела; затем потоковое чтение с лимитом (413),
|
||||||
|
проверка содержимого (415/413) и запись.
|
||||||
|
"""
|
||||||
|
user_id = current_user_id(request)
|
||||||
|
try:
|
||||||
|
await run_in_threadpool(db.check_member, chat_id, user_id)
|
||||||
|
name = uploads.validate_filename(filename)
|
||||||
|
declared = uploads.declared_type(request.headers.get("content-type"))
|
||||||
|
uploads.declared_length(request.headers.get("content-length"))
|
||||||
|
except ApiError:
|
||||||
|
await uploads.drain(request) # чтобы клиент получил ответ с ошибкой, а не обрыв на отправке
|
||||||
|
raise
|
||||||
|
upload = await uploads.receive(request)
|
||||||
|
try:
|
||||||
|
image = await run_in_threadpool(inspect_traced, upload, declared)
|
||||||
|
row = await run_in_threadpool(attachments.store, chat_id, user_id, name, upload, image)
|
||||||
|
finally:
|
||||||
|
upload.file.close() # временный файл удаляется сразу; исходник остаётся только в staging (S3)
|
||||||
|
return attachments.public(row)
|
||||||
|
|
||||||
|
|
||||||
|
def inspect_traced(upload, declared):
|
||||||
|
"""Проверка содержимого — CPU в обработчике (не БД): в трассе видно отдельным спаном."""
|
||||||
|
with telemetry.internal_span("inspect image", {"http.request.body.size": upload.size}):
|
||||||
|
return uploads.inspect(upload, declared)
|
||||||
|
|
||||||
|
|
||||||
|
@app.get("/api/v1/attachments/{attachment_id}", response_model=Attachment)
|
||||||
|
def get_attachment(request: Request, attachment_id: UUID):
|
||||||
|
return attachments.public(db.get_attachment(attachment_id, current_user_id(request)))
|
||||||
|
|
||||||
|
|
||||||
|
VARIANT_SUFFIX = {"image/jpeg": ".jpg", "image/png": ".png", "image/webp": ".webp"}
|
||||||
|
|
||||||
|
|
||||||
|
@app.get("/api/v1/attachments/{attachment_id}/content")
|
||||||
|
def download_attachment(
|
||||||
|
request: Request,
|
||||||
|
attachment_id: UUID,
|
||||||
|
variant: Literal["original", "thumbnail", "crop", "optimized"] = Query(default="original"),
|
||||||
|
):
|
||||||
|
"""Байты оригинала или варианта через API (без redirect): сначала участник чата, потом S3.
|
||||||
|
|
||||||
|
queued/processing/failed -> 409 (ещё/уже нечего отдавать); варианта нет у ready -> 404.
|
||||||
|
"""
|
||||||
|
row = db.get_attachment(attachment_id, current_user_id(request))
|
||||||
|
if row["state"] != "ready":
|
||||||
|
raise conflict("Attachment is not ready")
|
||||||
|
if variant == "original":
|
||||||
|
key, content_type, size, sha256 = row["object_key"], row["content_type"], row["size_bytes"], row["sha256"]
|
||||||
|
name = row["filename"]
|
||||||
|
else:
|
||||||
|
found = [v for v in row["variants"] if v["name"] == variant]
|
||||||
|
if not found:
|
||||||
|
raise not_found("Variant not found")
|
||||||
|
v = found[0]
|
||||||
|
key, content_type, size, sha256 = v["object_key"], v["content_type"], v["size_bytes"], v["sha256"]
|
||||||
|
stem = row["filename"].rsplit(".", 1)[0] or "image"
|
||||||
|
name = f"{stem}.{variant}{VARIANT_SUFFIX[content_type]}"
|
||||||
|
try:
|
||||||
|
stream = storage.open_read(key)
|
||||||
|
except storage.ObjectMissing:
|
||||||
|
log.error("Object %s of ready attachment %s is missing in S3", variant, row["id"],
|
||||||
|
extra={"event": "attachment.object_missing", "attachment.id": str(row["id"])})
|
||||||
|
raise unavailable("File is temporarily unavailable") from None
|
||||||
|
except storage.StorageUnavailable:
|
||||||
|
raise unavailable("File storage is temporarily unavailable") from None
|
||||||
|
headers = {
|
||||||
|
"Content-Length": str(size),
|
||||||
|
"Content-Disposition": "inline; filename*=UTF-8''" + quote(name, safe=""),
|
||||||
|
"X-Content-Type-Options": "nosniff",
|
||||||
|
"Cache-Control": "private, no-store",
|
||||||
|
"ETag": '"' + sha256 + '"',
|
||||||
|
}
|
||||||
|
return StreamingResponse(storage.iter_chunks(stream), media_type=content_type, headers=headers)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
Server(uvicorn.Config(
|
||||||
|
app,
|
||||||
|
host=settings.APP_HOST,
|
||||||
|
port=settings.APP_PORT,
|
||||||
|
timeout_graceful_shutdown=settings.SHUTDOWN_TIMEOUT_SECONDS,
|
||||||
|
access_log=False,
|
||||||
|
log_config=None, # логи uvicorn идут в общий JSON/OTLP-обработчик (telemetry.py)
|
||||||
|
)).run()
|
||||||
@@ -0,0 +1,8 @@
|
|||||||
|
import logging
|
||||||
|
|
||||||
|
import db
|
||||||
|
|
||||||
|
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(name)s: %(message)s")
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
db.wait_and_migrate()
|
||||||
@@ -0,0 +1,33 @@
|
|||||||
|
|
||||||
|
CREATE TABLE users (
|
||||||
|
id uuid PRIMARY KEY,
|
||||||
|
username text NOT NULL UNIQUE
|
||||||
|
CHECK (username ~ '^[a-z0-9_]{3,32}$'),
|
||||||
|
display_name text NOT NULL CHECK (char_length(display_name) BETWEEN 1 AND 100),
|
||||||
|
password_hash text NOT NULL,
|
||||||
|
created_at timestamptz NOT NULL DEFAULT now()
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE TABLE chats (
|
||||||
|
id uuid PRIMARY KEY,
|
||||||
|
title text NOT NULL CHECK (char_length(title) BETWEEN 1 AND 100),
|
||||||
|
created_at timestamptz NOT NULL DEFAULT now()
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE TABLE chat_members (
|
||||||
|
chat_id uuid NOT NULL REFERENCES chats (id),
|
||||||
|
user_id uuid NOT NULL REFERENCES users (id),
|
||||||
|
PRIMARY KEY (chat_id, user_id)
|
||||||
|
);
|
||||||
|
CREATE INDEX chat_members_by_user ON chat_members (user_id);
|
||||||
|
|
||||||
|
CREATE TABLE messages (
|
||||||
|
id uuid PRIMARY KEY,
|
||||||
|
chat_id uuid NOT NULL,
|
||||||
|
sender_id uuid NOT NULL,
|
||||||
|
text text NOT NULL CHECK (char_length(text) BETWEEN 1 AND 4000),
|
||||||
|
created_at timestamptz NOT NULL DEFAULT now(),
|
||||||
|
FOREIGN KEY (chat_id, sender_id) REFERENCES chat_members (chat_id, user_id)
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE INDEX messages_history ON messages (chat_id, created_at, id);
|
||||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user