ЛР6 - БПИ2301 - Антонов - на 4 #58
@@ -16,5 +16,9 @@ backups
|
||||
.venv
|
||||
__pycache__
|
||||
node_modules
|
||||
dist
|
||||
coverage
|
||||
evidence
|
||||
.DS_Store
|
||||
.idea
|
||||
.vscode
|
||||
|
||||
@@ -23,6 +23,10 @@ WORKER_CONCURRENCY=2
|
||||
JOB_MAX_ATTEMPTS=5
|
||||
JOB_TIMEOUT_SECONDS=60
|
||||
STAGING_RETENTION_SECONDS=86400
|
||||
JOB_LEASE_SECONDS=90
|
||||
JOB_STALE_SECONDS=300
|
||||
RECONCILE_GRACE_SECONDS=3600
|
||||
RECONCILE_INTERVAL_SECONDS=300
|
||||
OTEL_SERVICE_NAME=messenger-api
|
||||
# Worker must set a distinct service.name.
|
||||
OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4318
|
||||
|
||||
@@ -0,0 +1,2 @@
|
||||
* text=auto eol=lf
|
||||
*.png binary
|
||||
@@ -0,0 +1,6 @@
|
||||
{
|
||||
"useTabs": true,
|
||||
"tabWidth": 4,
|
||||
"singleQuote": true,
|
||||
"trailingComma": "all"
|
||||
}
|
||||
+45
@@ -0,0 +1,45 @@
|
||||
# syntax=docker/dockerfile:1
|
||||
|
||||
FROM node:20.17.0-slim AS base
|
||||
RUN apt-get update \
|
||||
&& apt-get install -y --no-install-recommends openssl \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
WORKDIR /app
|
||||
|
||||
FROM base AS deps
|
||||
COPY package.json package-lock.json ./
|
||||
RUN npm ci
|
||||
|
||||
FROM deps AS build
|
||||
COPY prisma ./prisma
|
||||
RUN npx prisma generate
|
||||
COPY tsconfig.json tsconfig.build.json nest-cli.json ./
|
||||
COPY src ./src
|
||||
RUN npm run build
|
||||
|
||||
FROM build AS test
|
||||
COPY contracts/openapi.json ./contracts/openapi.json
|
||||
COPY tests/fixtures/sample.png ./tests/fixtures/sample.png
|
||||
COPY test ./test
|
||||
CMD ["npm", "test"]
|
||||
|
||||
FROM build AS migrate
|
||||
CMD ["sh", "-c", "DATABASE_URL=\"$(cat \"$DATABASE_URL_FILE\")\" exec npx prisma migrate deploy"]
|
||||
|
||||
FROM build AS prod-deps
|
||||
RUN npm prune --omit=dev
|
||||
|
||||
FROM base AS runtime
|
||||
ENV NODE_ENV=production
|
||||
COPY package.json ./
|
||||
COPY --from=prod-deps /app/node_modules ./node_modules
|
||||
COPY --from=build /app/dist ./dist
|
||||
COPY contracts/openapi.json ./contracts/openapi.json
|
||||
|
||||
USER node
|
||||
EXPOSE 8080
|
||||
|
||||
HEALTHCHECK --interval=5s --timeout=2s --start-period=5s --retries=3 \
|
||||
CMD node -e "require('http').get('http://127.0.0.1:8080/health/core',r=>process.exit(r.statusCode===200?0:1)).on('error',()=>process.exit(1))"
|
||||
|
||||
CMD ["node", "dist/main.js"]
|
||||
@@ -1,37 +1,212 @@
|
||||
# Отчёт — лабораторная 6
|
||||
|
||||
- Автор, группа: TODO
|
||||
- Целевая оценка; выбранный трек 5, если нужен: TODO
|
||||
- Commit/PR: TODO после публикации
|
||||
- Использованные источники, библиотеки и помощь ИИ: TODO
|
||||
- Автор, группа: Антонов Егор, БПИ2301
|
||||
- Целевая оценка; выбранный трек 5, если нужен: **4**, трек 5 не заявляется
|
||||
- Commit/PR: ветка `solution/БПИ2301-Antonov`; ссылка на PR — после публикации
|
||||
- Использованные источники, библиотеки и помощь ИИ: документация OpenTelemetry (JS SDK 1.30/0.57, semantic conventions HTTP/messaging/DB, W3C Trace Context, Collector configuration: `memory_limiter`, `batch`, `sending_queue`), Prometheus (OTLP receiver, histograms and summaries, recording/alerting rules), Loki 3 (OTLP ingestion, structured metadata), Tempo 2.6, Grafana provisioning, Prisma (tracing, `metrics` preview). Новые библиотеки: `@opentelemetry/*` (api 1.9.0, sdk-node 0.57.2, sdk-metrics/trace 1.30.1, OTLP proto exporters 0.57.2, instrumentation-http 0.57.2, instrumentation-aws-sdk 0.49.1), `@prisma/instrumentation` 6.1.0. Перед началом работы выгружены и учтены замечания проверяющих к чужим PR этой лабораторной (см. «Сверка с замечаниями проверяющих»); код из чужих решений не использовался. ИИ-ассистент использовался как парный программист: код, конфигурации, сценарии инцидентов и текст отчёта по моим решениям с последующим ревью; проверки воспроизведены на своём стенде.
|
||||
|
||||
## Запуск из чистого clone
|
||||
|
||||
TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных.
|
||||
На хосте нужен **только Docker** (Engine + Compose v2); python/node/make/sh на хосте не требуются, команды одинаковы в PowerShell и bash. Проверено: Docker 29.3.1, Docker Desktop (WSL2), 12 vCPU, 8 GiB RAM.
|
||||
|
||||
| Компонент | Версия |
|
||||
| --- | --- |
|
||||
| API, worker, relay (один образ) | `node:20.17.0-slim`, NestJS 10 / Fastify, Prisma 6.1.0, OpenTelemetry JS SDK 1.30.1 / 0.57.2 |
|
||||
| OTel Collector | `otel/opentelemetry-collector-contrib:0.115.1`, закреплён digest |
|
||||
| Метрики | `prom/prometheus:v3.1.0`, закреплён digest |
|
||||
| Логи | `grafana/loki:3.3.2`, закреплён digest |
|
||||
| Трассы | `grafana/tempo:2.6.1`, закреплён digest |
|
||||
| Интерфейс | `grafana/grafana:11.4.0`, закреплён digest |
|
||||
| Остальное (lab5) | RabbitMQ 4.0.5, PostgreSQL 17.2, Redis 7.4.1, HAProxy 3.0.6, MinIO (`pgsty/minio`), tools `docker:27.3.1-cli` |
|
||||
|
||||
```sh
|
||||
# 1. Запуск всего стенда, включая Collector, Prometheus, Loki, Tempo и Grafana
|
||||
docker compose up -d --build --wait
|
||||
|
||||
# 2. Публичные проверки (телеметрия включена)
|
||||
docker compose run --rm --build tools make check
|
||||
docker compose run --rm tools make test
|
||||
|
||||
# 3. Наблюдаемость
|
||||
docker compose run --rm tools python3 scripts/trace-check.py # контрольный upload: trace в Tempo + логи в Loki
|
||||
docker compose run --rm tools python3 scripts/telemetry-privacy.py # секреты/контент не в telemetry, нет ID в labels
|
||||
docker compose run --rm tools python3 scripts/latency-compare.py # p50/p95/p99 dashboard против клиента
|
||||
docker compose run --rm tools python3 scripts/dashboard-check.py # каждый запрос dashboard возвращает данные
|
||||
|
||||
# 4. Инциденты (каждый ждёт firing → resolved своего alert)
|
||||
docker compose run --rm tools python3 scripts/incidents.py retry-trace # одна трасса через все попытки job
|
||||
docker compose run --rm tools python3 scripts/incidents.py worker-stop # инцидент 1
|
||||
docker compose run --rm tools python3 scripts/incidents.py db # инцидент 2: медленная и недоступная БД
|
||||
docker compose run --rm tools python3 scripts/incidents.py telemetry # инцидент 3: недоступен telemetry backend
|
||||
|
||||
# 5. Все доказательства в evidence/ одной командой (~50 минут)
|
||||
docker compose run --rm tools sh scripts/collect-evidence.sh
|
||||
|
||||
# Безопасная остановка (данные, метрики, логи, трассы, секреты сохраняются в volumes)
|
||||
docker compose stop
|
||||
# Намеренное удаление учебных данных, телеметрии и сгенерированных секретов
|
||||
docker compose down -v
|
||||
```
|
||||
|
||||
- **Grafana:** http://localhost:3000, анонимный вход с ролью Viewer (просмотр и Explore). Dashboard «Messenger overview» в папке Messenger создаётся provisioning'ом. Пароль `admin` генерирует `secrets-init`, посмотреть: `docker compose exec grafana cat /run/secrets/admin_password`.
|
||||
- **Наружу опубликованы только** LB `127.0.0.1:8080` и Grafana `127.0.0.1:3000`. OTLP (4318), Prometheus, Loki, Tempo наружу не открыты; хранилища находятся во внутренней сети `telemetry` (`internal: true`), куда из сети приложения подключены только Collector и Grafana.
|
||||
- `.env` необязателен; секреты — как в lab5 (`secrets-init`, раздельно по сервисам), добавлен `grafana/admin_password`.
|
||||
|
||||
## Архитектура и решения
|
||||
|
||||
TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab5, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации.
|
||||
```
|
||||
api ×2 ─┐ traces, metrics, logs (OTLP/HTTP protobuf, :4318)
|
||||
relay ──┼──────────────▶ otel-collector ── memory_limiter → batch → exporters (ограниченная sending_queue, retry 60 s)
|
||||
worker ─┘ default net │ telemetry net (internal)
|
||||
├─ metrics ─▶ prometheus (/api/v1/otlp, TSDB 7d/1GB, rules + alerts) ◀─ scrape collector :8888
|
||||
├─ logs ────▶ loki (/otlp, filesystem, 7d, trace_id/span_id = structured metadata)
|
||||
└─ traces ──▶ tempo (OTLP, local blocks, 7d)
|
||||
grafana (127.0.0.1:3000) ── datasources uid prometheus / loki / tempo, dashboard и связи trace ↔ logs из provisioning
|
||||
```
|
||||
|
||||
**Зачем каждое хранилище:**
|
||||
|
||||
| Сигнал | Хранилище | На какой вопрос отвечает |
|
||||
| --- | --- | --- |
|
||||
| Метрики | Prometheus | «Сколько и как быстро?» — агрегаты во времени: RPS, доля ошибок, перцентили из histogram, глубина очереди, насыщение. Дёшево хранить долго, на них строятся SLO и alerts. Не отвечает, *какой* запрос был медленным. |
|
||||
| Логи | Loki | «Что именно произошло?» — отдельные события с контекстом (`event`, `route`, `status`, `request_id`, `job_id`, причина ошибки). Индексируются только метки потока (`service_name`), остальное — structured metadata и текст. |
|
||||
| Трассы | Tempo | «Где было потрачено время и как связаны шаги?» — дерево spans одного запроса через API, relay, брокер, worker, S3 и БД. Хранит отдельные трассы по ID, без индекса по содержимому. |
|
||||
|
||||
Collector — получатель и маршрутизатор, а не БД: он буферизует ограниченную очередь и при длительной недоступности хранилища отбрасывает данные. Grafana — только интерфейс и запросы к трём источникам.
|
||||
|
||||
**Инструментация** (`src/telemetry/`):
|
||||
- `sdk.ts` — `NodeSDK` запускается первым импортом в `main.ts`, `worker.ts`, `relay.ts` (до загрузки `http`, Prisma, AWS SDK). Resource: `service.name` (`messenger-api`, `messenger-worker`, `messenger-relay` — relay вынесен в отдельный сервис, поэтому третье имя), `service.version` (из `package.json`), `service.instance.id` (hostname контейнера), `deployment.environment.name=lab` (из `OTEL_RESOURCE_ATTRIBUTES`). Автоинструментация: HTTP server (входящий `traceparent` продолжается, span kind SERVER), Prisma (`prisma:client:operation`, `prisma:engine:connection` — ожидание пула, `prisma:engine:db_query` — SQL без значений параметров), AWS SDK (`S3.PutObject` и т. д.). Исходящие HTTP-spans отключены (дублируют S3), health-запросы не трассируются, из `http.target`/`http.url` удаляется query (там `filename`). Sampler: корневые spans Prisma (опрос outbox relay, `SELECT 1` health) и служебные листовые spans сериализации не записываются.
|
||||
- `json-logger.ts` — замена Nest `Logger` во всех трёх процессах: каждая запись — JSON с `timestamp`, `severity`, `service.name`, `event`, `logger`, полями события, `trace_id`, `span_id` (только у записываемого span), в stdout и как OTLP log record. Ключевые события: `http.request` (method, route, status, duration_ms, request_id), `http.error` (5xx, причина), `job.handled` (job_id, attachment_id, attempt, outcome, duration_ms).
|
||||
- `http.ts` — метрики HTTP в hooks Fastify с шаблоном маршрута (`/api/v1/chats/{chat_id}/messages`); имя SERVER span = `METHOD route`.
|
||||
- `db-pool.ts` — пул Prisma через `$metrics` (preview `metrics`), `relay.ts` — очередь, `worker.ts` — задания и насыщение, `processor.ts` — span рендеринга и время до ready.
|
||||
|
||||
**Метрики** (имена в Prometheus):
|
||||
|
||||
| Сигнал контракта | Метрика | Labels | Тип |
|
||||
| --- | --- | --- | --- |
|
||||
| HTTP requests | `http_server_requests_total` | `http_request_method`, `http_route` (шаблон), `http_response_status_class` | counter |
|
||||
| HTTP duration | `http_server_request_duration_seconds` | те же | histogram, s; buckets 1 ms … 10 s |
|
||||
| Jobs processed/failed | `jobs_processed_total` | `job_type`, `outcome` (`success`/`retry`/`failed`/`skipped`/`malformed`/`interrupted`) | counter |
|
||||
| Job duration | `jobs_duration_seconds` | `job_type`, `outcome` | histogram, s; 10 ms … 120 s |
|
||||
| Queue depth / oldest age | `jobs_queue_depth`, `jobs_queue_oldest_age_seconds` | `messaging_destination_name`, `state` (`queued`/`due`/`processing`) | gauge |
|
||||
| DB pool | `db_client_connection_count`, `db_client_connection_pending_requests`, `db_client_connection_max` | `db_client_connection_state` (`used`/`idle`) | gauge |
|
||||
| Worker busy/capacity | `worker_jobs_busy`, `worker_jobs_capacity` | — | gauge |
|
||||
| CPU/RAM процесса | `process_cpu_time_seconds_total`, `process_memory_usage_bytes`, `process_memory_heap_used_bytes` | `cpu_mode` | counter, gauge |
|
||||
| Time to ready | `attachments_time_to_ready_seconds` | `job_type` | histogram, s; 0,1 … 600 s |
|
||||
|
||||
Ресурсные атрибуты становятся `job` (= `service.name`) и `instance` (= `service.instance.id`), остальные — в `target_info`. В labels нет `user_id`, `chat_id`, `message_id`, `attachment_id`, `request_id`, raw URL, filename, текста: маршрут — шаблон, метод вне списка → `_OTHER`, несопоставленный путь → `unmatched`. Это проверено по всем сериям (`telemetry-privacy.py`).
|
||||
|
||||
**Почему очередь считается по БД, а не по брокеру.** Источник истины — `attachment_jobs` (lab5). Gauge отдаёт relay (а не worker), поэтому при остановленном worker метрика продолжает расти. `oldest_age` — возраст старейшего `queued`-задания, срок которого наступил (`now − next_attempt_at`): для нового задания это время с момента принятия upload, для повтора — с момента, когда повтор стал разрешён (ожидание backoff не считается отставанием).
|
||||
|
||||
**Buckets.** Длительности в секундах, границы рассчитаны на миллисекундные запросы (1, 2,5, 5, 10, 25, 50, 100, 250, 500 ms, 1, 2, 2,5, 5, 10 s) и на пороги SLO (0,5 s; 10 s для готовности). Перцентили считаются `histogram_quantile` по **сумме buckets всех реплик** (`sum by (le)`), а не усреднением перцентилей реплик. Сверка с клиентом — в разделе экспериментов.
|
||||
|
||||
**Сквозная трасса через очередь (W3C Trace Context).**
|
||||
1. API продолжает входящий `traceparent` клиента (SERVER span).
|
||||
2. В транзакции upload `jobMessage()` записывает в payload outbox поля `traceparent`/`tracestate` активного контекста — они уже есть в `jobs.schema.json`. Публикация отложенная, поэтому контекст должен пережить outbox: обычная автоинструментация amqplib связала бы сообщение с циклом опроса relay, а не с upload.
|
||||
3. Relay для каждого сообщения создаёт span `attachment.jobs publish` (PRODUCER), **родитель — контекст из payload**, и кладёт новый `traceparent` в AMQP headers.
|
||||
4. Worker создаёт `attachment.jobs process` (CONSUMER) с родителем из headers (запасной вариант — из payload); внутри — `S3.GetObject`, `attachment.render_variants`, `S3.PutObject` ×4, транзакции Prisma, `S3.DeleteObject`.
|
||||
5. Повтор: `scheduleRetry` вызывается внутри CONSUMER span попытки N, новая запись outbox несёт его контекст, поэтому попытка N+1 — потомок попытки N в той же трассе. Выбрана семантика parent-child (одна логическая операция), а не links.
|
||||
|
||||
Отображение полей: payload `traceparent`/`tracestate` ↔ заголовки AMQP `traceparent`/`tracestate`; `messaging.message.id` = `job_id`, `messaging.job.attempt` = `attempt`. Parser сообщений проверяет формат `traceparent` (регулярное выражение схемы) и длину `tracestate` ≤ 512.
|
||||
|
||||
**Collector и поведение при backpressure** (`infra/otel/collector.yaml`): `memory_limiter` 160 MiB (+40 spike, контейнер 256 MiB) — при превышении receiver отказывает (`otelcol_receiver_refused_*`); `batch` 512/1024, 2 s; у каждого exporter `sending_queue` 200 batch в памяти, `retry_on_failure` до 60 s, затем данные отбрасываются (`otelcol_exporter_send_failed_*`). В SDK: batch processors с очередью 2048 записей и таймаутом экспорта 5 s, метрики раз в 10 s. Экспорт асинхронный: запросы пользователей не ждут телеметрию.
|
||||
|
||||
**SLI/SLO и alerts** — `infra/prometheus/rules/messenger.yml`, runbooks — [docs/runbooks.md](docs/runbooks.md).
|
||||
|
||||
| SLI | SLO | Окно |
|
||||
| --- | --- | --- |
|
||||
| Доступность: 1 − 5xx / бизнес-запросы (без `/health/*`) | ≥ 99% | 1 ч |
|
||||
| Задержка: доля бизнес-запросов ≤ 0,5 s | ≥ 95% | 1 ч |
|
||||
| Готовность: доля вложений `ready` за ≤ 10 s от принятия | ≥ 95% | 1 ч |
|
||||
|
||||
| Alert | Условие | `for` |
|
||||
| --- | --- | --- |
|
||||
| `MessengerHighErrorRate` | доля 5xx бизнес-запросов за 5m > 5% и трафик > 0,05 rps | 1m |
|
||||
| `MessengerApiNotReady` | > 10 ответов 5xx `/health/core` за 1m (реплики выведены из LB) | 30s |
|
||||
| `MessengerJobQueueLag` | старейшее готовое задание ждёт > 60 s | 1m |
|
||||
| `MessengerSlowApi` | p95 бизнес-запросов за 5m > 1 s и трафик > 0,05 rps | 2m |
|
||||
| `TelemetryExportFailing` | есть отказы отправки Collector или очередь exporter > 50% | 1m |
|
||||
|
||||
**Dashboard** (`infra/grafana/dashboards/messenger.json`, uid `messenger-overview`): SLO и история alerts; API (RPS по маршрутам, классы статуса, доля 5xx, p50/p95/p99, p95 по маршрутам); очередь и worker (глубина, возраст, busy/capacity/saturation, задания по исходу, длительность, время до ready); БД и ресурсы (пул used/waiting/max по сервисам, CPU, RSS); конвейер телеметрии (принято/отправлено/ошибки, очередь exporter); логи и таблица последних трасс. Datasources с явными UID `prometheus`, `loki`, `tempo` — те же UID в панелях; Tempo → Loki по `trace_id`, Loki → Tempo по полю `trace_id`.
|
||||
|
||||
**Перенесено из lab5** без изменения поведения: весь API, outbox, relay, worker, lease, retry, DLQ, replay, reconciler, тесты. **Изменено:** логирование стало JSON (лог HTTP-ошибки содержит шаблон маршрута вместо URL с query), сообщение задания может нести `traceparent`/`tracestate`, `PrismaService` регистрирует пул в метриках, preview `metrics` в `schema.prisma` (миграций нет).
|
||||
|
||||
**Соответствие `.env.example`:**
|
||||
|
||||
| Переменная | Где используется |
|
||||
| --- | --- |
|
||||
| переменные lab5 | как в lab5 |
|
||||
| `OTEL_SERVICE_NAME` | задаётся для каждого сервиса в `compose.yaml`: `messenger-api`, `messenger-worker`, `messenger-relay` |
|
||||
| `OTEL_EXPORTER_OTLP_ENDPOINT` | `http://otel-collector:4318`, общий якорь `x-otel` в `compose.yaml`; без него SDK не запускается (собственные тесты) |
|
||||
| `OTEL_EXPORTER_OTLP_PROTOCOL` | `http/protobuf` (используются `*-otlp-proto` exporters) |
|
||||
| `OTEL_RESOURCE_ATTRIBUTES` | `deployment.environment.name=lab` |
|
||||
|
||||
## Доказательства критериев
|
||||
|
||||
Файлы собраны `docker compose run --rm tools sh scripts/collect-evidence.sh` на commit `d83876f` с чистых volumes (`docker compose down -v`, затем `up`); в заголовке каждого файла — дата и commit. `incident-telemetry.txt` пересобран на commit `2718c3b`: фаза «Collector остановлен» удлинена с 60 до 300 s, код приложения не менялся.
|
||||
|
||||
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
|
||||
| --- | --- | --- | --- |
|
||||
| Публичный smoke | make test | TODO | TODO |
|
||||
| Остальные пункты уровня 3 | TODO | TODO | TODO |
|
||||
| Все пункты уровня 4, если заявлен | TODO | TODO | TODO |
|
||||
| Выбранный трек 5, если заявлен | TODO | TODO | TODO |
|
||||
|
||||
TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены.
|
||||
| Публичный smoke с телеметрией | `make check`, `make test` через LB | `make check` PASS; `PASS: lab6 public API scenario` при включённой телеметрии; запросы одной сессии обслуживают обе реплики | [evidence/smoke.txt](evidence/smoke.txt) |
|
||||
| **3**: минимум lab5 сохранён | собственные тесты, `make test-persistence`, `make test-failover`, `check-s3.sh`, сценарии `job-scenario.py` | 201/201 тест; persistence PASS; failover PASS (128 записей, 0 ошибок, восстановление 3,57 s); S3 PASS; все 7 сценариев lab5 (crash-after-store/commit, outbox-gap, relay-crash, retry-replay, broker) воспроизводятся с телеметрией | [evidence/tests.txt](evidence/tests.txt), [evidence/persistence.txt](evidence/persistence.txt), [evidence/failover.txt](evidence/failover.txt), [evidence/s3.txt](evidence/s3.txt), [evidence/jobs.txt](evidence/jobs.txt) |
|
||||
| **3**: Collector, хранилища трёх сигналов, Grafana в стенде; конфиги и provisioning в Git; volumes | `docker compose ps`, images, сети, datasources | сервисы `otel-collector`, `prometheus`, `loki`, `tempo`, `grafana` работают, образы закреплены по digest; наружу только `127.0.0.1:8080` и `127.0.0.1:3000`; хранилища только в сети `telemetry` (`internal=true`); volumes `prometheusdata`, `lokidata`, `tempodata`, `grafanadata`; datasources с UID `prometheus`/`loki`/`tempo` | [evidence/container.txt](evidence/container.txt), [evidence/telemetry.txt](evidence/telemetry.txt) |
|
||||
| **3**: API и worker отдают OTLP; RPS, ошибки, длительность API, jobs и длительность worker, логи, трассы видны в Grafana | `dashboard-check.py` | dashboard загружен из provisioning (папка Messenger); все запросы панелей возвращают данные, кроме истории alerts (в спокойном состоянии alerts нет); «время до ready» за последнюю минуту — NaN, т. к. новых готовых вложений в эту минуту не было | [evidence/telemetry.txt](evidence/telemetry.txt) |
|
||||
| **3**: логи с trace_id/span_id; для контрольного запроса находятся trace и лог, для job — trace и лог worker | `trace-check.py` | upload с клиентским `traceparent` → trace из 79 spans: SERVER API (родитель — span клиента) → PRODUCER relay → CONSUMER worker → S3 и SQL; в Loki по `trace_id` — `http.request` API и `job.handled` worker, у каждого свой `span_id` | [evidence/telemetry.txt](evidence/telemetry.txt) |
|
||||
| **3**: пароли, токены, тексты сообщений не экспортируются; нет ID в labels | `telemetry-privacy.py` | пароль, session token, Basic-учётка, заголовок Authorization, текст сообщения, filename — 0 вхождений в 6 трассах этих запросов и в Loki (контроль: 7 строк логов этих запросов найдены); 893 серии, 0 с UUID в значениях labels; `http_route` — только шаблоны | [evidence/telemetry.txt](evidence/telemetry.txt) |
|
||||
| **4**: W3C context API → broker → worker, producer/consumer spans, цепочка до S3/DB включая retry | `incidents.py retry-trace`, `trace-check.py` | одна загрузка — одна трасса: 3 пары PRODUCER → CONSUMER (`attempt` 1, 2, 3); в попытках 1–2 `S3.GetObject` с ошибкой (S3 остановлен), попытка 3 — S3 Put ×4, COMMIT, ready; события `enqueued → processing → retry_scheduled → processing → retry_scheduled → processing → ready` | [evidence/incident-retry-trace.txt](evidence/incident-retry-trace.txt) |
|
||||
| **4**: p50/p95/p99 из histogram, доля ошибок, очередь и возраст, пул БД, насыщение worker, CPU/RAM, time to ready | `dashboard-check.py`, `latency-compare.py` | панели p50/p95/p99 (`sum by (le)`), доля 5xx, глубина/возраст очереди, пул used/waiting/max, busy/capacity, CPU, RSS, time to ready возвращают данные; клиент / histogram: p50 18,3 / 19,7 ms, p95 89,2 / 92,2 ms, p99 163,3 / 176,8 ms (3411 POST, 113 rps) | [evidence/telemetry.txt](evidence/telemetry.txt) |
|
||||
| **4**: SLI/SLO с окном; alert rules на error rate и queue lag, firing → resolved | `promtool`, `incidents.py worker-stop`, `db` | `promtool`: 13 правил; все `health ok`; `MessengerJobQueueLag` и `MessengerApiNotReady` прошли pending → firing → inactive | [evidence/telemetry.txt](evidence/telemetry.txt), [evidence/incident-worker-stop.txt](evidence/incident-worker-stop.txt), [evidence/incident-db.txt](evidence/incident-db.txt) |
|
||||
| **4**: Collector с memory_limiter, batch, ограниченной очередью; остановка backend не ломает API и не растит RAM | `incidents.py telemetry` | Loki+Tempo остановлены: 0 ошибок API, очередь exporter упирается в 200, `memory_limiter` отказывает в приёме, `TelemetryExportFailing` firing → resolved; Collector остановлен 300 s: 0 ошибок API, память API ограничена и освобождается после восстановления; перезапусков/OOM нет | [evidence/incident-telemetry.txt](evidence/incident-telemetry.txt) |
|
||||
| **4**: инцидент 1 — worker остановлен | `incidents.py worker-stop` | см. «Эксперименты и отказы» | [evidence/incident-worker-stop.txt](evidence/incident-worker-stop.txt) |
|
||||
| **4**: инцидент 2 — медленная/недоступная БД отличима от медленного handler | `incidents.py db` | см. «Эксперименты и отказы» | [evidence/incident-db.txt](evidence/incident-db.txt) |
|
||||
| Собственные тесты | `docker compose -f compose.test.yaml run --rm --build tests` | `Tests: 201 passed, 201 total` | [evidence/tests.txt](evidence/tests.txt) |
|
||||
|
||||
## Эксперименты и отказы
|
||||
|
||||
TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения.
|
||||
Стенд: Docker Desktop WSL2, 12 vCPU, 8 GiB; лимиты контейнеров — в `compose.yaml` и `evidence/container.txt` (API 0,5 CPU / 256M ×2, worker 1 CPU / 768M, Collector 0,5 CPU / 256M, хранилища по 0,5 CPU / 512M). Каждый сценарий выполнен один раз в составе `collect-evidence.sh`. Фоновая нагрузка в инцидентах — `Traffic` из `scripts/incidents.py`: 4–6 потоков closed loop через LB, POST сообщения и GET истории по очереди, пауза 0,05–0,2 s; файл для job — `sample.png` 320×200. Клиентские p50/p95/p99 считаются по успешным ответам фазы.
|
||||
|
||||
**Сверка перцентилей** (`latency-compare.py`, 30 s, 4 потока, POST сообщений): клиент 3411 успешных запросов, 113 rps, 0 ошибок; p50 / p95 / p99 — 18,3 / 89,2 / 163,3 ms у клиента и 19,7 / 92,2 / 176,8 ms по `histogram_quantile` (`increase` за окно). Серверная гистограмма не включает сеть и клиент, значение интерполируется внутри bucket (50–100 ms и 100–250 ms), поэтому расхождение в пределах ширины bucket. Счётчик Prometheus за окно (3661) больше клиентского: `increase()` экстраполирует до границ окна.
|
||||
|
||||
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
|
||||
| --- | --- | --- | --- |
|
||||
| TODO | TODO | TODO | TODO |
|
||||
| **Инцидент 1:** `docker compose stop worker`, 12 загрузок по одной в 10 s | 202 продолжаются; растёт возраст старейшего задания; alert по lag; после старта backlog уменьшается | все загрузки 202; `jobs_queue_depth{state="due"}` до 12, `oldest_age` до 132,5 s (обновляется с шагом экспорта метрик), `worker_busy` 0, серии worker перестают обновляться; `MessengerJobQueueLag` pending → **firing через 151 s** простоя | после `start worker` очередь пуста через ≤ 20 s (12 → 2 → 0), alert **resolved через 30 s**; 12 из 12 `ready`; фактическое время до ready по БД 41,9–152,1 s, histogram p99 176 s (bucket 150–180 s); потерь нет |
|
||||
| **Инцидент 2а — медленный handler (контроль):** upload PNG 1700×1700 шума, 8,3 MiB | время в обработчике и S3, БД быстрая | trace 525 ms: вызовы БД 40 ms (SQL 22 ms, ожидание пула 1 ms), S3 246 ms, сам обработчик (полное декодирование) 240 ms | — |
|
||||
| **Инцидент 2б — медленная БД:** `docker update --cpus 0.05 postgres` на 150 s под нагрузкой | рост задержки, причина в SQL | 0 ошибок; клиент p95 23 → 196 ms, p99 49 → 637 ms; p95 в Prometheus (1m) до ~250 ms; trace 790 ms: БД 788 ms (SQL 781 ms, пул 0 ms), обработчик 1 ms; trace 505 ms: SQL 498 ms. `MessengerSlowApi` (порог p95 > 1 s) не сработал — деградация ниже порога | после `--cpus 1.0` p95 вернулся к ~24 ms за 1–2 мин |
|
||||
| **Инцидент 2в — БД недоступна:** `docker pause postgres` на ~105 s | 503 вместо зависаний; alert | API: ожидание пула 5 s → `503 database unavailable` (trace 5005 ms: `prisma:engine:connection` 5000 ms); `/health/core` 503 → HAProxy выводит обе реплики, клиенты получают `503 no API replica available` (1910 ответов) и 3 × 504; доля 5xx в метриках API не растёт — запросы не доходят до API; `MessengerApiNotReady` **firing**; в Loki `http.error` / `database unavailable` | после `unpause` реплики вернулись в LB за секунды, 25 ответов 503 в хвосте; alert **resolved через ~60 s**; p95 23 ms; данных не потеряно (неподтверждённые записи получили 503) |
|
||||
| **Инцидент 3а — Loki и Tempo остановлены** до срабатывания alert (~150 s) под нагрузкой 6 потоков | API работает; Collector ограничивает очередь и память; потери считаются | клиент: 13 901 успешный, **0 ошибок**, p95 43 ms (до — 53 ms); очередь exporter 0 → **200 из 200**; `send_failed` ~1350 spans и ~1100 log records в минуту; затем `memory_limiter` отказывает в приёме (`refused` до ~89 000 в минуту; в логе Collector: «above soft limit, refusing data», «above hard limit, forcing GC» при 120–170 MiB кучи); `TelemetryExportFailing` **firing**. Перезапусков и OOM Collector нет | после `start loki tempo` очередь опустела за ~15 s, alert **resolved** через ~120 s; трассы и логи за время простоя потеряны (посчитаны в `send_failed`/`refused`), логи остаются в `docker compose logs` |
|
||||
| **Инцидент 3б — Collector остановлен на 300 s** под той же нагрузкой | API работает; экспорт падает быстро (таймаут 5 s), очереди SDK ограничены | клиент: 30 904 успешных, **0 ошибок**, p95 38 ms; память контейнеров API (`docker stats`) 183 → 211 MiB и 174 → 197 MiB из 256M: +9 MiB за первые 45 s, затем ~3 MiB/мин; worker стабильно ~57 MiB; перезапусков и OOM нет | после `start otel-collector` экспорт возобновился без перезапуска API; через ~1 мин память реплик 159 и 149 MiB — буферы телеметрии освобождены. Отдельный предварительный замер (5 мин, `load.py` 6 потоков): RSS до ~228 MiB и сброс сборщиком мусора до ~185 MiB на 280-й секунде, 38 141 запрос без ошибок |
|
||||
| S3 остановлен на время первых попыток job (`retry-trace`) | повторы в одной трассе | см. критерий W3C выше: 3 попытки в одной трассе | ready через ~9 s после upload |
|
||||
| `restart.sh`, `stop-one.sh`, сценарии lab5 | как в lab5 | PASS; failover 128 записей, 0 ошибок, восстановление 3,57 s | потерь нет |
|
||||
|
||||
## Сверка с замечаниями проверяющих
|
||||
|
||||
Замечания к PR других студентов этой лабораторной выгружены **до начала работы** (2026-11-18, 42 PR, два содержательных отзыва) и учтены при проектировании. Код из чужих решений не использовался.
|
||||
|
||||
| Замечание к чужому PR | Наш результат |
|
||||
| --- | --- |
|
||||
| Стандартные buckets (0, 5, 10 … s) дают p95 4,75 s при реальных 5 ms; сверить график с контрольным измерением | buckets от 1 ms; `latency-compare.py` сравнивает с клиентом — p95 89 ms у клиента и 92 ms по histogram. Та же проблема найдена у нас для time to ready (p99 296 s при реальных ≤ 160 s между границами 120 и 300 s) и исправлена добавлением границ 150/180/240 s |
|
||||
| Datasources без UID → rules и панели с UID `prometheus`/`loki` в состоянии `data source not found` | UID заданы в provisioning и используются во всех панелях; `telemetry.txt` показывает UID, все запросы dashboard возвращают данные. Alerts вычисляет Prometheus, а не Grafana, health правил `ok` |
|
||||
| Alert «остановка worker» по числу retry не срабатывает, т. к. без worker retry не появляются; нужны queue depth и oldest age | alert по возрасту старейшего задания из БД, gauge отдаёт relay, поэтому он растёт при мёртвом worker; firing → resolved показан |
|
||||
| API создаёт новый trace вместо продолжения клиентского `traceparent`, нет SERVER spans | SERVER span — потомок клиентского span (`trace-check.py`: `parent=client`) |
|
||||
| В worker виден только общий span, нет S3/DB | `S3.GetObject`/`PutObject`/`DeleteObject`, `prisma:*`, `attachment.render_variants` — потомки CONSUMER span |
|
||||
| Нет DB pool, worker busy/capacity, histogram времени до ready | все три есть и на dashboard |
|
||||
|
||||
## Ограничения и следующие шаги
|
||||
|
||||
TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза.
|
||||
- **Окна SLO по 1 ч** — учебный масштаб, чтобы эффект был виден за время защиты; в эксплуатации — 28–30 дней и multi-window burn-rate alerts.
|
||||
- **Перцентили — оценка внутри bucket**, точность ограничена шириной bucket; события медленнее 10 s (HTTP) и 600 s (ready) попадают в `+Inf`.
|
||||
- **`MessengerHighErrorRate` не видит ответы LB**, когда все реплики выведены (HAProxy отвечает 503 сам) — это закрыто `MessengerApiNotReady`; метрик HAProxy в стенде нет.
|
||||
- **Запас памяти Collector небольшой.** `memory_limiter` ограничивает кучу Go (порог 160 MiB, отказ в приёме с 120 MiB), но RSS процесса больше кучи: метрика `otelcol_process_memory_rss` при остановленных Loki/Tempo показывала до 287 MiB (счёт процесса, с разделяемыми страницами), лимит контейнера 256M при этом не превышен, OOM не было. При большей нагрузке стоит снизить `limit_mib` или поднять лимит контейнера.
|
||||
- **Память API без Collector** за 5 минут выросла на ~25 MiB с замедлением и освободилась после восстановления; более длинный простой не измерялся. Ограничители — очереди SDK по 2048 записей, таймаут экспорта 5 s, лимит контейнера 256M.
|
||||
- **Потери телеметрии при недоступности backend допустимы:** после 60 s повторов и заполнения очереди Collector отбрасывает данные; логи при этом остаются в stdout контейнеров. Персистентной очереди (file storage) у Collector нет.
|
||||
- **Пропуски метрик при остановке процесса** (worker stop): серии worker перестают обновляться — это видно, но отдельного alert на отсутствие серий нет; признак остановки — рост `oldest_age` при `busy = 0`.
|
||||
- **Метрики очереди зависят от relay:** если упал relay, исчезают и они (но не задания — они в БД).
|
||||
- **Один экземпляр** Collector и каждого хранилища, без репликации; TLS между компонентами — lab7.
|
||||
- Уведомления (Alertmanager, почта) не настроены — по заданию не требуются; состояние alerts видно в Prometheus API и на dashboard.
|
||||
|
||||
## Ответы на вопросы защиты
|
||||
|
||||
- **Лог, метрика и trace — на какие вопросы отвечает каждый?** Метрика: «сколько, как часто, насколько медленно в целом» — дешёвые агрегаты для dashboard, SLO и alerts. Лог: «что именно случилось в этом событии» — причина ошибки, ID запроса/задания. Trace: «где ушло время и через какие компоненты прошёл этот конкретный запрос», включая асинхронное продолжение в worker.
|
||||
- **Почему `chat_id` в labels может исчерпать хранилище?** Каждое уникальное сочетание labels — отдельная временная серия со своим индексом и чанками; для histogram это ещё ×(число buckets + 2). Число чатов растёт неограниченно, значит число серий тоже (cardinality explosion): растут память и диск Prometheus, замедляются запросы. ID место в логах и трассах.
|
||||
- **Как очередь переносит причинную связь?** Контекст W3C (`traceparent` = версия, trace-id, parent span-id, флаги) кладётся в сообщение: у нас в payload outbox (переживает отложенную публикацию) и в AMQP headers. Producer span — потомок контекста upload, consumer span — потомок producer; повтор несёт контекст предыдущей попытки. Это parent-child; при batch-обработке нескольких сообщений правильнее links.
|
||||
- **Почему среднее не заменяет p99 и почему нельзя усреднять p99 реплик?** Среднее скрывает хвост: 99 быстрых и 1 очень медленный запрос дают «хорошее» среднее, а пользователь из хвоста ждёт долго. Перцентиль — не аддитивная величина: среднее p99 двух реплик не равно p99 объединённого трафика (реплика с малым трафиком и плохим хвостом искажает результат). Правильно — суммировать buckets histogram всех реплик (`sum by (le)`) и считать `histogram_quantile` по сумме.
|
||||
|
||||
@@ -0,0 +1,55 @@
|
||||
name: messenger-lab6-test
|
||||
|
||||
services:
|
||||
postgres:
|
||||
image: postgres:17.2-alpine
|
||||
environment:
|
||||
POSTGRES_USER: test
|
||||
POSTGRES_PASSWORD: test
|
||||
POSTGRES_DB: messenger_test
|
||||
tmpfs:
|
||||
- /var/lib/postgresql/data
|
||||
healthcheck:
|
||||
test: ["CMD", "pg_isready", "-U", "test", "-d", "messenger_test"]
|
||||
interval: 2s
|
||||
timeout: 3s
|
||||
retries: 15
|
||||
|
||||
redis:
|
||||
image: redis:7.4.1-alpine
|
||||
command: ["redis-server", "--save", "", "--appendonly", "no"]
|
||||
|
||||
s3:
|
||||
image: pgsty/minio:RELEASE.2026-08-04T00-00-00Z@sha256:b6bfe7239bfc83fb90d31612d9704d86039dd714f7904b3f1ad68f211e602372
|
||||
command: ["server", "/data", "--address", ":9000"]
|
||||
environment:
|
||||
MINIO_ROOT_USER: test-root
|
||||
MINIO_ROOT_PASSWORD: test-root-password
|
||||
MINIO_BROWSER: "off"
|
||||
tmpfs:
|
||||
- /data
|
||||
healthcheck:
|
||||
test: ["CMD", "curl", "-fsS", "-o", "/dev/null", "http://127.0.0.1:9000/minio/health/live"]
|
||||
interval: 2s
|
||||
timeout: 3s
|
||||
retries: 15
|
||||
|
||||
tests:
|
||||
build:
|
||||
context: .
|
||||
target: test
|
||||
image: mtusi-messenger-lab6-test:dev
|
||||
environment:
|
||||
DATABASE_URL: postgresql://test:test@postgres:5432/messenger_test?connection_limit=20
|
||||
CACHE_URL: redis://redis:6379/0
|
||||
S3_ENDPOINT: http://s3:9000
|
||||
S3_BUCKET: messenger-test
|
||||
S3_ACCESS_KEY_ID: test-root
|
||||
S3_SECRET_ACCESS_KEY: test-root-password
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
redis:
|
||||
condition: service_started
|
||||
s3:
|
||||
condition: service_healthy
|
||||
+448
@@ -0,0 +1,448 @@
|
||||
name: messenger-lab6
|
||||
|
||||
x-otel: &otel
|
||||
OTEL_EXPORTER_OTLP_ENDPOINT: ${OTEL_EXPORTER_OTLP_ENDPOINT:-http://otel-collector:4318}
|
||||
OTEL_EXPORTER_OTLP_PROTOCOL: http/protobuf
|
||||
OTEL_RESOURCE_ATTRIBUTES: ${OTEL_RESOURCE_ATTRIBUTES:-deployment.environment.name=lab}
|
||||
|
||||
services:
|
||||
secrets-init:
|
||||
image: alpine:3.20.3
|
||||
volumes:
|
||||
- secrets:/secrets
|
||||
command:
|
||||
- sh
|
||||
- -ec
|
||||
- |
|
||||
cd /secrets
|
||||
mkdir -p postgres redis api migrate s3 s3-init reconciler broker rabbitmq relay worker grafana
|
||||
for entry in pg_password:postgres:24 redis_password:redis:24 s3_root_user:s3:10 s3_root_password:s3:20 s3_access_key_id:api:10 s3_secret_access_key:api:20 broker_password:broker:24 admin_password:grafana:16; do
|
||||
file=$${entry%%:*}; rest=$${entry#*:}; dir=$${rest%%:*}; size=$${rest##*:}
|
||||
[ -s "$$dir/$$file" ] || head -c "$$size" /dev/urandom | od -An -tx1 | tr -d ' \n' > "$$dir/$$file"
|
||||
done
|
||||
rm -f s3-init/* rabbitmq/* relay/* worker/*
|
||||
cp s3/s3_root_user s3/s3_root_password api/s3_access_key_id api/s3_secret_access_key s3-init/
|
||||
db_url() { printf 'postgresql://messenger:%s@postgres:5432/messenger?connection_limit=%s&pool_timeout=5&connect_timeout=5&socket_timeout=10&application_name=%s&options=-c%%20statement_timeout%%3D5000' "$$(cat postgres/pg_password)" "$$1" "$$2"; }
|
||||
db_url 10 api > api/database_url
|
||||
db_url 5 worker > worker/database_url
|
||||
db_url 3 relay > relay/database_url
|
||||
printf 'redis://:%s@redis:6379/0' "$$(cat redis/redis_password)" > api/cache_url
|
||||
printf 'postgresql://messenger:%s@postgres:5432/messenger' "$$(cat postgres/pg_password)" > migrate/database_url
|
||||
printf 'amqp://messenger:%s@rabbitmq:5672/' "$$(cat broker/broker_password)" > worker/broker_url
|
||||
cp worker/broker_url relay/
|
||||
printf 'default_user = messenger\ndefault_pass = %s\nloopback_users = none\nlog.console = true\nlisteners.tcp.default = 5672\n' "$$(cat broker/broker_password)" > rabbitmq/10-messenger.conf
|
||||
cp api/database_url api/s3_access_key_id api/s3_secret_access_key reconciler/
|
||||
cp api/s3_access_key_id api/s3_secret_access_key worker/
|
||||
chmod 0444 postgres/* redis/* api/* migrate/* s3/* s3-init/* reconciler/* broker/* rabbitmq/* relay/* worker/* grafana/*
|
||||
deploy:
|
||||
resources:
|
||||
limits: { cpus: "0.1", memory: 32M }
|
||||
|
||||
postgres:
|
||||
image: postgres:17.2-alpine
|
||||
environment:
|
||||
POSTGRES_USER: messenger
|
||||
POSTGRES_DB: messenger
|
||||
POSTGRES_PASSWORD_FILE: /run/secrets/pg_password
|
||||
volumes:
|
||||
- pgdata:/var/lib/postgresql/data
|
||||
- { type: volume, source: secrets, target: /run/secrets, read_only: true, volume: { subpath: postgres } }
|
||||
deploy:
|
||||
resources:
|
||||
limits: { cpus: "1.0", memory: 512M }
|
||||
healthcheck:
|
||||
test: ["CMD", "pg_isready", "-U", "messenger", "-d", "messenger"]
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 10
|
||||
depends_on:
|
||||
secrets-init:
|
||||
condition: service_completed_successfully
|
||||
restart: unless-stopped
|
||||
|
||||
redis:
|
||||
image: redis:7.4.1-alpine
|
||||
user: redis
|
||||
command:
|
||||
- sh
|
||||
- -c
|
||||
- exec redis-server --save '' --appendonly no --maxmemory 64mb --maxmemory-policy allkeys-lru --requirepass "$$(cat /run/secrets/redis_password)"
|
||||
volumes:
|
||||
- { type: volume, source: secrets, target: /run/secrets, read_only: true, volume: { subpath: redis } }
|
||||
deploy:
|
||||
resources:
|
||||
limits: { cpus: "0.25", memory: 128M }
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "redis-cli --no-auth-warning -a \"$$(cat /run/secrets/redis_password)\" ping | grep -q PONG"]
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 10
|
||||
depends_on:
|
||||
secrets-init:
|
||||
condition: service_completed_successfully
|
||||
restart: unless-stopped
|
||||
|
||||
s3:
|
||||
image: pgsty/minio:RELEASE.2026-08-04T00-00-00Z@sha256:b6bfe7239bfc83fb90d31612d9704d86039dd714f7904b3f1ad68f211e602372
|
||||
command: ["server", "/data", "--address", ":9000"]
|
||||
environment:
|
||||
MINIO_ROOT_USER_FILE: /run/secrets/s3_root_user
|
||||
MINIO_ROOT_PASSWORD_FILE: /run/secrets/s3_root_password
|
||||
MINIO_BROWSER: "off"
|
||||
volumes:
|
||||
- s3data:/data
|
||||
- { type: volume, source: secrets, target: /run/secrets, read_only: true, volume: { subpath: s3 } }
|
||||
deploy:
|
||||
resources:
|
||||
limits: { cpus: "0.5", memory: 512M }
|
||||
healthcheck:
|
||||
test: ["CMD", "curl", "-fsS", "-o", "/dev/null", "http://127.0.0.1:9000/minio/health/live"]
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 10
|
||||
depends_on:
|
||||
secrets-init:
|
||||
condition: service_completed_successfully
|
||||
restart: unless-stopped
|
||||
|
||||
s3-init:
|
||||
image: pgsty/mc:RELEASE.2026-10-26T00-00-00Z@sha256:ce62c9053c69ce12a527a322345b5ad9f44b5f8a7b8189f82e516f2e3f30f1d8
|
||||
entrypoint: ["sh", "-ec"]
|
||||
command:
|
||||
- |
|
||||
cd /run/secrets
|
||||
export MC_HOST_root="http://$$(cat s3_root_user):$$(cat s3_root_password)@s3:9000"
|
||||
mc mb --ignore-existing "root/$$S3_BUCKET"
|
||||
mc anonymous set none "root/$$S3_BUCKET"
|
||||
cat > /tmp/policy.json <<POLICY
|
||||
{"Version":"2012-10-17","Statement":[
|
||||
{"Effect":"Allow","Action":["s3:ListBucket","s3:GetBucketLocation","s3:ListBucketMultipartUploads"],"Resource":["arn:aws:s3:::$$S3_BUCKET"]},
|
||||
{"Effect":"Allow","Action":["s3:GetObject","s3:PutObject","s3:DeleteObject","s3:AbortMultipartUpload","s3:ListMultipartUploadParts"],"Resource":["arn:aws:s3:::$$S3_BUCKET/*"]}
|
||||
]}
|
||||
POLICY
|
||||
mc admin policy create root messenger-api /tmp/policy.json
|
||||
mc admin user add root "$$(cat s3_access_key_id)" "$$(cat s3_secret_access_key)" > /dev/null
|
||||
mc admin policy attach root messenger-api --user "$$(cat s3_access_key_id)" > /dev/null
|
||||
echo "bucket $$S3_BUCKET ready"
|
||||
environment:
|
||||
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
|
||||
volumes:
|
||||
- { type: volume, source: secrets, target: /run/secrets, read_only: true, volume: { subpath: s3-init } }
|
||||
deploy:
|
||||
resources:
|
||||
limits: { cpus: "0.25", memory: 256M }
|
||||
depends_on:
|
||||
s3:
|
||||
condition: service_healthy
|
||||
|
||||
migrate:
|
||||
build:
|
||||
context: .
|
||||
target: migrate
|
||||
image: mtusi-messenger-lab6-migrate:dev
|
||||
environment:
|
||||
DATABASE_URL_FILE: /run/secrets/database_url
|
||||
volumes:
|
||||
- { type: volume, source: secrets, target: /run/secrets, read_only: true, volume: { subpath: migrate } }
|
||||
deploy:
|
||||
resources:
|
||||
limits: { cpus: "0.5", memory: 512M }
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
|
||||
api:
|
||||
build:
|
||||
context: .
|
||||
target: runtime
|
||||
image: mtusi-messenger-lab6-api:dev
|
||||
environment:
|
||||
APP_HOST: 0.0.0.0
|
||||
APP_PORT: 8080
|
||||
APP_ENV: ${APP_ENV:-lab}
|
||||
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600}
|
||||
IDEMPOTENCY_TTL_SECONDS: ${IDEMPOTENCY_TTL_SECONDS:-86400}
|
||||
SHUTDOWN_DELAY_MS: 3000
|
||||
DATABASE_URL_FILE: /run/secrets/database_url
|
||||
CACHE_URL_FILE: /run/secrets/cache_url
|
||||
S3_ENDPOINT: http://s3:9000
|
||||
S3_REGION: ${S3_REGION:-us-east-1}
|
||||
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
|
||||
S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id
|
||||
S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key
|
||||
MAX_UPLOAD_BYTES: ${MAX_UPLOAD_BYTES:-10485760}
|
||||
MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS:-40000000}
|
||||
JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5}
|
||||
<<: *otel
|
||||
OTEL_SERVICE_NAME: messenger-api
|
||||
volumes:
|
||||
- { type: volume, source: secrets, target: /run/secrets, read_only: true, volume: { subpath: api } }
|
||||
deploy:
|
||||
replicas: ${API_REPLICAS:-2}
|
||||
resources:
|
||||
limits: { cpus: "0.5", memory: 256M }
|
||||
stop_grace_period: 10s
|
||||
depends_on:
|
||||
migrate:
|
||||
condition: service_completed_successfully
|
||||
redis:
|
||||
condition: service_started
|
||||
s3-init:
|
||||
condition: service_completed_successfully
|
||||
restart: unless-stopped
|
||||
|
||||
reconciler:
|
||||
build:
|
||||
context: .
|
||||
target: runtime
|
||||
image: mtusi-messenger-lab6-api:dev
|
||||
command: ["node", "dist/reconcile.js", "--apply", "--loop", "--grace-seconds=${RECONCILE_GRACE_SECONDS:-3600}", "--interval-seconds=${RECONCILE_INTERVAL_SECONDS:-300}"]
|
||||
environment:
|
||||
DATABASE_URL_FILE: /run/secrets/database_url
|
||||
STAGING_RETENTION_SECONDS: ${STAGING_RETENTION_SECONDS:-86400}
|
||||
S3_ENDPOINT: http://s3:9000
|
||||
S3_REGION: ${S3_REGION:-us-east-1}
|
||||
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
|
||||
S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id
|
||||
S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key
|
||||
volumes:
|
||||
- { type: volume, source: secrets, target: /run/secrets, read_only: true, volume: { subpath: reconciler } }
|
||||
deploy:
|
||||
resources:
|
||||
limits: { cpus: "0.25", memory: 128M }
|
||||
healthcheck:
|
||||
disable: true
|
||||
depends_on:
|
||||
migrate:
|
||||
condition: service_completed_successfully
|
||||
s3-init:
|
||||
condition: service_completed_successfully
|
||||
restart: unless-stopped
|
||||
|
||||
rabbitmq:
|
||||
image: rabbitmq:4.0.5-alpine@sha256:e045b5166b9188a38eee5c99411709aa1f0ffa2ac75664053e64cb69411f7c32
|
||||
hostname: rabbitmq
|
||||
volumes:
|
||||
- rabbitmqdata:/var/lib/rabbitmq
|
||||
- { type: volume, source: secrets, target: /etc/rabbitmq/conf.d, read_only: true, volume: { subpath: rabbitmq } }
|
||||
deploy:
|
||||
resources:
|
||||
limits: { cpus: "0.5", memory: 512M }
|
||||
healthcheck:
|
||||
test: ["CMD", "rabbitmq-diagnostics", "-q", "check_port_connectivity"]
|
||||
interval: 10s
|
||||
timeout: 10s
|
||||
retries: 12
|
||||
start_period: 20s
|
||||
stop_grace_period: 30s
|
||||
depends_on:
|
||||
secrets-init:
|
||||
condition: service_completed_successfully
|
||||
restart: unless-stopped
|
||||
|
||||
relay:
|
||||
build:
|
||||
context: .
|
||||
target: runtime
|
||||
image: mtusi-messenger-lab6-api:dev
|
||||
command: ["node", "dist/relay.js"]
|
||||
environment:
|
||||
DATABASE_URL_FILE: /run/secrets/database_url
|
||||
BROKER_URL_FILE: /run/secrets/broker_url
|
||||
JOB_STALE_SECONDS: ${JOB_STALE_SECONDS:-300}
|
||||
RELAY_FAULT: ${RELAY_FAULT:-}
|
||||
<<: *otel
|
||||
OTEL_SERVICE_NAME: messenger-relay
|
||||
volumes:
|
||||
- { type: volume, source: secrets, target: /run/secrets, read_only: true, volume: { subpath: relay } }
|
||||
deploy:
|
||||
resources:
|
||||
limits: { cpus: "0.25", memory: 128M }
|
||||
healthcheck:
|
||||
disable: true
|
||||
stop_grace_period: 10s
|
||||
depends_on:
|
||||
migrate:
|
||||
condition: service_completed_successfully
|
||||
rabbitmq:
|
||||
condition: service_healthy
|
||||
restart: unless-stopped
|
||||
|
||||
worker:
|
||||
build:
|
||||
context: .
|
||||
target: runtime
|
||||
image: mtusi-messenger-lab6-api:dev
|
||||
command: ["node", "dist/worker.js"]
|
||||
environment:
|
||||
DATABASE_URL_FILE: /run/secrets/database_url
|
||||
BROKER_URL_FILE: /run/secrets/broker_url
|
||||
S3_ENDPOINT: http://s3:9000
|
||||
S3_REGION: ${S3_REGION:-us-east-1}
|
||||
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
|
||||
S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id
|
||||
S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key
|
||||
MAX_UPLOAD_BYTES: ${MAX_UPLOAD_BYTES:-10485760}
|
||||
MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS:-40000000}
|
||||
WORKER_CONCURRENCY: ${WORKER_CONCURRENCY:-2}
|
||||
JOB_TIMEOUT_SECONDS: ${JOB_TIMEOUT_SECONDS:-60}
|
||||
JOB_LEASE_SECONDS: ${JOB_LEASE_SECONDS:-90}
|
||||
WORKER_FAULT: ${WORKER_FAULT:-}
|
||||
<<: *otel
|
||||
OTEL_SERVICE_NAME: messenger-worker
|
||||
volumes:
|
||||
- { type: volume, source: secrets, target: /run/secrets, read_only: true, volume: { subpath: worker } }
|
||||
deploy:
|
||||
resources:
|
||||
limits: { cpus: "1.0", memory: 768M }
|
||||
healthcheck:
|
||||
disable: true
|
||||
stop_grace_period: 30s
|
||||
depends_on:
|
||||
migrate:
|
||||
condition: service_completed_successfully
|
||||
rabbitmq:
|
||||
condition: service_healthy
|
||||
s3-init:
|
||||
condition: service_completed_successfully
|
||||
restart: unless-stopped
|
||||
|
||||
lb:
|
||||
image: haproxy:3.0.6-alpine
|
||||
ports:
|
||||
- "127.0.0.1:${APP_PORT:-8080}:8080"
|
||||
volumes:
|
||||
- ./infra/haproxy/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro
|
||||
deploy:
|
||||
resources:
|
||||
limits: { cpus: "0.25", memory: 64M }
|
||||
depends_on:
|
||||
api:
|
||||
condition: service_healthy
|
||||
restart: unless-stopped
|
||||
|
||||
otel-collector:
|
||||
image: otel/opentelemetry-collector-contrib:0.115.1@sha256:d2da12c4336a79758826700be9e21ecf4a9f7d945b7f8a58ba55ee3fa45427c8
|
||||
command: ["--config=/etc/otelcol/collector.yaml"]
|
||||
volumes:
|
||||
- ./infra/otel/collector.yaml:/etc/otelcol/collector.yaml:ro
|
||||
networks: [default, telemetry]
|
||||
deploy:
|
||||
resources:
|
||||
limits: { cpus: "0.5", memory: 256M }
|
||||
restart: unless-stopped
|
||||
|
||||
prometheus:
|
||||
image: prom/prometheus:v3.1.0@sha256:6559acbd5d770b15bb3c954629ce190ac3cbbdb2b7f1c30f0385c4e05104e218
|
||||
command:
|
||||
- --config.file=/etc/prometheus/prometheus.yml
|
||||
- --storage.tsdb.path=/prometheus
|
||||
- --storage.tsdb.retention.time=7d
|
||||
- --storage.tsdb.retention.size=1GB
|
||||
- --web.enable-otlp-receiver
|
||||
volumes:
|
||||
- ./infra/prometheus:/etc/prometheus:ro
|
||||
- prometheusdata:/prometheus
|
||||
networks: [telemetry]
|
||||
deploy:
|
||||
resources:
|
||||
limits: { cpus: "0.5", memory: 512M }
|
||||
healthcheck:
|
||||
test: ["CMD", "wget", "-q", "-O", "/dev/null", "http://127.0.0.1:9090/-/ready"]
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 20
|
||||
restart: unless-stopped
|
||||
|
||||
loki:
|
||||
image: grafana/loki:3.3.2@sha256:8af2de1abbdd7aa92b27c9bcc96f0f4140c9096b507c77921ffddf1c6ad6c48f
|
||||
command: ["-config.file=/etc/loki/loki.yaml"]
|
||||
volumes:
|
||||
- ./infra/loki/loki.yaml:/etc/loki/loki.yaml:ro
|
||||
- lokidata:/loki
|
||||
networks: [telemetry]
|
||||
deploy:
|
||||
resources:
|
||||
limits: { cpus: "0.5", memory: 512M }
|
||||
restart: unless-stopped
|
||||
|
||||
tempo:
|
||||
image: grafana/tempo:2.6.1@sha256:ef4384fce6e8ad22b95b243d8fc165628cda655376fd50e7850536ad89d71d50
|
||||
command: ["-config.file=/etc/tempo/tempo.yaml"]
|
||||
volumes:
|
||||
- ./infra/tempo/tempo.yaml:/etc/tempo/tempo.yaml:ro
|
||||
- tempodata:/var/tempo
|
||||
networks: [telemetry]
|
||||
deploy:
|
||||
resources:
|
||||
limits: { cpus: "0.5", memory: 512M }
|
||||
restart: unless-stopped
|
||||
|
||||
grafana:
|
||||
image: grafana/grafana:11.4.0@sha256:d8ea37798ccc41061a62ab080f2676dda6bf7815558499f901bdb0f533a456fb
|
||||
ports:
|
||||
- "127.0.0.1:${GRAFANA_PORT:-3000}:3000"
|
||||
environment:
|
||||
GF_SECURITY_ADMIN_USER: admin
|
||||
GF_SECURITY_ADMIN_PASSWORD__FILE: /run/secrets/admin_password
|
||||
GF_AUTH_ANONYMOUS_ENABLED: "true"
|
||||
GF_AUTH_ANONYMOUS_ORG_ROLE: Viewer
|
||||
GF_USERS_VIEWERS_CAN_EDIT: "true"
|
||||
GF_ANALYTICS_REPORTING_ENABLED: "false"
|
||||
GF_ANALYTICS_CHECK_FOR_UPDATES: "false"
|
||||
GF_NEWS_NEWS_FEED_ENABLED: "false"
|
||||
volumes:
|
||||
- ./infra/grafana/provisioning:/etc/grafana/provisioning:ro
|
||||
- ./infra/grafana/dashboards:/var/lib/grafana/dashboards:ro
|
||||
- grafanadata:/var/lib/grafana
|
||||
- { type: volume, source: secrets, target: /run/secrets, read_only: true, volume: { subpath: grafana } }
|
||||
networks: [default, telemetry]
|
||||
deploy:
|
||||
resources:
|
||||
limits: { cpus: "0.5", memory: 256M }
|
||||
healthcheck:
|
||||
test: ["CMD", "wget", "-q", "-O", "/dev/null", "http://127.0.0.1:3000/api/health"]
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 20
|
||||
depends_on:
|
||||
secrets-init:
|
||||
condition: service_completed_successfully
|
||||
prometheus:
|
||||
condition: service_healthy
|
||||
loki:
|
||||
condition: service_started
|
||||
tempo:
|
||||
condition: service_started
|
||||
restart: unless-stopped
|
||||
|
||||
tools:
|
||||
profiles: [tools]
|
||||
build:
|
||||
dockerfile_inline: |
|
||||
FROM docker:27.3.1-cli
|
||||
RUN apk add --no-cache python3 make git && git config --global --add safe.directory /work
|
||||
WORKDIR /work
|
||||
image: mtusi-messenger-lab6-tools:dev
|
||||
environment:
|
||||
BASE_URL: http://lb:8080
|
||||
volumes:
|
||||
- .:/work
|
||||
- /var/run/docker.sock:/var/run/docker.sock
|
||||
deploy:
|
||||
resources:
|
||||
limits: { cpus: "1.0", memory: 256M }
|
||||
|
||||
volumes:
|
||||
pgdata:
|
||||
s3data:
|
||||
rabbitmqdata:
|
||||
prometheusdata:
|
||||
lokidata:
|
||||
tempodata:
|
||||
grafanadata:
|
||||
secrets:
|
||||
|
||||
networks:
|
||||
telemetry:
|
||||
internal: true
|
||||
@@ -0,0 +1,36 @@
|
||||
{
|
||||
"openapi": "3.1.0",
|
||||
"info": {
|
||||
"title": "MTUSI Messenger — расширения лабораторной 5",
|
||||
"version": "1.5.0",
|
||||
"description": "Обратно совместимое дополнение к contracts/openapi.json (см. COURSE.md, раздел «Контракт и совместимость»). Обязательный контракт не меняется. Обоснование — REPORT.md, раздел «Критичные зависимости и проверка готовности»."
|
||||
},
|
||||
"paths": {
|
||||
"/health/core": {
|
||||
"get": {
|
||||
"operationId": "getCoreReadiness",
|
||||
"summary": "Служебная проверка для балансировщика и Docker healthcheck: может ли реплика обслуживать API без вложений",
|
||||
"description": "В отличие от /health/ready не учитывает S3. Проверяет PostgreSQL и то, что процесс не останавливается. Падение общего S3 не выводит реплики из балансировки: сообщения продолжают работать, вложения отвечают 503.",
|
||||
"responses": {
|
||||
"200": {
|
||||
"description": "Реплика готова",
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"type": "object",
|
||||
"properties": { "status": { "const": "ok" } },
|
||||
"required": ["status"],
|
||||
"additionalProperties": false
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"503": {
|
||||
"description": "PostgreSQL недоступен или процесс останавливается; тело в общем формате Error",
|
||||
"content": { "application/json": { "schema": { "type": "object" } } }
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,75 @@
|
||||
# Runbooks
|
||||
|
||||
Alert rules: [infra/prometheus/rules/messenger.yml](../infra/prometheus/rules/messenger.yml). Состояние: Grafana → dashboard «Messenger overview» → панель «Alerts»; или Prometheus API `GET /api/v1/alerts` (из контейнера `tools`: `wget -qO- http://grafana:3000/api/datasources/proxy/uid/prometheus/api/v1/alerts`). Уведомления наружу не отправляются.
|
||||
|
||||
## SLO
|
||||
|
||||
| SLI | Формула | SLO | Окно |
|
||||
| --- | --- | --- | --- |
|
||||
| Доступность API | 1 − 5xx / все бизнес-запросы (без `/health/*`) | ≥ 99% | 1 ч (учебный стенд; в эксплуатации — 30 дней) |
|
||||
| Задержка API | доля бизнес-запросов с длительностью ≤ 0,5 с (bucket `le="0.5"`) | ≥ 95% | 1 ч |
|
||||
| Готовность вложений | доля вложений, ставших `ready` за ≤ 10 с от принятия upload (bucket `le="10"`) | ≥ 95% | 1 ч |
|
||||
|
||||
## MessengerHighErrorRate
|
||||
|
||||
**Условие:** доля 5xx среди бизнес-запросов за 5 минут > 5% при трафике > 0,05 rps, держится 1 минуту. При SLO 99% это сжигание бюджета ошибок в 5 раз быстрее допустимого.
|
||||
|
||||
**Проверить:**
|
||||
1. Панель «Ответы по классам статуса» и «p95 по маршрутам»: какой маршрут даёт 5xx.
|
||||
2. Loki: `{service_name="messenger-api"} | event="http.error"` — поле `message` (`database unavailable`, `object storage unavailable`, `Internal server error`), `route`, `request_id`. По `trace_id` записи — trace в Tempo.
|
||||
3. В trace: на каком span ошибка — `prisma:engine:db_query`, `S3.PutObject` или код обработчика.
|
||||
4. `docker compose ps` — живы ли postgres, s3, redis; `/health/ready` реплик.
|
||||
|
||||
**Действия:** восстановить зависимость (`docker compose start postgres` и т. п.); при ошибке кода — откат образа. Readiness исключает реплику без БД из HAProxy автоматически.
|
||||
|
||||
**Resolved:** доля 5xx за 5 минут снова ниже 5%.
|
||||
|
||||
## MessengerApiNotReady
|
||||
|
||||
**Условие:** больше 10 ответов 5xx на `/health/core` (проверка HAProxy раз в секунду, `SELECT 1` в PostgreSQL) за минуту, держится 30 с. Штатная остановка реплики даёт 3–4 таких ответа (3 с drain) и alert не вызывает; недоступная БД — около 140 в минуту на две реплики.
|
||||
|
||||
**Почему отдельный alert:** когда все реплики не готовы, HAProxy сам отвечает клиентам `503 no API replica available`; эти ответы не проходят через API и не входят в `http_server_requests_total`, поэтому доля 5xx бизнес-запросов может оставаться нулевой.
|
||||
|
||||
**Проверить:** Loki `{service_name="messenger-api"} | event="http.error"` — `route=/health/core`, `message` (`database unavailable` или `shutting down`); `docker compose ps postgres`; панель «Пул БД» (рост `waiting`).
|
||||
|
||||
**Действия:** восстановить PostgreSQL (`docker compose start postgres` / `docker compose unpause postgres`). После первых успешных проверок (`rise 2`) HAProxy возвращает реплики.
|
||||
|
||||
**Resolved:** проверки готовности снова 200 в течение минуты.
|
||||
|
||||
## MessengerJobQueueLag
|
||||
|
||||
**Условие:** старейшее задание в состоянии `queued` с наступившим `next_attempt_at` ждёт дольше 60 с, держится 1 минуту. Источник — таблица `attachment_jobs` (метрика `jobs_queue_oldest_age_seconds` от relay), а не брокер: она показывает и задания, сообщения которых ещё не опубликованы.
|
||||
|
||||
**Проверить:**
|
||||
1. Панель «Worker: занято / ёмкость». `busy = capacity` — worker насыщен (медленная обработка); метрик worker нет совсем — процесс остановлен; `busy = 0` при растущей очереди — worker не получает сообщения (брокер, relay).
|
||||
2. «Задания по исходу»: много `retry` — зависимость worker (S3/БД) отвечает ошибками.
|
||||
3. `docker compose ps worker relay rabbitmq`; логи worker в Loki: `{service_name="messenger-worker"}`.
|
||||
4. Возраст отдельных заданий: `SELECT id, status, attempt, next_attempt_at FROM attachment_jobs WHERE status='queued' ORDER BY next_attempt_at LIMIT 10;`
|
||||
|
||||
**Действия:** запустить worker (`docker compose start worker`) или добавить реплик/увеличить `WORKER_CONCURRENCY` при насыщении. Задания не теряются: при старте worker обрабатывает backlog.
|
||||
|
||||
**Resolved:** backlog разобран, возраст старейшего задания < 60 с.
|
||||
|
||||
## MessengerSlowApi
|
||||
|
||||
**Условие:** p95 бизнес-запросов за 5 минут > 1 с при трафике > 0,05 rps, держится 2 минуты.
|
||||
|
||||
**Как отличить медленную БД от медленного обработчика:**
|
||||
- медленная/недоступная БД: растёт `db_client_connection_pending_requests` (ожидание пула) и/или в trace медленного запроса основное время — spans `prisma:engine:db_query`; ошибки `database unavailable` (503);
|
||||
- медленный обработчик: пул свободен, spans БД короткие, время приходится на сам SERVER span или S3/рендеринг; растёт CPU API.
|
||||
|
||||
**Действия:** для БД — проверить postgres (`pg_stat_activity`, блокировки, ресурсы контейнера); для обработчика — профилировать маршрут, найденный в «p95 по маршрутам».
|
||||
|
||||
**Resolved:** p95 за 5 минут < 1 с.
|
||||
|
||||
## TelemetryExportFailing
|
||||
|
||||
**Условие:** Collector получает ошибки при отправке в Loki/Tempo/Prometheus или очередь exporter заполнена больше чем наполовину, держится 1 минуту.
|
||||
|
||||
**Что происходит:** API и worker продолжают работать: SDK отправляет телеметрию асинхронно с таймаутом 5 с и ограниченными очередями (2048 записей), Collector держит ограниченную очередь (`queue_size: 200` batch на exporter), повторяет отправку до 60 с и затем отбрасывает данные. `memory_limiter` (160 MiB) отказывает в приёме при нехватке памяти. Потери видны в `otelcol_exporter_send_failed_*` и `otelcol_receiver_refused_*`.
|
||||
|
||||
**Проверить:** панель «Конвейер телеметрии»; `docker compose ps loki tempo prometheus`; логи Collector `docker compose logs otel-collector`.
|
||||
|
||||
**Действия:** запустить хранилище (`docker compose start loki tempo`). Данные, отброшенные за время недоступности, не восстанавливаются; логи остаются в stdout контейнеров (`docker compose logs`).
|
||||
|
||||
**Resolved:** отказов отправки нет, очередь exporter опустела.
|
||||
@@ -0,0 +1,318 @@
|
||||
# Стенд: сервисы, порты, лимиты, секреты, миграции, топология брокера
|
||||
# date: 2026-11-20T14:30:52Z
|
||||
# commit: d83876f
|
||||
|
||||
$ docker compose ps -a --format table {{.Service}}\t{{.Name}}\t{{.Image}}\t{{.Status}}\t{{.Ports}}
|
||||
SERVICE NAME IMAGE STATUS PORTS
|
||||
api messenger-lab6-api-1 mtusi-messenger-lab6-api:dev Up 13 seconds (healthy) 8080/tcp
|
||||
api messenger-lab6-api-2 mtusi-messenger-lab6-api:dev Up 13 seconds (healthy) 8080/tcp
|
||||
grafana messenger-lab6-grafana-1 grafana/grafana:11.4.0@sha256:d8ea37798ccc41061a62ab080f2676dda6bf7815558499f901bdb0f533a456fb Up 19 seconds (healthy) 127.0.0.1:3000->3000/tcp
|
||||
lb messenger-lab6-lb-1 haproxy:3.0.6-alpine Up 7 seconds 127.0.0.1:8080->8080/tcp
|
||||
loki messenger-lab6-loki-1 grafana/loki:3.3.2@sha256:8af2de1abbdd7aa92b27c9bcc96f0f4140c9096b507c77921ffddf1c6ad6c48f Up 25 seconds 3100/tcp
|
||||
migrate messenger-lab6-migrate-1 mtusi-messenger-lab6-migrate:dev Exited (0) 14 seconds ago
|
||||
otel-collector messenger-lab6-otel-collector-1 otel/opentelemetry-collector-contrib:0.115.1@sha256:d2da12c4336a79758826700be9e21ecf4a9f7d945b7f8a58ba55ee3fa45427c8 Up 25 seconds 4317-4318/tcp, 55678-55679/tcp
|
||||
postgres messenger-lab6-postgres-1 postgres:17.2-alpine Up 24 seconds (healthy) 5432/tcp
|
||||
prometheus messenger-lab6-prometheus-1 prom/prometheus:v3.1.0@sha256:6559acbd5d770b15bb3c954629ce190ac3cbbdb2b7f1c30f0385c4e05104e218 Up 25 seconds (healthy) 9090/tcp
|
||||
rabbitmq messenger-lab6-rabbitmq-1 rabbitmq:4.0.5-alpine@sha256:e045b5166b9188a38eee5c99411709aa1f0ffa2ac75664053e64cb69411f7c32 Up 24 seconds (healthy) 4369/tcp, 5671-5672/tcp, 15691-15692/tcp, 25672/tcp
|
||||
reconciler messenger-lab6-reconciler-1 mtusi-messenger-lab6-api:dev Up 13 seconds 8080/tcp
|
||||
redis messenger-lab6-redis-1 redis:7.4.1-alpine Up 24 seconds (healthy) 6379/tcp
|
||||
relay messenger-lab6-relay-1 mtusi-messenger-lab6-api:dev Up 7 seconds 8080/tcp
|
||||
s3 messenger-lab6-s3-1 pgsty/minio:RELEASE.2026-08-04T00-00-00Z@sha256:b6bfe7239bfc83fb90d31612d9704d86039dd714f7904b3f1ad68f211e602372 Up 24 seconds (healthy) 9000/tcp
|
||||
s3-init messenger-lab6-s3-init-1 pgsty/mc:RELEASE.2026-10-26T00-00-00Z@sha256:ce62c9053c69ce12a527a322345b5ad9f44b5f8a7b8189f82e516f2e3f30f1d8 Exited (0) 14 seconds ago
|
||||
secrets-init messenger-lab6-secrets-init-1 alpine:3.20.3 Exited (0) 24 seconds ago
|
||||
tempo messenger-lab6-tempo-1 grafana/tempo:2.6.1@sha256:ef4384fce6e8ad22b95b243d8fc165628cda655376fd50e7850536ad89d71d50 Up 25 seconds
|
||||
tools messenger-lab6-tools-run-68bd7ed38f92 mtusi-messenger-lab6-tools:dev Up 5 seconds
|
||||
worker messenger-lab6-worker-1 mtusi-messenger-lab6-api:dev Up 7 seconds 8080/tcp
|
||||
|
||||
## CPU/RAM-лимиты (NanoCpus: 1e9 = 1 CPU; Memory в байтах)
|
||||
$ sh -c docker inspect -f '{{.Name}} cpus={{.HostConfig.NanoCpus}} memory={{.HostConfig.Memory}}' $(docker compose ps -aq)
|
||||
/messenger-lab6-api-1 cpus=500000000 memory=268435456
|
||||
/messenger-lab6-api-2 cpus=500000000 memory=268435456
|
||||
/messenger-lab6-grafana-1 cpus=500000000 memory=268435456
|
||||
/messenger-lab6-lb-1 cpus=250000000 memory=67108864
|
||||
/messenger-lab6-loki-1 cpus=500000000 memory=536870912
|
||||
/messenger-lab6-migrate-1 cpus=500000000 memory=536870912
|
||||
/messenger-lab6-otel-collector-1 cpus=500000000 memory=268435456
|
||||
/messenger-lab6-postgres-1 cpus=1000000000 memory=536870912
|
||||
/messenger-lab6-prometheus-1 cpus=500000000 memory=536870912
|
||||
/messenger-lab6-rabbitmq-1 cpus=500000000 memory=536870912
|
||||
/messenger-lab6-reconciler-1 cpus=250000000 memory=134217728
|
||||
/messenger-lab6-redis-1 cpus=250000000 memory=134217728
|
||||
/messenger-lab6-relay-1 cpus=250000000 memory=134217728
|
||||
/messenger-lab6-s3-1 cpus=500000000 memory=536870912
|
||||
/messenger-lab6-s3-init-1 cpus=250000000 memory=268435456
|
||||
/messenger-lab6-secrets-init-1 cpus=100000000 memory=33554432
|
||||
/messenger-lab6-tempo-1 cpus=500000000 memory=536870912
|
||||
/messenger-lab6-tools-run-68bd7ed38f92 cpus=1000000000 memory=268435456
|
||||
/messenger-lab6-worker-1 cpus=1000000000 memory=805306368
|
||||
|
||||
## Опубликованные порты: только LB и Grafana на 127.0.0.1; OTLP, Prometheus, Loki, Tempo наружу не открыты
|
||||
$ sh -c docker compose ps --format '{{.Service}} {{.Ports}}' | grep -v '^\S* *$' || true
|
||||
api 8080/tcp
|
||||
api 8080/tcp
|
||||
grafana 127.0.0.1:3000->3000/tcp
|
||||
lb 127.0.0.1:8080->8080/tcp
|
||||
loki 3100/tcp
|
||||
otel-collector 4317-4318/tcp, 55678-55679/tcp
|
||||
postgres 5432/tcp
|
||||
prometheus 9090/tcp
|
||||
rabbitmq 4369/tcp, 5671-5672/tcp, 15691-15692/tcp, 25672/tcp
|
||||
reconciler 8080/tcp
|
||||
redis 6379/tcp
|
||||
relay 8080/tcp
|
||||
s3 9000/tcp
|
||||
worker 8080/tcp
|
||||
|
||||
## Сети телеметрии: хранилища только во внутренней сети telemetry
|
||||
$ sh -c for s in otel-collector prometheus loki tempo grafana api worker; do printf '%s: ' $s; docker inspect -f '{{range $k, $v := .NetworkSettings.Networks}}{{$k}} {{end}}' $(docker compose ps -q $s | head -1); done
|
||||
otel-collector: messenger-lab6_default messenger-lab6_telemetry
|
||||
prometheus: messenger-lab6_telemetry
|
||||
loki: messenger-lab6_telemetry
|
||||
tempo: messenger-lab6_telemetry
|
||||
grafana: messenger-lab6_default messenger-lab6_telemetry
|
||||
api: messenger-lab6_default
|
||||
worker: messenger-lab6_default
|
||||
|
||||
$ docker network inspect -f {{.Name}} internal={{.Internal}} messenger-lab6_telemetry
|
||||
messenger-lab6_telemetry internal=true
|
||||
|
||||
## Секреты: каждый сервис видит только свои файлы (содержимое не выводится)
|
||||
$ docker compose exec -T postgres ls /run/secrets
|
||||
pg_password
|
||||
|
||||
$ docker compose exec -T redis ls /run/secrets
|
||||
redis_password
|
||||
|
||||
$ docker compose exec -T s3 ls /run/secrets
|
||||
s3_root_password
|
||||
s3_root_user
|
||||
|
||||
$ docker compose exec -T api ls /run/secrets
|
||||
cache_url
|
||||
database_url
|
||||
s3_access_key_id
|
||||
s3_secret_access_key
|
||||
|
||||
$ docker compose exec -T reconciler ls /run/secrets
|
||||
database_url
|
||||
s3_access_key_id
|
||||
s3_secret_access_key
|
||||
|
||||
$ docker compose exec -T relay ls /run/secrets
|
||||
broker_url
|
||||
database_url
|
||||
|
||||
$ docker compose exec -T worker ls /run/secrets
|
||||
broker_url
|
||||
database_url
|
||||
s3_access_key_id
|
||||
s3_secret_access_key
|
||||
|
||||
$ docker compose exec -T grafana ls /run/secrets
|
||||
admin_password
|
||||
|
||||
$ docker compose exec -T rabbitmq ls /etc/rabbitmq/conf.d
|
||||
10-messenger.conf
|
||||
|
||||
$ docker compose exec -T lb sh -c ls /run/secrets 2>/dev/null || echo "no /run/secrets"
|
||||
no /run/secrets
|
||||
|
||||
## Версии
|
||||
$ docker compose exec -T lb haproxy -v
|
||||
HAProxy version 3.0.6-c2c0090 2024/11/07 - https://haproxy.org/
|
||||
Status: long-term supported branch - will stop receiving fixes around Q2 2029.
|
||||
Known bugs: http://www.haproxy.org/bugs/bugs-3.0.6.html
|
||||
Running on: Linux 5.15.153.1-microsoft-standard-WSL2 #1 SMP Fri Mar 29 23:14:13 UTC 2024 x86_64
|
||||
|
||||
$ docker compose exec -T postgres postgres --version
|
||||
postgres (PostgreSQL) 17.2
|
||||
|
||||
$ docker compose exec -T redis redis-server --version
|
||||
Redis server v=7.4.1 sha=00000000:0 malloc=jemalloc-5.3.0 bits=64 build=a93ce81042b3a6d7
|
||||
|
||||
$ docker compose exec -T s3 minio --version
|
||||
minio version RELEASE.2026-08-04T00-00-00Z (commit-id=d88f46ccee345a9c2fabe2d221d9a9e56bc11aec)
|
||||
Runtime: go1.26.5 linux/amd64
|
||||
License: GNU AGPLv3 - https://www.gnu.org/licenses/agpl-3.0.html
|
||||
Copyright: 2015-2026 MinIO, Inc.
|
||||
|
||||
$ docker compose exec -T rabbitmq rabbitmqctl -q version
|
||||
4.0.5
|
||||
|
||||
$ sh -c docker compose images otel-collector prometheus loki tempo grafana
|
||||
CONTAINER REPOSITORY TAG IMAGE ID SIZE
|
||||
messenger-lab6-grafana-1 grafana/grafana <none> d8ea37798ccc 133MB
|
||||
messenger-lab6-loki-1 grafana/loki <none> 8af2de1abbdd 33.3MB
|
||||
messenger-lab6-otel-collector-1 otel/opentelemetry-collector-contrib <none> d2da12c4336a 74.3MB
|
||||
messenger-lab6-prometheus-1 prom/prometheus <none> 6559acbd5d77 117MB
|
||||
messenger-lab6-tempo-1 grafana/tempo <none> ef4384fce6e8 52.4MB
|
||||
|
||||
## Топология брокера: durable quorum-очереди, delivery-limit и DLQ
|
||||
$ docker compose exec -T rabbitmq rabbitmqctl -q list_queues name type durable arguments messages
|
||||
name type durable arguments messages
|
||||
attachment.jobs quorum true [{"x-queue-type","quorum"},{"x-delivery-limit",20},{"x-dead-letter-exchange",[]},{"x-dead-letter-routing-key","attachment.jobs.dlq"}] 0
|
||||
attachment.jobs.dlq quorum true [{"x-queue-type","quorum"}] 0
|
||||
|
||||
$ docker compose exec -T rabbitmq rabbitmqctl -q list_consumers queue_name prefetch_count ack_required
|
||||
queue_name ack_required prefetch_count
|
||||
attachment.jobs true 2
|
||||
|
||||
## Таблицы
|
||||
$ docker compose exec -T postgres psql -U messenger -d messenger -c \d attachments -c \d attachment_variants -c \d attachment_jobs -c \d outbox -c \d job_events
|
||||
Table "public.attachments"
|
||||
Column | Type | Collation | Nullable | Default
|
||||
--------------+-----------------------------+-----------+----------+-------------------
|
||||
id | uuid | | not null |
|
||||
chat_id | uuid | | not null |
|
||||
owner_id | uuid | | not null |
|
||||
filename | character varying(255) | | not null |
|
||||
content_type | character varying(32) | | not null |
|
||||
size_bytes | integer | | not null |
|
||||
sha256 | character(64) | | not null |
|
||||
object_key | character varying(128) | | not null |
|
||||
created_at | timestamp(3) with time zone | | not null | CURRENT_TIMESTAMP
|
||||
updated_at | timestamp(3) with time zone | | not null | CURRENT_TIMESTAMP
|
||||
staging_key | character varying(128) | | |
|
||||
status | character varying(16) | | not null |
|
||||
error_code | character varying(64) | | |
|
||||
Indexes:
|
||||
"attachments_pkey" PRIMARY KEY, btree (id)
|
||||
"attachments_chat_id_idx" btree (chat_id)
|
||||
"attachments_object_key_key" UNIQUE, btree (object_key)
|
||||
"attachments_owner_id_idx" btree (owner_id)
|
||||
"attachments_staging_key_key" UNIQUE, btree (staging_key)
|
||||
Check constraints:
|
||||
"attachments_content_type_allowed" CHECK (content_type::text = ANY (ARRAY['image/png'::character varying, 'image/jpeg'::character varying]::text[]))
|
||||
"attachments_error_code_when_failed" CHECK ((status::text = 'failed'::text) = (error_code IS NOT NULL))
|
||||
"attachments_filename_not_blank" CHECK (btrim(filename::text) <> ''::text)
|
||||
"attachments_sha256_format" CHECK (sha256 ~ '^[a-f0-9]{64}$'::text)
|
||||
"attachments_size_range" CHECK (size_bytes >= 1 AND size_bytes <= 10485760)
|
||||
"attachments_status_allowed" CHECK (status::text = ANY (ARRAY['queued'::character varying, 'processing'::character varying, 'ready'::character varying, 'failed'::character varying]::text[]))
|
||||
Foreign-key constraints:
|
||||
"attachments_chat_id_fkey" FOREIGN KEY (chat_id) REFERENCES chats(id) ON UPDATE CASCADE ON DELETE CASCADE
|
||||
"attachments_owner_id_fkey" FOREIGN KEY (owner_id) REFERENCES users(id) ON UPDATE CASCADE ON DELETE RESTRICT
|
||||
Referenced by:
|
||||
TABLE "attachment_jobs" CONSTRAINT "attachment_jobs_attachment_id_fkey" FOREIGN KEY (attachment_id) REFERENCES attachments(id) ON UPDATE CASCADE ON DELETE CASCADE
|
||||
TABLE "attachment_variants" CONSTRAINT "attachment_variants_attachment_id_fkey" FOREIGN KEY (attachment_id) REFERENCES attachments(id) ON UPDATE CASCADE ON DELETE CASCADE
|
||||
TABLE "message_attachments" CONSTRAINT "message_attachments_attachment_id_fkey" FOREIGN KEY (attachment_id) REFERENCES attachments(id) ON UPDATE CASCADE
|
||||
|
||||
Table "public.attachment_variants"
|
||||
Column | Type | Collation | Nullable | Default
|
||||
------------------+-----------------------------+-----------+----------+-------------------
|
||||
attachment_id | uuid | | not null |
|
||||
pipeline_version | integer | | not null |
|
||||
name | character varying(16) | | not null |
|
||||
content_type | character varying(32) | | not null |
|
||||
object_key | character varying(128) | | not null |
|
||||
size_bytes | integer | | not null |
|
||||
width | integer | | not null |
|
||||
height | integer | | not null |
|
||||
sha256 | character(64) | | not null |
|
||||
created_at | timestamp(3) with time zone | | not null | CURRENT_TIMESTAMP
|
||||
Indexes:
|
||||
"attachment_variants_pkey" PRIMARY KEY, btree (attachment_id, pipeline_version, name)
|
||||
"attachment_variants_object_key_key" UNIQUE, btree (object_key)
|
||||
Check constraints:
|
||||
"attachment_variants_content_type_allowed" CHECK (content_type::text = ANY (ARRAY['image/jpeg'::character varying, 'image/png'::character varying, 'image/webp'::character varying]::text[]))
|
||||
"attachment_variants_name_allowed" CHECK (name::text = ANY (ARRAY['thumbnail'::character varying, 'crop'::character varying, 'optimized'::character varying]::text[]))
|
||||
"attachment_variants_positive" CHECK (pipeline_version >= 1 AND size_bytes >= 1 AND width >= 1 AND height >= 1)
|
||||
"attachment_variants_sha256_format" CHECK (sha256 ~ '^[a-f0-9]{64}$'::text)
|
||||
Foreign-key constraints:
|
||||
"attachment_variants_attachment_id_fkey" FOREIGN KEY (attachment_id) REFERENCES attachments(id) ON UPDATE CASCADE ON DELETE CASCADE
|
||||
|
||||
Table "public.attachment_jobs"
|
||||
Column | Type | Collation | Nullable | Default
|
||||
------------------+-----------------------------+-----------+----------+-------------------
|
||||
id | uuid | | not null |
|
||||
attachment_id | uuid | | not null |
|
||||
pipeline_version | integer | | not null |
|
||||
status | character varying(16) | | not null |
|
||||
attempt | integer | | not null |
|
||||
max_attempts | integer | | not null |
|
||||
next_attempt_at | timestamp(3) with time zone | | not null |
|
||||
lease_owner | uuid | | |
|
||||
lease_until | timestamp(3) with time zone | | |
|
||||
error_code | character varying(64) | | |
|
||||
last_error | character varying(200) | | |
|
||||
created_at | timestamp(3) with time zone | | not null | CURRENT_TIMESTAMP
|
||||
updated_at | timestamp(3) with time zone | | not null | CURRENT_TIMESTAMP
|
||||
finished_at | timestamp(3) with time zone | | |
|
||||
Indexes:
|
||||
"attachment_jobs_pkey" PRIMARY KEY, btree (id)
|
||||
"attachment_jobs_attachment_id_pipeline_version_key" UNIQUE, btree (attachment_id, pipeline_version)
|
||||
"attachment_jobs_status_lease_until_idx" btree (status, lease_until)
|
||||
Check constraints:
|
||||
"attachment_jobs_attempt_range" CHECK (pipeline_version >= 1 AND attempt >= 1 AND attempt <= max_attempts)
|
||||
"attachment_jobs_error_code_when_failed" CHECK ((status::text = 'failed'::text) = (error_code IS NOT NULL))
|
||||
"attachment_jobs_lease_when_processing" CHECK (status::text <> 'processing'::text OR lease_owner IS NOT NULL AND lease_until IS NOT NULL)
|
||||
"attachment_jobs_status_allowed" CHECK (status::text = ANY (ARRAY['queued'::character varying, 'processing'::character varying, 'ready'::character varying, 'failed'::character varying]::text[]))
|
||||
Foreign-key constraints:
|
||||
"attachment_jobs_attachment_id_fkey" FOREIGN KEY (attachment_id) REFERENCES attachments(id) ON UPDATE CASCADE ON DELETE CASCADE
|
||||
Referenced by:
|
||||
TABLE "job_events" CONSTRAINT "job_events_job_id_fkey" FOREIGN KEY (job_id) REFERENCES attachment_jobs(id) ON UPDATE CASCADE ON DELETE CASCADE
|
||||
TABLE "outbox" CONSTRAINT "outbox_job_id_fkey" FOREIGN KEY (job_id) REFERENCES attachment_jobs(id) ON UPDATE CASCADE ON DELETE CASCADE
|
||||
|
||||
Table "public.outbox"
|
||||
Column | Type | Collation | Nullable | Default
|
||||
------------------+-----------------------------+-----------+----------+------------------------------------
|
||||
id | bigint | | not null | nextval('outbox_id_seq'::regclass)
|
||||
job_id | uuid | | not null |
|
||||
payload | jsonb | | not null |
|
||||
available_at | timestamp(3) with time zone | | not null | CURRENT_TIMESTAMP
|
||||
published_at | timestamp(3) with time zone | | |
|
||||
publish_attempts | integer | | not null | 0
|
||||
created_at | timestamp(3) with time zone | | not null | CURRENT_TIMESTAMP
|
||||
Indexes:
|
||||
"outbox_pkey" PRIMARY KEY, btree (id)
|
||||
"outbox_job_id_idx" btree (job_id)
|
||||
"outbox_published_at_available_at_idx" btree (published_at, available_at)
|
||||
Foreign-key constraints:
|
||||
"outbox_job_id_fkey" FOREIGN KEY (job_id) REFERENCES attachment_jobs(id) ON UPDATE CASCADE ON DELETE CASCADE
|
||||
|
||||
Table "public.job_events"
|
||||
Column | Type | Collation | Nullable | Default
|
||||
------------+-----------------------------+-----------+----------+----------------------------------------
|
||||
id | bigint | | not null | nextval('job_events_id_seq'::regclass)
|
||||
job_id | uuid | | not null |
|
||||
attempt | integer | | not null |
|
||||
event | character varying(32) | | not null |
|
||||
detail | character varying(200) | | |
|
||||
created_at | timestamp(3) with time zone | | not null | CURRENT_TIMESTAMP
|
||||
Indexes:
|
||||
"job_events_pkey" PRIMARY KEY, btree (id)
|
||||
"job_events_job_id_idx" btree (job_id)
|
||||
Foreign-key constraints:
|
||||
"job_events_job_id_fkey" FOREIGN KEY (job_id) REFERENCES attachment_jobs(id) ON UPDATE CASCADE ON DELETE CASCADE
|
||||
|
||||
|
||||
## Миграции: отдельный one-shot сервис, повторный запуск ничего не применяет, схема совпадает с БД
|
||||
$ docker compose run --rm migrate
|
||||
Container messenger-lab6-secrets-init-1 Created
|
||||
Container messenger-lab6-postgres-1 Running
|
||||
Container messenger-lab6-secrets-init-1 Starting
|
||||
Container messenger-lab6-secrets-init-1 Started
|
||||
Container messenger-lab6-secrets-init-1 Waiting
|
||||
Container messenger-lab6-secrets-init-1 Exited
|
||||
Prisma schema loaded from prisma/schema.prisma
|
||||
Datasource "db": PostgreSQL database "messenger", schema "public" at "postgres:5432"
|
||||
|
||||
5 migrations found in prisma/migrations
|
||||
|
||||
|
||||
No pending migrations to apply.
|
||||
|
||||
$ docker compose run --rm --no-deps migrate sh -c export DATABASE_URL="$(cat "$DATABASE_URL_FILE")"; npx prisma migrate diff --from-url "$DATABASE_URL" --to-schema-datamodel prisma/schema.prisma --exit-code
|
||||
No difference detected.
|
||||
|
||||
|
||||
$ docker compose exec -T postgres psql -U messenger -d messenger -c SELECT migration_name, finished_at FROM _prisma_migrations ORDER BY started_at
|
||||
migration_name | finished_at
|
||||
---------------------------------+-------------------------------
|
||||
20261021000000_init | 2026-11-20 14:30:37.860315+00
|
||||
20261021010000_auth_sessions | 2026-11-20 14:30:37.880522+00
|
||||
20261101000000_idempotency_keys | 2026-11-20 14:30:37.895909+00
|
||||
20261105000000_attachments | 2026-11-20 14:30:37.918762+00
|
||||
20261115000000_attachment_jobs | 2026-11-20 14:30:37.951618+00
|
||||
(5 rows)
|
||||
|
||||
|
||||
@@ -0,0 +1,17 @@
|
||||
# make test-failover: stop-one.sh останавливает реплику, обслужившую тест
|
||||
# date: 2026-11-20T14:33:10Z
|
||||
# commit: d83876f
|
||||
|
||||
## до
|
||||
/messenger-lab6-api-1 id=64377e499417 running exit=0 finished=2026-11-20T14:33:08.109900069Z
|
||||
/messenger-lab6-api-2 id=25ff01cef182 running exit=0 finished=2026-11-20T14:33:05.646476717Z
|
||||
|
||||
$ make test-failover ACTION_SCRIPT=scripts/stop-one.sh
|
||||
python3 tests/resilience.py --mode failover --action-script "scripts/stop-one.sh" --base-url "http://lb:8080"
|
||||
Executing explicit failover hook; hook output suppressed to avoid leaking secrets.
|
||||
{"observed_instances_before": 2, "successful_writes": 128, "failed_attempts": 0, "first_recovery_seconds": 3.571}
|
||||
PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.
|
||||
|
||||
## после: остановленная реплика Exited (0) — штатное завершение по SIGTERM
|
||||
/messenger-lab6-api-1 id=64377e499417 running exit=0 finished=2026-11-20T14:33:08.109900069Z
|
||||
/messenger-lab6-api-2 id=25ff01cef182 exited exit=0 finished=2026-11-20T14:33:19.073873959Z
|
||||
@@ -0,0 +1,64 @@
|
||||
# Инцидент: scripts/incidents.py db
|
||||
# date: 2026-11-20T14:41:32Z
|
||||
# commit: d83876f
|
||||
|
||||
$ python3 scripts/incidents.py db
|
||||
[ 0.1s] user df6eb947-ae87-4aa5-9d75-0c0c8091ab4d chat e46df417-75d8-4eaa-bca3-e19e3a86da1a
|
||||
[ 0.1s] == baseline: 60 s of traffic
|
||||
[ 0.2s] p95_1m=16ms 5xx_1m=0.0% pool_wait=0 pool_used=0 core_5xx_1m=- api_cpu=0 MessengerSlowApi=inactive MessengerHighErrorRate=inactive MessengerApiNotReady=inactive
|
||||
[ 20.2s] p95_1m=25ms 5xx_1m=0.0% pool_wait=0 pool_used=0 core_5xx_1m=- api_cpu=0 MessengerSlowApi=inactive MessengerHighErrorRate=inactive MessengerApiNotReady=inactive
|
||||
[ 40.2s] p95_1m=24ms 5xx_1m=0.0% pool_wait=0 pool_used=0 core_5xx_1m=- api_cpu=0.1 MessengerSlowApi=inactive MessengerHighErrorRate=inactive MessengerApiNotReady=inactive
|
||||
[ 60.3s] p95_1m=24ms 5xx_1m=0.0% pool_wait=0 pool_used=0 core_5xx_1m=- api_cpu=0.1 MessengerSlowApi=inactive MessengerHighErrorRate=inactive MessengerApiNotReady=inactive
|
||||
[ 60.3s] client [baseline]: 200:567 201:568; p50=11ms p95=23ms p99=49ms
|
||||
[ 60.3s] == slow handler (not DB): 1700x1700 noise PNG upload — decoding and S3 dominate
|
||||
[ 61.1s] POST 8.3 MiB attachment -> 202
|
||||
[ 76.1s] slow handler: trace 7c5c169b5e229a27d34765e20c7dc102 POST /api/v1/chats/{chat_id}/attachments -> 202, 525 ms
|
||||
[ 76.1s] DB calls 40 ms (pool wait 1 ms, SQL 22 ms); S3 246 ms; handler itself 240 ms
|
||||
[ 76.1s] == slow DB: postgres limited to 0.05 CPU for 150 s
|
||||
[ 76.3s] $ docker update --cpus 0.05 27a9b1f465438247ed81c48a9b572b24ab12ea5ffc7112d9817be18636e47310
|
||||
[ 76.3s] p95_1m=24ms 5xx_1m=0.0% pool_wait=0 pool_used=0 core_5xx_1m=- api_cpu=0.2 MessengerSlowApi=inactive MessengerHighErrorRate=inactive MessengerApiNotReady=inactive
|
||||
[ 91.4s] p95_1m=24ms 5xx_1m=0.0% pool_wait=0 pool_used=0 core_5xx_1m=- api_cpu=0.1 MessengerSlowApi=inactive MessengerHighErrorRate=inactive MessengerApiNotReady=inactive
|
||||
[ 106.4s] p95_1m=75ms 5xx_1m=0.0% pool_wait=0 pool_used=0 core_5xx_1m=- api_cpu=0.1 MessengerSlowApi=inactive MessengerHighErrorRate=inactive MessengerApiNotReady=inactive
|
||||
[ 121.4s] p95_1m=102ms 5xx_1m=0.0% pool_wait=0 pool_used=1 core_5xx_1m=- api_cpu=0.1 MessengerSlowApi=inactive MessengerHighErrorRate=inactive MessengerApiNotReady=inactive
|
||||
[ 136.5s] p95_1m=319ms 5xx_1m=0.0% pool_wait=0 pool_used=1 core_5xx_1m=- api_cpu=0.1 MessengerSlowApi=inactive MessengerHighErrorRate=inactive MessengerApiNotReady=inactive
|
||||
[ 151.5s] p95_1m=309ms 5xx_1m=0.0% pool_wait=0 pool_used=0 core_5xx_1m=- api_cpu=0.1 MessengerSlowApi=inactive MessengerHighErrorRate=inactive MessengerApiNotReady=inactive
|
||||
[ 166.5s] p95_1m=268ms 5xx_1m=0.0% pool_wait=0 pool_used=0 core_5xx_1m=- api_cpu=0.1 MessengerSlowApi=inactive MessengerHighErrorRate=inactive MessengerApiNotReady=inactive
|
||||
[ 181.6s] p95_1m=282ms 5xx_1m=0.0% pool_wait=0 pool_used=0 core_5xx_1m=- api_cpu=0.1 MessengerSlowApi=inactive MessengerHighErrorRate=inactive MessengerApiNotReady=inactive
|
||||
[ 196.6s] p95_1m=180ms 5xx_1m=0.0% pool_wait=0 pool_used=0 core_5xx_1m=- api_cpu=0.1 MessengerSlowApi=inactive MessengerHighErrorRate=inactive MessengerApiNotReady=inactive
|
||||
[ 211.6s] p95_1m=187ms 5xx_1m=0.0% pool_wait=0 pool_used=1 core_5xx_1m=- api_cpu=0.1 MessengerSlowApi=inactive MessengerHighErrorRate=inactive MessengerApiNotReady=inactive
|
||||
[ 226.7s] p95_1m=183ms 5xx_1m=0.0% pool_wait=0 pool_used=0 core_5xx_1m=- api_cpu=0.1 MessengerSlowApi=inactive MessengerHighErrorRate=inactive MessengerApiNotReady=inactive
|
||||
[ 226.7s] client [slow DB]: 200:1391 201:1390; p50=12ms p95=196ms p99=637ms
|
||||
[ 226.7s] slow DB: trace 4fa46b85e073eddb48c09bce00a7b150 POST /api/v1/chats/{chat_id}/messages -> 201, 790 ms
|
||||
[ 226.7s] DB calls 788 ms (pool wait 0 ms, SQL 781 ms); S3 0 ms; handler itself 1 ms
|
||||
[ 226.7s] slow DB: trace 20868735bb9111efc35f20b042032875 POST /api/v1/chats/{chat_id}/messages -> 201, 505 ms
|
||||
[ 226.7s] DB calls 502 ms (pool wait 0 ms, SQL 498 ms); S3 0 ms; handler itself 3 ms
|
||||
[ 226.7s] $ docker update --cpus 1.0 27a9b1f465438247ed81c48a9b572b24ab12ea5ffc7112d9817be18636e47310
|
||||
[ 226.8s] p95_1m=183ms 5xx_1m=0.0% pool_wait=0 pool_used=0 core_5xx_1m=- api_cpu=0.1 MessengerSlowApi=inactive MessengerHighErrorRate=inactive MessengerApiNotReady=inactive
|
||||
[ 226.8s] client [after slow DB]: ; p50=0ms p95=0ms p99=0ms
|
||||
[ 226.8s] == DB unavailable: postgres paused for 120 s
|
||||
[ 226.8s] $ docker pause 27a9b1f465438247ed81c48a9b572b24ab12ea5ffc7112d9817be18636e47310
|
||||
[ 226.9s] p95_1m=183ms 5xx_1m=0.0% pool_wait=0 pool_used=0 core_5xx_1m=- api_cpu=0.1 MessengerSlowApi=inactive MessengerHighErrorRate=inactive MessengerApiNotReady=inactive
|
||||
[ 242.0s] p95_1m=176ms 5xx_1m=0.0% pool_wait=5 pool_used=5 core_5xx_1m=- api_cpu=0.1 MessengerSlowApi=inactive MessengerHighErrorRate=inactive MessengerApiNotReady=inactive
|
||||
[ 257.0s] p95_1m=212ms 5xx_1m=0.0% pool_wait=6 pool_used=0 core_5xx_1m=8 api_cpu=0.1 MessengerSlowApi=inactive MessengerHighErrorRate=inactive MessengerApiNotReady=inactive
|
||||
[ 272.1s] p95_1m=197ms 5xx_1m=0.0% pool_wait=7 pool_used=0 core_5xx_1m=10.9 api_cpu=0 MessengerSlowApi=inactive MessengerHighErrorRate=inactive MessengerApiNotReady=inactive
|
||||
[ 287.1s] p95_1m=nanms 5xx_1m=nan% pool_wait=7 pool_used=0 core_5xx_1m=15.7 api_cpu=0 MessengerSlowApi=inactive MessengerHighErrorRate=inactive MessengerApiNotReady=inactive
|
||||
[ 302.1s] p95_1m=nanms 5xx_1m=nan% pool_wait=6 pool_used=0 core_5xx_1m=18 api_cpu=0 MessengerSlowApi=inactive MessengerHighErrorRate=inactive MessengerApiNotReady=pending
|
||||
[ 317.2s] p95_1m=nanms 5xx_1m=nan% pool_wait=7 pool_used=0 core_5xx_1m=16.8 api_cpu=0 MessengerSlowApi=inactive MessengerHighErrorRate=inactive MessengerApiNotReady=pending
|
||||
[ 332.2s] p95_1m=nanms 5xx_1m=nan% pool_wait=7 pool_used=0 core_5xx_1m=16.8 api_cpu=0 MessengerSlowApi=inactive MessengerHighErrorRate=inactive MessengerApiNotReady=firing
|
||||
[ 332.2s] client [DB paused]: 201:1 503:1910 504:3; p50=41ms p95=41ms p99=41ms
|
||||
[ 332.3s] DB unavailable: trace 96521ffdb7e21480980d75738932c94d GET /api/v1/chats/{chat_id}/messages -> 503, 5005 ms
|
||||
[ 332.3s] DB calls 5001 ms (pool wait 5000 ms, SQL 5000 ms); S3 0 ms; handler itself 3 ms
|
||||
[ 332.3s] spans with error status: GET /api/v1/chats/{chat_id}/messages
|
||||
[ 332.3s] loki: {"timestamp":"2026-11-20T14:47:01.051Z","severity":"ERROR","service.name":"messenger-api","event":"http.error","logger":"HttpExceptionFilter","message":"database unavailable","method":"GET","route":"/
|
||||
[ 332.3s] loki: {"timestamp":"2026-11-20T14:47:01.115Z","severity":"ERROR","service.name":"messenger-api","event":"http.error","logger":"HttpExceptionFilter","message":"database unavailable","method":"GET","route":"/
|
||||
[ 332.3s] loki: {"timestamp":"2026-11-20T14:47:03.054Z","severity":"ERROR","service.name":"messenger-api","event":"http.error","logger":"HttpExceptionFilter","message":"database unavailable","method":"GET","route":"/
|
||||
[ 332.3s] $ docker unpause 27a9b1f465438247ed81c48a9b572b24ab12ea5ffc7112d9817be18636e47310
|
||||
[ 332.4s] p95_1m=nanms 5xx_1m=nan% pool_wait=7 pool_used=0 core_5xx_1m=16.8 api_cpu=0 MessengerSlowApi=inactive MessengerHighErrorRate=inactive MessengerApiNotReady=firing
|
||||
[ 347.5s] p95_1m=24ms 5xx_1m=0.0% pool_wait=0 pool_used=0 core_5xx_1m=13.2 api_cpu=0 MessengerSlowApi=inactive MessengerHighErrorRate=inactive MessengerApiNotReady=firing
|
||||
[ 362.5s] p95_1m=24ms 5xx_1m=0.0% pool_wait=0 pool_used=0 core_5xx_1m=9.6 api_cpu=0.1 MessengerSlowApi=inactive MessengerHighErrorRate=inactive MessengerApiNotReady=firing
|
||||
[ 377.5s] p95_1m=24ms 5xx_1m=0.0% pool_wait=0 pool_used=0 core_5xx_1m=3.6 api_cpu=0.1 MessengerSlowApi=inactive MessengerHighErrorRate=inactive MessengerApiNotReady=firing
|
||||
[ 392.6s] p95_1m=24ms 5xx_1m=0.0% pool_wait=0 pool_used=0 core_5xx_1m=0 api_cpu=0.1 MessengerSlowApi=inactive MessengerHighErrorRate=inactive MessengerApiNotReady=inactive
|
||||
[ 392.6s] client [after DB unpaused]: 200:556 201:556 503:25; p50=11ms p95=23ms p99=42ms
|
||||
[ 393.1s] MessengerApiNotReady fired: True, resolved after recovery: True
|
||||
[ 393.1s] PASS
|
||||
|
||||
@@ -0,0 +1,36 @@
|
||||
# Инцидент: scripts/incidents.py retry-trace
|
||||
# date: 2026-11-20T14:37:50Z
|
||||
# commit: d83876f
|
||||
|
||||
$ python3 scripts/incidents.py retry-trace
|
||||
[ 0.1s] user cd196c28-14b5-40a2-9c7d-c9a822ff9787 chat ef415dfb-4b85-421e-9e6f-45ecba61630e
|
||||
[ 0.1s] $ docker compose stop worker
|
||||
[ 0.8s] upload -> 202 b1a9f7a4-1eeb-4549-8fea-702494f5e4b7 trace d703294a338bfef3bf9f088ecc76f645
|
||||
[ 0.8s] $ docker compose stop s3
|
||||
[ 1.3s] $ docker start worker (without dependencies)
|
||||
[ 6.8s] $ docker start s3 (without dependencies)
|
||||
[ 7.2s] GET attachment -> status=queued error_code=None
|
||||
[ 8.2s] GET attachment -> status=processing error_code=None
|
||||
[ 9.2s] GET attachment -> status=ready error_code=None
|
||||
[ 9.4s] job: d7ba0597-706c-4e41-bf88-353c21ab4e76 | ready | 3 | 5 |
|
||||
[ 9.5s] variants rows: 3
|
||||
[ 9.7s] events: enqueued -> processing -> retry_scheduled -> processing -> retry_scheduled -> processing -> ready
|
||||
[ 24.7s] messenger-api SERVER POST /api/v1/chats/{chat_id}/attachments attempt= 57.8 ms
|
||||
[ 24.7s] messenger-api CLIENT S3.PutObject attempt= 14.8 ms
|
||||
[ 24.7s] messenger-relay PRODUCER attachment.jobs publish attempt=1 4.5 ms
|
||||
[ 24.7s] messenger-worker CONSUMER attachment.jobs process attempt=1 1065.4 ms
|
||||
[ 24.7s] messenger-worker CLIENT S3.GetObject attempt= 1013.7 ms ERROR
|
||||
[ 24.7s] messenger-relay PRODUCER attachment.jobs publish attempt=2 10.5 ms
|
||||
[ 24.7s] messenger-worker CONSUMER attachment.jobs process attempt=2 1028.0 ms
|
||||
[ 24.7s] messenger-worker CLIENT S3.GetObject attempt= 1003.1 ms ERROR
|
||||
[ 24.7s] messenger-relay PRODUCER attachment.jobs publish attempt=3 7.4 ms
|
||||
[ 24.7s] messenger-worker CONSUMER attachment.jobs process attempt=3 1052.5 ms
|
||||
[ 24.7s] messenger-worker CLIENT S3.GetObject attempt= 969.5 ms
|
||||
[ 24.7s] messenger-worker CLIENT S3.PutObject attempt= 13.8 ms
|
||||
[ 24.7s] messenger-worker CLIENT S3.PutObject attempt= 6.5 ms
|
||||
[ 24.7s] messenger-worker CLIENT S3.PutObject attempt= 5.9 ms
|
||||
[ 24.7s] messenger-worker CLIENT S3.PutObject attempt= 6.0 ms
|
||||
[ 24.7s] messenger-worker CLIENT S3.DeleteObject attempt= 3.5 ms
|
||||
[ 24.7s] consumer spans in one trace: 3 (attempts ['1', '2', '3'])
|
||||
[ 24.7s] PASS
|
||||
|
||||
@@ -0,0 +1,59 @@
|
||||
# Инцидент: scripts/incidents.py telemetry
|
||||
# date: 2026-11-20T15:12:09Z
|
||||
# commit: 2718c3b
|
||||
|
||||
$ python3 scripts/incidents.py telemetry
|
||||
[ 0.1s] user 82d06d15-aef5-49d9-b445-a93d0dd4b886 chat a583d7a0-7c8c-4261-971b-8affca86fb22
|
||||
[ 0.2s] spans_failed_1m=0 logs_failed_1m=0 queue=0 queue_cap=200 refused_1m=0 collector_rss=169MiB api_rss=202MiB TelemetryExportFailing=inactive MessengerHighErrorRate=inactive
|
||||
[ 15.3s] spans_failed_1m=0 logs_failed_1m=0 queue=0 queue_cap=200 refused_1m=0 collector_rss=169MiB api_rss=198MiB TelemetryExportFailing=inactive MessengerHighErrorRate=inactive
|
||||
[ 30.3s] spans_failed_1m=0 logs_failed_1m=0 queue=0 queue_cap=200 refused_1m=0 collector_rss=188MiB api_rss=193MiB TelemetryExportFailing=inactive MessengerHighErrorRate=inactive
|
||||
[ 30.3s] client [baseline]: 200:1350 201:1353; p50=11ms p95=53ms p99=110ms
|
||||
[ 30.3s] == Loki and Tempo stopped until TelemetryExportFailing fires (max 240 s)
|
||||
[ 30.3s] $ docker compose stop loki tempo
|
||||
[ 32.4s] spans_failed_1m=0 logs_failed_1m=0 queue=0 queue_cap=200 refused_1m=0 collector_rss=188MiB api_rss=193MiB TelemetryExportFailing=inactive MessengerHighErrorRate=inactive
|
||||
[ 47.5s] spans_failed_1m=0 logs_failed_1m=0 queue=6 queue_cap=200 refused_1m=0 collector_rss=188MiB api_rss=196MiB TelemetryExportFailing=inactive MessengerHighErrorRate=inactive
|
||||
[ 62.6s] spans_failed_1m=0 logs_failed_1m=0 queue=46 queue_cap=200 refused_1m=0 collector_rss=194MiB api_rss=195MiB TelemetryExportFailing=inactive MessengerHighErrorRate=inactive
|
||||
[ 77.6s] spans_failed_1m=0 logs_failed_1m=0 queue=86 queue_cap=200 refused_1m=0 collector_rss=209MiB api_rss=194MiB TelemetryExportFailing=inactive MessengerHighErrorRate=inactive
|
||||
[ 92.7s] spans_failed_1m=0 logs_failed_1m=0 queue=128 queue_cap=200 refused_1m=0 collector_rss=226MiB api_rss=194MiB TelemetryExportFailing=inactive MessengerHighErrorRate=inactive
|
||||
[ 107.7s] spans_failed_1m=674.2 logs_failed_1m=529.4 queue=168 queue_cap=200 refused_1m=0 collector_rss=234MiB api_rss=195MiB TelemetryExportFailing=inactive MessengerHighErrorRate=inactive
|
||||
[ 122.7s] spans_failed_1m=1349.4 logs_failed_1m=1096.5 queue=199 queue_cap=200 refused_1m=5892.3 collector_rss=247MiB api_rss=199MiB TelemetryExportFailing=pending MessengerHighErrorRate=inactive
|
||||
[ 137.8s] spans_failed_1m=1350.5 logs_failed_1m=1097.4 queue=200 queue_cap=200 refused_1m=37471.3 collector_rss=286MiB api_rss=199MiB TelemetryExportFailing=pending MessengerHighErrorRate=inactive
|
||||
[ 152.8s] spans_failed_1m=682.7 logs_failed_1m=573.4 queue=200 queue_cap=200 refused_1m=65169.9 collector_rss=286MiB api_rss=201MiB TelemetryExportFailing=pending MessengerHighErrorRate=inactive
|
||||
[ 167.9s] spans_failed_1m=1365.5 logs_failed_1m=589.4 queue=200 queue_cap=200 refused_1m=84554.2 collector_rss=287MiB api_rss=196MiB TelemetryExportFailing=pending MessengerHighErrorRate=inactive
|
||||
[ 182.9s] spans_failed_1m=1365.4 logs_failed_1m=1224.1 queue=200 queue_cap=200 refused_1m=84860.3 collector_rss=287MiB api_rss=196MiB TelemetryExportFailing=firing MessengerHighErrorRate=inactive
|
||||
[ 182.9s] client [Loki+Tempo down]: 200:6951 201:6950; p50=11ms p95=43ms p99=106ms
|
||||
[ 182.9s] $ docker compose start loki tempo
|
||||
[ 183.6s] spans_failed_1m=1365.4 logs_failed_1m=1224.1 queue=200 queue_cap=200 refused_1m=88988.6 collector_rss=287MiB api_rss=196MiB TelemetryExportFailing=firing MessengerHighErrorRate=inactive
|
||||
[ 198.6s] spans_failed_1m=0 logs_failed_1m=634.7 queue=0 queue_cap=200 refused_1m=76515.4 collector_rss=266MiB api_rss=176MiB TelemetryExportFailing=firing MessengerHighErrorRate=inactive
|
||||
[ 213.6s] spans_failed_1m=0 logs_failed_1m=0 queue=0 queue_cap=200 refused_1m=44296.3 collector_rss=200MiB api_rss=175MiB TelemetryExportFailing=firing MessengerHighErrorRate=inactive
|
||||
[ 228.7s] spans_failed_1m=0 logs_failed_1m=0 queue=0 queue_cap=200 refused_1m=12213.9 collector_rss=196MiB api_rss=174MiB TelemetryExportFailing=firing MessengerHighErrorRate=inactive
|
||||
[ 243.7s] spans_failed_1m=0 logs_failed_1m=0 queue=0 queue_cap=200 refused_1m=0 collector_rss=194MiB api_rss=174MiB TelemetryExportFailing=firing MessengerHighErrorRate=inactive
|
||||
[ 258.7s] spans_failed_1m=0 logs_failed_1m=0 queue=0 queue_cap=200 refused_1m=0 collector_rss=194MiB api_rss=173MiB TelemetryExportFailing=firing MessengerHighErrorRate=inactive
|
||||
[ 273.8s] spans_failed_1m=0 logs_failed_1m=0 queue=0 queue_cap=200 refused_1m=0 collector_rss=194MiB api_rss=171MiB TelemetryExportFailing=firing MessengerHighErrorRate=inactive
|
||||
[ 288.8s] spans_failed_1m=0 logs_failed_1m=0 queue=0 queue_cap=200 refused_1m=0 collector_rss=194MiB api_rss=174MiB TelemetryExportFailing=firing MessengerHighErrorRate=inactive
|
||||
[ 303.9s] spans_failed_1m=0 logs_failed_1m=0 queue=0 queue_cap=200 refused_1m=0 collector_rss=194MiB api_rss=172MiB TelemetryExportFailing=inactive MessengerHighErrorRate=inactive
|
||||
[ 303.9s] client [after start]: 200:5660 201:5663; p50=10ms p95=36ms p99=93ms
|
||||
[ 303.9s] == Collector stopped for 300 s: SDK export fails fast (5 s timeout, bounded queues); RSS saw-tooth until GC
|
||||
[ 303.9s] $ docker compose stop otel-collector
|
||||
[ 327.6s] docker stats: messenger-lab6-api-1 183.4MiB / 256MiB; messenger-lab6-api-2 173.5MiB / 256MiB; messenger-lab6-worker-1 58.01MiB / 768MiB;
|
||||
[ 350.4s] docker stats: messenger-lab6-api-1 192.4MiB / 256MiB; messenger-lab6-api-2 183.2MiB / 256MiB; messenger-lab6-worker-1 58.72MiB / 768MiB;
|
||||
[ 373.2s] docker stats: messenger-lab6-api-1 192.5MiB / 256MiB; messenger-lab6-api-2 183.4MiB / 256MiB; messenger-lab6-worker-1 58.29MiB / 768MiB;
|
||||
[ 396.1s] docker stats: messenger-lab6-api-1 193.4MiB / 256MiB; messenger-lab6-api-2 186.8MiB / 256MiB; messenger-lab6-worker-1 58.77MiB / 768MiB;
|
||||
[ 419.1s] docker stats: messenger-lab6-api-1 194.8MiB / 256MiB; messenger-lab6-api-2 189.2MiB / 256MiB; messenger-lab6-worker-1 58.55MiB / 768MiB;
|
||||
[ 441.7s] docker stats: messenger-lab6-api-1 198MiB / 256MiB; messenger-lab6-api-2 189.1MiB / 256MiB; messenger-lab6-worker-1 58.1MiB / 768MiB;
|
||||
[ 464.6s] docker stats: messenger-lab6-api-1 198.4MiB / 256MiB; messenger-lab6-api-2 191.2MiB / 256MiB; messenger-lab6-worker-1 58.59MiB / 768MiB;
|
||||
[ 487.5s] docker stats: messenger-lab6-api-1 201.1MiB / 256MiB; messenger-lab6-api-2 193.2MiB / 256MiB; messenger-lab6-worker-1 58.2MiB / 768MiB;
|
||||
[ 509.6s] docker stats: messenger-lab6-api-1 202.7MiB / 256MiB; messenger-lab6-api-2 190.5MiB / 256MiB; messenger-lab6-worker-1 56.81MiB / 768MiB;
|
||||
[ 531.6s] docker stats: messenger-lab6-api-1 203.5MiB / 256MiB; messenger-lab6-api-2 193.2MiB / 256MiB; messenger-lab6-worker-1 57.3MiB / 768MiB;
|
||||
[ 554.5s] docker stats: messenger-lab6-api-1 206.1MiB / 256MiB; messenger-lab6-api-2 195.3MiB / 256MiB; messenger-lab6-worker-1 56.75MiB / 768MiB;
|
||||
[ 577.4s] docker stats: messenger-lab6-api-1 206.5MiB / 256MiB; messenger-lab6-api-2 191.7MiB / 256MiB; messenger-lab6-worker-1 56.92MiB / 768MiB;
|
||||
[ 599.7s] docker stats: messenger-lab6-api-1 208.1MiB / 256MiB; messenger-lab6-api-2 193.7MiB / 256MiB; messenger-lab6-worker-1 56.79MiB / 768MiB;
|
||||
[ 621.7s] docker stats: messenger-lab6-api-1 208.2MiB / 256MiB; messenger-lab6-api-2 194.9MiB / 256MiB; messenger-lab6-worker-1 56.46MiB / 768MiB;
|
||||
[ 644.6s] docker stats: messenger-lab6-api-1 210.6MiB / 256MiB; messenger-lab6-api-2 197.3MiB / 256MiB; messenger-lab6-worker-1 56.95MiB / 768MiB;
|
||||
[ 644.6s] client [collector down]: 200:15451 201:15453; p50=11ms p95=38ms p99=131ms
|
||||
[ 644.6s] $ docker compose start otel-collector
|
||||
[ 675.2s] client [collector back]: 200:1376 201:1372; p50=11ms p95=49ms p99=133ms
|
||||
[ 675.7s] containers: /messenger-lab6-api-1 restarts=0 oom=false; /messenger-lab6-api-2 restarts=0 oom=false; /messenger-lab6-worker-1 restarts=0 oom=false
|
||||
[ 675.7s] TelemetryExportFailing fired: True, resolved: True
|
||||
[ 675.7s] PASS
|
||||
|
||||
@@ -0,0 +1,47 @@
|
||||
# Инцидент: scripts/incidents.py worker-stop
|
||||
# date: 2026-11-20T14:38:15Z
|
||||
# commit: d83876f
|
||||
|
||||
$ python3 scripts/incidents.py worker-stop
|
||||
[ 0.1s] user d494cd19-eea4-48ae-a471-64b35e821933 chat 0369989a-4572-4abd-8dd4-7aafb56e9d84
|
||||
[ 0.1s] $ docker compose stop worker
|
||||
[ 0.7s] due=0 processing=0 oldest_age_s=0 worker_busy=0 worker_up=1 ready_per_s=0 MessengerJobQueueLag=inactive
|
||||
[ 0.7s] POST attachment -> 202 status=queued id=2a8a08ff-2af0-47be-ac68-9404ea8c0f70 instance=25ff01cef182
|
||||
[ 10.7s] due=1 processing=0 oldest_age_s=0 worker_busy=0 worker_up=1 ready_per_s=0 MessengerJobQueueLag=inactive
|
||||
[ 10.8s] POST attachment -> 202 status=queued id=90500e1c-2b31-45d7-b584-a3df08532497 instance=64377e499417
|
||||
[ 20.8s] due=2 processing=0 oldest_age_s=2.5 worker_busy=0 worker_up=1 ready_per_s=0 MessengerJobQueueLag=inactive
|
||||
[ 20.8s] POST attachment -> 202 status=queued id=d388cce7-ef51-4bc6-a990-c9f7c25934cc instance=25ff01cef182
|
||||
[ 30.9s] due=3 processing=0 oldest_age_s=2.5 worker_busy=0 worker_up=- ready_per_s=0 MessengerJobQueueLag=inactive
|
||||
[ 30.9s] POST attachment -> 202 status=queued id=93a3b0d8-3a39-47ac-8bc0-79fa17c5aed0 instance=64377e499417
|
||||
[ 40.9s] due=4 processing=0 oldest_age_s=12.5 worker_busy=0 worker_up=- ready_per_s=0 MessengerJobQueueLag=inactive
|
||||
[ 41.0s] POST attachment -> 202 status=queued id=26b1886f-f54c-4fff-aea7-ebddbeba4662 instance=25ff01cef182
|
||||
[ 51.0s] due=5 processing=0 oldest_age_s=32.5 worker_busy=0 worker_up=- ready_per_s=0 MessengerJobQueueLag=inactive
|
||||
[ 51.1s] POST attachment -> 202 status=queued id=43da824d-5d9f-4ecb-9621-58a1d0ff0f73 instance=64377e499417
|
||||
[ 61.1s] due=6 processing=0 oldest_age_s=32.5 worker_busy=0 worker_up=- ready_per_s=- MessengerJobQueueLag=inactive
|
||||
[ 61.1s] POST attachment -> 202 status=queued id=1810e49b-e341-4820-8d01-40151a8ed795 instance=25ff01cef182
|
||||
[ 71.2s] due=7 processing=0 oldest_age_s=42.5 worker_busy=0 worker_up=- ready_per_s=- MessengerJobQueueLag=inactive
|
||||
[ 71.2s] POST attachment -> 202 status=queued id=9884d9e1-6e5b-4e2b-953e-c08af021adfd instance=64377e499417
|
||||
[ 81.2s] due=8 processing=0 oldest_age_s=62.5 worker_busy=0 worker_up=- ready_per_s=- MessengerJobQueueLag=inactive
|
||||
[ 81.3s] POST attachment -> 202 status=queued id=cb6af66e-bc34-41d8-ba90-1f5664611708 instance=25ff01cef182
|
||||
[ 91.3s] due=9 processing=0 oldest_age_s=62.5 worker_busy=0 worker_up=- ready_per_s=- MessengerJobQueueLag=pending
|
||||
[ 91.3s] POST attachment -> 202 status=queued id=614d8080-709e-4574-96c5-8d5b2ebe66f8 instance=64377e499417
|
||||
[ 101.4s] due=10 processing=0 oldest_age_s=72.5 worker_busy=0 worker_up=- ready_per_s=- MessengerJobQueueLag=pending
|
||||
[ 101.4s] POST attachment -> 202 status=queued id=5b201a4e-3f9e-4927-b282-10530a4c2be2 instance=25ff01cef182
|
||||
[ 111.4s] due=11 processing=0 oldest_age_s=92.5 worker_busy=0 worker_up=- ready_per_s=- MessengerJobQueueLag=pending
|
||||
[ 111.5s] POST attachment -> 202 status=queued id=607a53d8-9583-4d79-93c4-c87f320ad7c6 instance=64377e499417
|
||||
[ 121.5s] due=12 processing=0 oldest_age_s=92.5 worker_busy=0 worker_up=- ready_per_s=- MessengerJobQueueLag=pending
|
||||
[ 131.5s] due=12 processing=0 oldest_age_s=102.5 worker_busy=0 worker_up=- ready_per_s=- MessengerJobQueueLag=pending
|
||||
[ 141.6s] due=12 processing=0 oldest_age_s=122.5 worker_busy=0 worker_up=- ready_per_s=- MessengerJobQueueLag=pending
|
||||
[ 151.6s] due=12 processing=0 oldest_age_s=122.5 worker_busy=0 worker_up=- ready_per_s=- MessengerJobQueueLag=firing
|
||||
[ 151.6s] alert MessengerJobQueueLag firing after 151 s of worker downtime: True
|
||||
[ 151.6s] $ docker start worker (without dependencies)
|
||||
[ 152.0s] due=12 processing=0 oldest_age_s=122.5 worker_busy=0 worker_up=- ready_per_s=- MessengerJobQueueLag=firing
|
||||
[ 162.0s] due=2 processing=0 oldest_age_s=132.5 worker_busy=0 worker_up=- ready_per_s=- MessengerJobQueueLag=firing
|
||||
[ 172.0s] due=0 processing=0 oldest_age_s=51.8 worker_busy=0 worker_up=1 ready_per_s=- MessengerJobQueueLag=firing
|
||||
[ 182.0s] due=0 processing=0 oldest_age_s=51.8 worker_busy=0 worker_up=1 ready_per_s=0 MessengerJobQueueLag=inactive
|
||||
[ 182.0s] backlog drained and alert resolved 30 s after worker start: True
|
||||
[ 182.1s] uploads during downtime: 12, ready now: 12
|
||||
[ 197.1s] time to ready, histogram over 5m: p50=100 s, p99=176.4 s
|
||||
[ 197.4s] time to ready from the database (finished_at - created_at), s: 41.9 .. 152.1
|
||||
[ 197.4s] PASS
|
||||
|
||||
@@ -0,0 +1,214 @@
|
||||
# Фоновая обработка: сценарии защиты уровней 3 и 4 (scripts/job-scenario.py)
|
||||
# date: 2026-11-20T14:33:45Z
|
||||
# commit: d83876f
|
||||
|
||||
## restart
|
||||
$ python3 scripts/job-scenario.py restart
|
||||
[ 0.2s] user 83a421c6-238a-430e-b23d-7065f92692ea chat 37e2da02-e39f-4a5b-bc3e-99e3072079ad
|
||||
[ 0.2s] $ docker compose stop worker
|
||||
[ 1.2s] POST attachment -> 202 status=queued id=b5032f63-71ca-4684-8071-8e546a2b1d38 instance=64377e499417
|
||||
[ 1.2s] content while queued -> 409
|
||||
[ 1.2s] POST message with 1 attachment(s) -> 201 id=3133055e-c217-4956-8b29-6c195ad7d832
|
||||
[ 1.2s] POST message with 0 attachment(s) -> 201 id=485a4b3d-e797-4cd8-8c3a-b6fbd22e51fa
|
||||
[ 10.3s] queues: name type durable messages_ready messages_unacknowledged
|
||||
attachment.jobs quorum true 1 0
|
||||
attachment.jobs.dlq quorum true 0 0
|
||||
[ 10.3s] $ docker compose restart api rabbitmq
|
||||
[ 27.8s] queues after broker restart: name type durable messages_ready messages_unacknowledged
|
||||
attachment.jobs quorum true 1 0
|
||||
attachment.jobs.dlq quorum true 0 0
|
||||
[ 27.8s] status after restarts: queued
|
||||
[ 27.8s] $ docker compose start worker
|
||||
[ 32.3s] GET attachment -> status=queued error_code=None
|
||||
[ 33.3s] GET attachment -> status=ready error_code=None
|
||||
[ 33.3s] original -> 200 sha256 matches upload: True
|
||||
[ 33.4s] thumbnail -> 200 image/jpeg 256x160 sha256 matches metadata: True
|
||||
[ 33.4s] crop -> 200 image/jpeg 128x128 sha256 matches metadata: True
|
||||
[ 33.4s] optimized -> 200 image/webp 320x200 sha256 matches metadata: True
|
||||
[ 33.6s] job: 4e569601-78cf-43c7-8e00-d11cb4929ef8 | ready | 1 | 5 |
|
||||
[ 33.7s] variants rows: 3
|
||||
[ 33.9s] events: enqueued -> processing -> ready
|
||||
|
||||
## crash-after-store
|
||||
$ python3 scripts/job-scenario.py crash-after-store
|
||||
[ 0.1s] user 00aee8d4-3de1-4190-ba59-145bbd9ab2e3 chat eb05fa1f-3a6f-412b-a924-23472af26d2b
|
||||
[ 0.1s] $ docker compose stop worker
|
||||
[ 0.7s] $ docker compose run --no-deps -e WORKER_FAULT=exit-after-store -e JOB_LEASE_SECONDS=15 worker
|
||||
[ 2.7s] POST attachment -> 202 status=queued id=6c09adc1-6c17-43ad-a8ed-2fa6582fe5c5 instance=25ff01cef182
|
||||
[ 3.4s] one-off: {"timestamp":"2026-11-20T14:34:22.942Z","severity":"WARN","service.name":"messenger-worker","event":"worker","logger":"Worker","message":"fault injection: worker killed after storing objects, before commit and ack","trace_id":"0f573fa6f35061a106d47bf5eb093c3d","span_id":"39a42f14e6f1823b"}
|
||||
[ 3.6s] one-off process exit code 137 (137 = killed without ack)
|
||||
[ 3.6s] status after crash: processing
|
||||
[ 3.9s] lease: processing | 1 | t
|
||||
[ 11.7s] queues (unacked message returned): name type durable messages_ready messages_unacknowledged
|
||||
attachment.jobs quorum true 1 0
|
||||
attachment.jobs.dlq quorum true 0 0
|
||||
[ 11.7s] $ docker compose start worker
|
||||
[ 16.8s] GET attachment -> status=processing error_code=None
|
||||
[ 19.8s] GET attachment -> status=queued error_code=None
|
||||
[ 20.8s] GET attachment -> status=processing error_code=None
|
||||
[ 21.9s] GET attachment -> status=ready error_code=None
|
||||
[ 21.9s] original -> 200 sha256 matches upload: True
|
||||
[ 21.9s] thumbnail -> 200 image/jpeg 256x160 sha256 matches metadata: True
|
||||
[ 21.9s] crop -> 200 image/jpeg 128x128 sha256 matches metadata: True
|
||||
[ 21.9s] optimized -> 200 image/webp 320x200 sha256 matches metadata: True
|
||||
[ 22.0s] job: d619929c-d660-4311-bb5e-0d3fbc1813ef | ready | 2 | 5 |
|
||||
[ 22.2s] variants rows: 3
|
||||
[ 22.3s] events: enqueued -> processing -> lease_expired -> processing -> ready
|
||||
[ 22.6s] worker: {"timestamp":"2026-11-20T14:34:37.365Z","severity":"INFO","service.name":"messenger-worker","event":"job_processor","logger":"JobProcessor","message":"job d619929c-d660-4311-bb5e-0d3fbc1813ef: processing, not due or leased elsewhere, skipped","trace_id":"0f573fa6f35061a106d47bf5eb093c3d","span_id":"17a7f5d766544282"}
|
||||
[ 22.6s] worker: {"timestamp":"2026-11-20T14:34:37.366Z","severity":"INFO","service.name":"messenger-worker","event":"job.handled","logger":"Worker","job_id":"d619929c-d660-4311-bb5e-0d3fbc1813ef","attachment_id":"6c09adc1-6c17-43ad-a8ed-2fa6582fe5c5","attempt":1,"redelivered":true,"outcome":"skipped","duration_ms":25,"trace_id":"0f573fa6f35061a106d47bf5eb093c3d","span_id":"17a7f5d766544282"}
|
||||
[ 22.6s] worker: {"timestamp":"2026-11-20T14:34:40.735Z","severity":"INFO","service.name":"messenger-worker","event":"job.handled","logger":"Worker","job_id":"d619929c-d660-4311-bb5e-0d3fbc1813ef","attachment_id":"6c09adc1-6c17-43ad-a8ed-2fa6582fe5c5","attempt":2,"redelivered":false,"outcome":"success","duration_ms":159}
|
||||
[ 22.7s] relay: {"timestamp":"2026-11-20T14:34:39.557Z","severity":"INFO","service.name":"messenger-relay","event":"outbox_relay","logger":"OutboxRelay","message":"lease expired: 1, stale queued republished: 0"}
|
||||
|
||||
## crash-after-commit
|
||||
$ python3 scripts/job-scenario.py crash-after-commit
|
||||
[ 0.1s] user 34fae015-fbb7-48eb-ad9d-8f23ee67784c chat 801de767-107d-4eb7-92ff-b6aa38dc95cf
|
||||
[ 0.1s] $ docker compose stop worker
|
||||
[ 0.6s] $ docker compose run --no-deps -e WORKER_FAULT=exit-after-commit worker
|
||||
[ 2.7s] POST attachment -> 202 status=queued id=e02d5568-ce5b-485d-9649-f9b4d7387d41 instance=25ff01cef182
|
||||
[ 3.4s] one-off: {"timestamp":"2026-11-20T14:34:45.757Z","severity":"WARN","service.name":"messenger-worker","event":"worker","logger":"Worker","message":"fault injection: worker killed after commit, before ack","trace_id":"e15e253daf15a01b76410070a68e7aff","span_id":"b5b5eb6d044c2f38"}
|
||||
[ 3.5s] one-off process exit code 137 (137 = killed without ack)
|
||||
[ 3.9s] status after crash: ready staging_key: staging/e02d5568-ce5b-485d-9649-f9b4d7387d41
|
||||
[ 11.7s] queues (unacked message returned): name type durable messages_ready messages_unacknowledged
|
||||
attachment.jobs quorum true 1 0
|
||||
attachment.jobs.dlq quorum true 0 0
|
||||
[ 11.7s] $ docker compose start worker
|
||||
[ 17.6s] staging_key after redelivery: NULL
|
||||
[ 17.6s] original -> 200 sha256 matches upload: True
|
||||
[ 17.6s] thumbnail -> 200 image/jpeg 256x160 sha256 matches metadata: True
|
||||
[ 17.6s] crop -> 200 image/jpeg 128x128 sha256 matches metadata: True
|
||||
[ 17.6s] optimized -> 200 image/webp 320x200 sha256 matches metadata: True
|
||||
[ 17.8s] job: dbc6450f-1513-4fd0-9923-3c3339985866 | ready | 1 | 5 |
|
||||
[ 18.0s] variants rows: 3
|
||||
[ 18.2s] events: enqueued -> processing -> ready
|
||||
[ 18.5s] worker: {"timestamp":"2026-11-20T14:34:59.494Z","severity":"INFO","service.name":"messenger-worker","event":"job_processor","logger":"JobProcessor","message":"job dbc6450f-1513-4fd0-9923-3c3339985866: ready, not due or leased elsewhere, skipped","trace_id":"e15e253daf15a01b76410070a68e7aff","span_id":"db61235444dfbbbc"}
|
||||
[ 18.5s] worker: {"timestamp":"2026-11-20T14:34:59.494Z","severity":"INFO","service.name":"messenger-worker","event":"job.handled","logger":"Worker","job_id":"dbc6450f-1513-4fd0-9923-3c3339985866","attachment_id":"e02d5568-ce5b-485d-9649-f9b4d7387d41","attempt":1,"redelivered":true,"outcome":"skipped","duration_ms":50,"trace_id":"e15e253daf15a01b76410070a68e7aff","span_id":"db61235444dfbbbc"}
|
||||
|
||||
## outbox-gap
|
||||
$ python3 scripts/job-scenario.py outbox-gap
|
||||
[ 0.1s] user 8646554f-5ce9-4e60-951f-9403d8e951d1 chat 196660eb-f3fc-4117-96dd-9287c6c85681
|
||||
[ 0.1s] $ docker compose stop relay
|
||||
[ 0.9s] POST attachment -> 202 status=queued id=14e198ac-f973-4692-9267-8dba3a1acbc0 instance=64377e499417
|
||||
[ 1.1s] outbox after COMMIT: 8 | NOT PUBLISHED | 0
|
||||
[ 8.8s] queues: name type durable messages_ready messages_unacknowledged
|
||||
attachment.jobs quorum true 0 0
|
||||
attachment.jobs.dlq quorum true 0 0
|
||||
[ 8.8s] $ docker compose kill api
|
||||
[ 9.4s] API killed (SIGKILL) after COMMIT, nothing was published
|
||||
[ 9.4s] $ docker compose start api
|
||||
[ 18.6s] status after API restart: queued
|
||||
[ 18.6s] $ docker compose start relay
|
||||
[ 22.9s] GET attachment -> status=queued error_code=None
|
||||
[ 26.0s] GET attachment -> status=ready error_code=None
|
||||
[ 26.2s] outbox after relay recovery: 8 | 2026-11-20 14:35:26.35+00 | 1
|
||||
[ 26.2s] original -> 200 sha256 matches upload: True
|
||||
[ 26.2s] thumbnail -> 200 image/jpeg 256x160 sha256 matches metadata: True
|
||||
[ 26.2s] crop -> 200 image/jpeg 128x128 sha256 matches metadata: True
|
||||
[ 26.3s] optimized -> 200 image/webp 320x200 sha256 matches metadata: True
|
||||
[ 26.4s] job: 29554704-45b1-4335-87fe-8bcb0621b000 | ready | 1 | 5 |
|
||||
[ 26.6s] variants rows: 3
|
||||
[ 26.7s] events: enqueued -> processing -> ready
|
||||
|
||||
## relay-crash
|
||||
$ python3 scripts/job-scenario.py relay-crash
|
||||
[ 0.1s] user da9df65a-2adb-4552-b09a-063e143f7e42 chat 59b25b73-439e-4eb9-abf5-51a8851b5dd5
|
||||
[ 0.1s] $ docker compose stop relay
|
||||
[ 0.7s] $ docker compose run --no-deps -e RELAY_FAULT=exit-after-confirm relay
|
||||
[ 3.7s] POST attachment -> 202 status=queued id=95f87871-7463-4cec-b73e-556b32ce1f57 instance=64377e499417
|
||||
[ 4.3s] one-off: {"timestamp":"2026-11-20T14:35:31.985Z","severity":"INFO","service.name":"messenger-relay","event":"outbox_relay","logger":"OutboxRelay","message":"published 1 job message(s), broker confirmed"}
|
||||
[ 4.3s] one-off: {"timestamp":"2026-11-20T14:35:31.985Z","severity":"WARN","service.name":"messenger-relay","event":"outbox_relay","logger":"OutboxRelay","message":"fault injection: exit after broker confirm, before outbox is marked"}
|
||||
[ 4.4s] one-off process exit code 137 (137 = killed without ack)
|
||||
[ 4.8s] outbox after relay crash: 9 | NOT MARKED | 0
|
||||
[ 4.8s] $ docker compose start relay
|
||||
[ 9.6s] GET attachment -> status=ready error_code=None
|
||||
[ 12.8s] outbox after relay restart: 9 | 2026-11-20 14:35:39.707+00 | 1
|
||||
[ 12.8s] original -> 200 sha256 matches upload: True
|
||||
[ 12.8s] thumbnail -> 200 image/jpeg 256x160 sha256 matches metadata: True
|
||||
[ 12.8s] crop -> 200 image/jpeg 128x128 sha256 matches metadata: True
|
||||
[ 12.8s] optimized -> 200 image/webp 320x200 sha256 matches metadata: True
|
||||
[ 13.0s] job: 5449c7df-1ee6-435d-a8e4-60b0f672d65e | ready | 1 | 5 |
|
||||
[ 13.1s] variants rows: 3
|
||||
[ 13.4s] events: enqueued -> processing -> ready
|
||||
[ 13.6s] worker: {"timestamp":"2026-11-20T14:35:32.064Z","severity":"INFO","service.name":"messenger-worker","event":"job.handled","logger":"Worker","job_id":"5449c7df-1ee6-435d-a8e4-60b0f672d65e","attachment_id":"95f87871-7463-4cec-b73e-556b32ce1f57","attempt":1,"redelivered":false,"outcome":"success","duration_ms":79,"trace_id":"c24603daec760a57d467aff4b2a1d515","span_id":"99e7c21fe5d815b2"}
|
||||
[ 13.6s] worker: {"timestamp":"2026-11-20T14:35:39.712Z","severity":"INFO","service.name":"messenger-worker","event":"job_processor","logger":"JobProcessor","message":"job 5449c7df-1ee6-435d-a8e4-60b0f672d65e: ready, not due or leased elsewhere, skipped","trace_id":"c24603daec760a57d467aff4b2a1d515","span_id":"ee8d2b5d22aac1b4"}
|
||||
[ 13.6s] worker: {"timestamp":"2026-11-20T14:35:39.713Z","severity":"INFO","service.name":"messenger-worker","event":"job.handled","logger":"Worker","job_id":"5449c7df-1ee6-435d-a8e4-60b0f672d65e","attachment_id":"95f87871-7463-4cec-b73e-556b32ce1f57","attempt":1,"redelivered":false,"outcome":"skipped","duration_ms":6,"trace_id":"c24603daec760a57d467aff4b2a1d515","span_id":"ee8d2b5d22aac1b4"}
|
||||
|
||||
## retry-replay
|
||||
$ python3 scripts/job-scenario.py retry-replay
|
||||
[ 0.1s] user beb0f294-3e65-411c-bda5-ce3f29e2c1d7 chat a253ff82-2c8c-457b-85cf-6b8c9beee260
|
||||
[ 0.1s] $ docker compose stop worker
|
||||
[ 0.7s] POST attachment -> 202 status=queued id=9832fbf4-2f69-484e-87e4-463d3e8ef3a7 instance=64377e499417
|
||||
[ 0.7s] $ docker compose stop s3
|
||||
[ 1.2s] $ docker start worker (without dependencies)
|
||||
[ 1.7s] GET attachment -> status=queued error_code=None
|
||||
[ 2.7s] GET attachment -> status=processing error_code=None
|
||||
[ 3.7s] GET attachment -> status=queued error_code=None
|
||||
[ 4.8s] GET attachment -> status=processing error_code=None
|
||||
[ 5.8s] GET attachment -> status=queued error_code=None
|
||||
[ 7.8s] GET attachment -> status=processing error_code=None
|
||||
[ 8.8s] GET attachment -> status=queued error_code=None
|
||||
[ 12.8s] GET attachment -> status=processing error_code=None
|
||||
[ 13.8s] GET attachment -> status=queued error_code=None
|
||||
[ 19.9s] GET attachment -> status=processing error_code=None
|
||||
[ 20.9s] GET attachment -> status=failed error_code=retry_exhausted
|
||||
[ 20.9s] POST message with 1 attachment(s) -> 409 id=None
|
||||
[ 20.9s] failed attachment in a new message -> 409
|
||||
[ 20.9s] retries:
|
||||
[ 21.0s] 1 | enqueued | | 14:35:42.501
|
||||
[ 21.0s] 1 | processing | | 14:35:44.160
|
||||
[ 21.0s] 1 | retry_scheduled | temporary failure: ApiError; next attempt 2 in 860 ms | 14:35:45.201
|
||||
[ 21.0s] 2 | processing | | 14:35:46.335
|
||||
[ 21.0s] 2 | retry_scheduled | temporary failure: ApiError; next attempt 3 in 1537 ms | 14:35:47.349
|
||||
[ 21.0s] 3 | processing | | 14:35:49.390
|
||||
[ 21.0s] 3 | retry_scheduled | temporary failure: ApiError; next attempt 4 in 3959 ms | 14:35:50.403
|
||||
[ 21.0s] 4 | processing | | 14:35:54.463
|
||||
[ 21.0s] 4 | retry_scheduled | temporary failure: ApiError; next attempt 5 in 5443 ms | 14:35:55.491
|
||||
[ 21.0s] 5 | processing | | 14:36:01.035
|
||||
[ 21.0s] 5 | failed | retry_exhausted: temporary failure: ApiError | 14:36:02.047
|
||||
[ 21.0s] $ replay.js --list
|
||||
[ 21.4s] replay: {"job_id":"b88d0626-5470-4148-83eb-cac17feb8b4c","attachment_id":"9832fbf4-2f69-484e-87e4-463d3e8ef3a7","error_code":"retry_exhausted","attempt":5,"last_error":"temporary failure: ApiError","finished_at":"2026-11-20T14:36:02.047Z"}
|
||||
[ 21.4s] $ docker compose start s3
|
||||
[ 23.7s] $ replay.js 9832fbf4-2f69-484e-87e4-463d3e8ef3a7
|
||||
[ 24.0s] replay: {"id":"9832fbf4-2f69-484e-87e4-463d3e8ef3a7","job_id":"b88d0626-5470-4148-83eb-cac17feb8b4c","result":"replayed"}
|
||||
[ 24.0s] GET attachment -> status=queued error_code=None
|
||||
[ 25.1s] GET attachment -> status=ready error_code=None
|
||||
[ 25.1s] original -> 200 sha256 matches upload: True
|
||||
[ 25.1s] thumbnail -> 200 image/jpeg 256x160 sha256 matches metadata: True
|
||||
[ 25.1s] crop -> 200 image/jpeg 128x128 sha256 matches metadata: True
|
||||
[ 25.1s] optimized -> 200 image/webp 320x200 sha256 matches metadata: True
|
||||
[ 25.1s] $ replay.js 9832fbf4-2f69-484e-87e4-463d3e8ef3a7 (already ready)
|
||||
[ 25.4s] replay: {"id":"9832fbf4-2f69-484e-87e4-463d3e8ef3a7","job_id":"b88d0626-5470-4148-83eb-cac17feb8b4c","result":"already_ready"}
|
||||
[ 25.6s] job: b88d0626-5470-4148-83eb-cac17feb8b4c | ready | 1 | 5 |
|
||||
[ 25.7s] variants rows: 3
|
||||
[ 25.9s] events: enqueued -> processing -> retry_scheduled -> processing -> retry_scheduled -> processing -> retry_scheduled -> processing -> retry_scheduled -> processing -> failed -> replayed -> processing -> ready
|
||||
|
||||
## broker
|
||||
$ python3 scripts/job-scenario.py broker
|
||||
[ 2.1s] consumers: queue_name ack_required prefetch_count
|
||||
attachment.jobs true 2
|
||||
[ 2.2s] worker limits: cpus=1000000000 memory=805306368
|
||||
[ 2.5s] malformed message published
|
||||
[ 10.3s] queues: name type durable messages_ready messages_unacknowledged
|
||||
attachment.jobs quorum true 0 0
|
||||
attachment.jobs.dlq quorum true 1 0
|
||||
[ 10.4s] worker: {"timestamp":"2026-11-20T14:36:10.245Z","severity":"WARN","service.name":"messenger-worker","event":"worker","logger":"Worker","message":"malformed job message moved to dead letter queue","trace_id":"733415ad4bdbd66df930af833ef44f18","span_id":"abc299a80e5c1e42"}
|
||||
|
||||
## Итог по всем заданиям стенда
|
||||
$ docker compose exec -T postgres psql -U messenger -d messenger -c SELECT status, count(*) FROM attachment_jobs GROUP BY status -c SELECT count(*) AS unpublished_outbox FROM outbox WHERE published_at IS NULL -c SELECT count(*) AS staging_left_on_ready FROM attachments WHERE status = 'ready' AND staging_key IS NOT NULL
|
||||
status | count
|
||||
--------+-------
|
||||
ready | 9
|
||||
(1 row)
|
||||
|
||||
unpublished_outbox
|
||||
--------------------
|
||||
0
|
||||
(1 row)
|
||||
|
||||
staging_left_on_ready
|
||||
-----------------------
|
||||
0
|
||||
(1 row)
|
||||
|
||||
|
||||
## Последний проход reconciler
|
||||
{"mode":"apply","grace_seconds":3600,"staging_expired":0,"scanned":0,"fresh_skipped":0,"referenced":0,"orphans":0,"deleted":0,"failed":0,"orphan_keys":[],"duration_ms":351}
|
||||
@@ -0,0 +1,26 @@
|
||||
# make test-persistence: restart.sh перезапускает postgres, redis, s3, rabbitmq, api, lb с сохранением volumes
|
||||
# date: 2026-11-20T14:33:01Z
|
||||
# commit: d83876f
|
||||
|
||||
## до
|
||||
/messenger-lab6-api-1 running started=2026-11-20T14:32:32.185792399Z
|
||||
/messenger-lab6-api-2 running started=2026-11-20T14:32:32.538220625Z
|
||||
/messenger-lab6-lb-1 running started=2026-11-20T14:32:38.299124143Z
|
||||
/messenger-lab6-postgres-1 running started=2026-11-20T14:32:22.796529774Z
|
||||
/messenger-lab6-rabbitmq-1 running started=2026-11-20T14:32:22.722217174Z
|
||||
/messenger-lab6-redis-1 running started=2026-11-20T14:32:22.787053855Z
|
||||
/messenger-lab6-s3-1 running started=2026-11-20T14:32:22.813514565Z
|
||||
|
||||
$ make test-persistence ACTION_SCRIPT=scripts/restart.sh
|
||||
python3 tests/resilience.py --mode persistence --action-script "scripts/restart.sh" --base-url "http://lb:8080"
|
||||
Executing explicit persistence hook; hook output suppressed to avoid leaking secrets.
|
||||
PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence.
|
||||
|
||||
## после
|
||||
/messenger-lab6-api-1 running started=2026-11-20T14:33:08.406683646Z
|
||||
/messenger-lab6-api-2 running started=2026-11-20T14:33:05.908734161Z
|
||||
/messenger-lab6-lb-1 running started=2026-11-20T14:33:03.32738495Z
|
||||
/messenger-lab6-postgres-1 running started=2026-11-20T14:33:03.697395758Z
|
||||
/messenger-lab6-rabbitmq-1 running started=2026-11-20T14:33:04.196322444Z
|
||||
/messenger-lab6-redis-1 running started=2026-11-20T14:33:03.432938355Z
|
||||
/messenger-lab6-s3-1 running started=2026-11-20T14:33:03.540481157Z
|
||||
@@ -0,0 +1,37 @@
|
||||
# S3 (минимум lab4): идемпотентный bootstrap, приватность мимо API, права пользователя API
|
||||
# date: 2026-11-20T14:33:34Z
|
||||
# commit: d83876f
|
||||
|
||||
$ docker compose run --rm --no-deps -T s3-init
|
||||
Bucket created successfully `root/messenger-attachments`.
|
||||
Access permission for `root/messenger-attachments` is set to `private`
|
||||
Created policy `messenger-api` successfully.
|
||||
bucket messenger-attachments ready
|
||||
|
||||
$ docker compose run --rm --no-deps -T s3-init
|
||||
Bucket created successfully `root/messenger-attachments`.
|
||||
Access permission for `root/messenger-attachments` is set to `private`
|
||||
Created policy `messenger-api` successfully.
|
||||
bucket messenger-attachments ready
|
||||
|
||||
$ sh scripts/check-s3.sh
|
||||
## S3 не опубликован наружу, консоль выключена
|
||||
s3 ports=9000/tcp
|
||||
MINIO_BROWSER=off
|
||||
|
||||
## Права пользователя API: только свой bucket
|
||||
PUT в свой bucket -> allowed
|
||||
GET из своего bucket -> allowed
|
||||
создание другого bucket -> denied
|
||||
GET из другого bucket -> denied
|
||||
PUT в другой bucket -> denied
|
||||
изменение политики bucket -> denied
|
||||
администрирование пользователей -> denied
|
||||
|
||||
## Анонимный доступ напрямую к S3, мимо API
|
||||
GET messenger-attachments/<object> -> 403
|
||||
GET messenger-attachments/ -> 403
|
||||
PUT messenger-attachments/anon.txt -> 403
|
||||
|
||||
PASS: bucket приватный, у API нет прав вне своего bucket
|
||||
|
||||
@@ -0,0 +1,22 @@
|
||||
# Публичные проверки через LB; разные реплики обслуживают одну сессию
|
||||
# date: 2026-11-20T14:32:58Z
|
||||
# commit: d83876f
|
||||
|
||||
$ make check
|
||||
python3 scripts/check_contract.py
|
||||
PASS: offline skeleton structure, local refs and Python syntax; application NOT tested.
|
||||
|
||||
$ make test
|
||||
python3 tests/smoke.py --base-url "http://lb:8080"
|
||||
PASS: lab6 public API scenario; infrastructure and grading evidence remain separate.
|
||||
|
||||
$ python3 scripts/probe.py instances 6
|
||||
register -> 201
|
||||
login -> 201 expires_at: 2026-11-20T15:33:01.208Z
|
||||
GET /api/v1/users/me -> 200 X-Instance-Id: 64377e499417
|
||||
GET /api/v1/users/me -> 200 X-Instance-Id: 25ff01cef182
|
||||
GET /api/v1/users/me -> 200 X-Instance-Id: 64377e499417
|
||||
GET /api/v1/users/me -> 200 X-Instance-Id: 25ff01cef182
|
||||
GET /api/v1/users/me -> 200 X-Instance-Id: 64377e499417
|
||||
GET /api/v1/users/me -> 200 X-Instance-Id: 25ff01cef182
|
||||
|
||||
@@ -0,0 +1,158 @@
|
||||
# Наблюдаемость: контрольный запрос (trace + логи), dashboard, перцентили, приватность, правила
|
||||
# date: 2026-11-20T14:36:18Z
|
||||
# commit: d83876f
|
||||
|
||||
## Prometheus: правила загружены (promtool и /api/v1/rules)
|
||||
$ docker compose exec -T prometheus promtool check rules /etc/prometheus/rules/messenger.yml
|
||||
Checking /etc/prometheus/rules/messenger.yml
|
||||
SUCCESS: 13 rules found
|
||||
|
||||
|
||||
$ sh -c wget -qO- http://grafana:3000/api/datasources/proxy/uid/prometheus/api/v1/rules | python3 -c 'import json,sys; [print(r["type"], r["name"], r["health"]) for g in json.load(sys.stdin)["data"]["groups"] for r in g["rules"]]'
|
||||
alerting MessengerHighErrorRate ok
|
||||
alerting MessengerApiNotReady ok
|
||||
alerting MessengerJobQueueLag ok
|
||||
alerting MessengerSlowApi ok
|
||||
alerting TelemetryExportFailing ok
|
||||
recording messenger:http_requests:rate5m ok
|
||||
recording messenger:http_errors:rate5m ok
|
||||
recording messenger:http_error_ratio:rate5m ok
|
||||
recording messenger:http_error_ratio:rate1h ok
|
||||
recording messenger:http_latency_p95:rate5m ok
|
||||
recording messenger:http_fast_ratio:rate1h ok
|
||||
recording messenger:time_to_ready_fast_ratio:rate1h ok
|
||||
recording messenger:jobs_queue_oldest_age:max ok
|
||||
|
||||
## Grafana: datasources с фиксированными UID, dashboard из provisioning
|
||||
$ sh -c for uid in prometheus loki tempo; do wget -qO- http://grafana:3000/api/datasources/uid/$uid | python3 -c 'import json,sys; d=json.load(sys.stdin); print(d["uid"], d["type"], d["url"])'; done
|
||||
prometheus prometheus http://prometheus:9090
|
||||
loki loki http://loki:3100
|
||||
tempo tempo http://tempo:3200
|
||||
|
||||
## Контрольный upload с клиентским traceparent: trace в Tempo и логи в Loki
|
||||
$ python3 scripts/trace-check.py
|
||||
upload -> 202 queued traceparent 00-0a9700e372765e5f2774c2c5347f8c1b-0de651da523f691b-01
|
||||
attachment 0a91d4b0-0624-419a-824e-d4ae671e7d2b status ready after 0.57 s
|
||||
|
||||
trace 0a9700e372765e5f2774c2c5347f8c1b: 79 spans (prisma:* internals except db_query omitted below)
|
||||
messenger-api SPAN_KIND_SERVER POST /api/v1/chats/{chat_id}/attachments 50.36 ms parent=client
|
||||
messenger-api SPAN_KIND_CLIENT prisma:engine:db_query 0.48 ms parent=prisma:engine:query
|
||||
messenger-api SPAN_KIND_CLIENT S3.PutObject 25.34 ms parent=POST /api/v1/chats/{chat_id}/attachments
|
||||
messenger-api SPAN_KIND_INTERNAL prisma:client:transaction 7.80 ms parent=POST /api/v1/chats/{chat_id}/attachments
|
||||
messenger-api SPAN_KIND_CLIENT prisma:engine:db_query 0.15 ms parent=prisma:engine:start_transaction
|
||||
messenger-api SPAN_KIND_CLIENT prisma:engine:db_query 0.81 ms parent=prisma:engine:query
|
||||
messenger-api SPAN_KIND_CLIENT prisma:engine:db_query 0.51 ms parent=prisma:engine:query
|
||||
messenger-api SPAN_KIND_CLIENT prisma:engine:db_query 0.27 ms parent=prisma:engine:query
|
||||
messenger-api SPAN_KIND_CLIENT prisma:engine:db_query 0.31 ms parent=prisma:engine:query
|
||||
messenger-api SPAN_KIND_CLIENT prisma:engine:db_query 1.33 ms parent=prisma:engine:commit_transaction
|
||||
messenger-relay SPAN_KIND_PRODUCER attachment.jobs publish 5.71 ms parent=prisma:client:transaction
|
||||
messenger-worker SPAN_KIND_CONSUMER attachment.jobs process 66.86 ms parent=attachment.jobs publish
|
||||
messenger-worker SPAN_KIND_CLIENT prisma:engine:db_query 0.25 ms parent=prisma:engine:connection
|
||||
messenger-worker SPAN_KIND_CLIENT prisma:engine:db_query 0.34 ms parent=prisma:engine:query
|
||||
messenger-worker SPAN_KIND_CLIENT prisma:engine:db_query 0.26 ms parent=prisma:engine:query
|
||||
messenger-worker SPAN_KIND_INTERNAL prisma:client:transaction 7.26 ms parent=attachment.jobs process
|
||||
messenger-worker SPAN_KIND_CLIENT prisma:engine:db_query 0.14 ms parent=prisma:engine:start_transaction
|
||||
messenger-worker SPAN_KIND_CLIENT prisma:engine:db_query 0.48 ms parent=prisma:engine:query
|
||||
messenger-worker SPAN_KIND_CLIENT prisma:engine:db_query 0.50 ms parent=prisma:engine:query
|
||||
messenger-worker SPAN_KIND_CLIENT prisma:engine:db_query 0.26 ms parent=prisma:engine:query
|
||||
messenger-worker SPAN_KIND_CLIENT prisma:engine:db_query 0.27 ms parent=prisma:engine:query
|
||||
messenger-worker SPAN_KIND_CLIENT prisma:engine:db_query 1.24 ms parent=prisma:engine:commit_transaction
|
||||
messenger-worker SPAN_KIND_CLIENT S3.GetObject 3.00 ms parent=attachment.jobs process
|
||||
messenger-worker SPAN_KIND_INTERNAL attachment.render_variants 12.41 ms parent=attachment.jobs process
|
||||
messenger-worker SPAN_KIND_CLIENT S3.PutObject 7.85 ms parent=attachment.jobs process
|
||||
messenger-worker SPAN_KIND_CLIENT S3.PutObject 5.94 ms parent=attachment.jobs process
|
||||
messenger-worker SPAN_KIND_CLIENT S3.PutObject 5.73 ms parent=attachment.jobs process
|
||||
messenger-worker SPAN_KIND_CLIENT S3.PutObject 5.69 ms parent=attachment.jobs process
|
||||
messenger-worker SPAN_KIND_INTERNAL prisma:client:transaction 7.42 ms parent=attachment.jobs process
|
||||
messenger-worker SPAN_KIND_CLIENT prisma:engine:db_query 0.16 ms parent=prisma:engine:start_transaction
|
||||
messenger-worker SPAN_KIND_CLIENT prisma:engine:db_query 0.61 ms parent=prisma:engine:query
|
||||
messenger-worker SPAN_KIND_CLIENT prisma:engine:db_query 0.71 ms parent=prisma:engine:query
|
||||
messenger-worker SPAN_KIND_CLIENT prisma:engine:db_query 0.44 ms parent=prisma:engine:query
|
||||
messenger-worker SPAN_KIND_CLIENT prisma:engine:db_query 0.26 ms parent=prisma:engine:query
|
||||
messenger-worker SPAN_KIND_CLIENT prisma:engine:db_query 1.32 ms parent=prisma:engine:commit_transaction
|
||||
messenger-worker SPAN_KIND_CLIENT S3.DeleteObject 3.23 ms parent=attachment.jobs process
|
||||
messenger-worker SPAN_KIND_CLIENT prisma:engine:db_query 1.71 ms parent=prisma:engine:query
|
||||
|
||||
Loki {service_name=~"messenger-.+"} | trace_id="0a9700e372765e5f2774c2c5347f8c1b"
|
||||
messenger-worker INFO job.handled span_id=8f3cc8917f60f19d {"timestamp":"2026-11-20T14:36:19.298Z","severity":"INFO","s
|
||||
messenger-api INFO http.request span_id=70c807e8b4f94da9 {"timestamp":"2026-11-20T14:36:19.155Z","severity":"INFO","s
|
||||
|
||||
PASS: client trace continued through api -> relay -> worker, logs linked by trace_id
|
||||
|
||||
## Секреты и содержимое не попадают в telemetry; в labels метрик нет ID
|
||||
$ python3 scripts/telemetry-privacy.py
|
||||
sent 6 traced requests with known password, token, message text and filename; waiting 25 s for export
|
||||
Tempo: 6/6 traces fetched (79945 bytes); Loki: 7 log lines carry these trace_ids (control)
|
||||
password in traces: False log lines in Loki (5 min): 0
|
||||
message text in traces: False log lines in Loki (5 min): 0
|
||||
filename in traces: False log lines in Loki (5 min): 0
|
||||
session token in traces: False log lines in Loki (5 min): 0
|
||||
basic credentials in traces: False log lines in Loki (5 min): 0
|
||||
Authorization hdr in traces: False log lines in Loki (5 min): 0
|
||||
Prometheus: 893 series, 22 metric names from messenger services; label keys: cpu_mode, db_client_connection_state, deployment_environment_name, http_request_method, http_response_status_class, http_route, instance, job, job_type, le, messaging_destination_name, outcome, service_version, state, telemetry_sdk_language, telemetry_sdk_name, telemetry_sdk_version
|
||||
series with a UUID in any label value: 0
|
||||
http_route values: /api/v1/attachments/{attachment_id}, /api/v1/attachments/{attachment_id}/content, /api/v1/auth/sessions, /api/v1/auth/sessions/current, /api/v1/chats, /api/v1/chats/{chat_id}, /api/v1/chats/{chat_id}/attachments, /api/v1/chats/{chat_id}/messages, /api/v1/users, /api/v1/users/me, /health/core, /health/live, /health/ready
|
||||
PASS: no secrets/content in logs and traces, no IDs in metric labels
|
||||
|
||||
## Перцентили dashboard против клиентского замера tests/load.py
|
||||
$ python3 scripts/latency-compare.py --duration 30
|
||||
waiting 25 s for the metric export interval
|
||||
client (tests/load.py): 3411 requests, 113.4 rps, errors 0
|
||||
prometheus: 3661 requests in POST /api/v1/chats/{chat_id}/messages over 60 s
|
||||
client, ms prometheus, ms
|
||||
p50 18.33 19.68
|
||||
p95 89.20 92.20
|
||||
p99 163.29 176.76
|
||||
server-side histogram excludes network/client overhead; values are interpolated inside buckets (…, 5, 10, 25, 50, 100 ms, …).
|
||||
|
||||
## Все запросы dashboard (пустые панели alerts/WARN-логов допустимы без инцидентов)
|
||||
$ python3 scripts/dashboard-check.py
|
||||
Grafana dashboard: Messenger overview folder: Messenger
|
||||
OK Доступность API, 1 ч (SLO ≥ 99% без 5xx) A 1 =1
|
||||
OK Запросы быстрее 500 мс, 1 ч (SLO ≥ 95%) A 1 =1
|
||||
OK Вложения готовы за ≤ 10 с, 1 ч (SLO ≥ 95%) A 1 =0.6666666666666667
|
||||
EMPTY Alerts (pending / firing) A 0
|
||||
OK RPS по маршрутам (без health) A 12 =0, =0, =0, =0
|
||||
OK Ответы по классам статуса (без health) A 2 2xx=61.1782202581422, 4xx=0
|
||||
OK Доля 5xx среди бизнес-запросов (окно 5m, порог alert 5%) A 1 =0
|
||||
OK Длительность API p50 / p95 / p99 (все реплики, без health) A 1 =0.019676691515573837
|
||||
OK Длительность API p50 / p95 / p99 (все реплики, без health) B 1 =0.09219914944691271
|
||||
OK Длительность API p50 / p95 / p99 (все реплики, без health) C 1 =0.17675359699995996
|
||||
OK p95 по маршрутам A 12 =NaN, =NaN, =NaN, =NaN
|
||||
OK Глубина очереди attachment.jobs (из attachment_jobs) A 3 =0, =0, =0
|
||||
OK Возраст старейшего готового задания (порог alert 60 s) A 1 =0
|
||||
OK Worker: занято / ёмкость A 1 =0
|
||||
OK Worker: занято / ёмкость B 1 =2
|
||||
OK Worker: занято / ёмкость C 1 =0
|
||||
OK Задания по исходу (в секунду) A 5 failed=0, malformed=0, retry=0, skipped=0.025
|
||||
OK Длительность обработки задания p50 / p95 / p99 A 1 =NaN
|
||||
OK Длительность обработки задания p50 / p95 / p99 B 1 =NaN
|
||||
OK Длительность обработки задания p50 / p95 / p99 C 1 =NaN
|
||||
OK Время от upload до ready p50 / p95 / p99 A 1 =NaN
|
||||
OK Время от upload до ready p50 / p95 / p99 B 1 =NaN
|
||||
OK Время от upload до ready p50 / p95 / p99 C 1 =NaN
|
||||
OK Пул БД: занято / ожидают соединения A 3 =0, =0, =0
|
||||
OK Пул БД: занято / ожидают соединения B 3 =0, =0, =0
|
||||
OK Пул БД: занято / ожидают соединения C 3 =20, =3, =5
|
||||
OK CPU процессов (доля ядра) A 4 =0.2771702986118047, =0.2803168326733069, =0.006292157529451772, =0.00
|
||||
OK CPU процессов (доля ядра) B 2 =0.013556760600942309, =0.0006813251539196734
|
||||
OK RAM процессов (RSS) A 4 =207585280, =208904192, =90546176, =106270720
|
||||
OK RAM процессов (RSS) B 2 =210243584, =164040704
|
||||
OK Принято / отправлено / ошибки отправки (в секунду) A 1 =1512.6751237147007
|
||||
OK Принято / отправлено / ошибки отправки (в секунду) B 1 =1512.6751237147007
|
||||
OK Принято / отправлено / ошибки отправки (в секунду) C 1 =0
|
||||
OK Принято / отправлено / ошибки отправки (в секунду) D 1 =75.96230776068984
|
||||
OK Принято / отправлено / ошибки отправки (в секунду) E 1 =0
|
||||
OK Принято / отправлено / ошибки отправки (в секунду) F 1 =0
|
||||
OK Очередь exporter (ограничена queue_size) A 3 =0, =0, =0
|
||||
OK Очередь exporter (ограничена queue_size) B 1 =200
|
||||
OK Логи API и worker (WARN и выше) A 20
|
||||
OK Все логи (http.request, job.handled, …) A 20
|
||||
OK Последние трассы (HTTP server и consumer spans) A 20
|
||||
|
||||
WARN: 1 empty queries
|
||||
EXIT CODE 1
|
||||
|
||||
## Пример структурного лога (stdout API)
|
||||
{"timestamp":"2026-11-20T14:37:24.827Z","severity":"INFO","service.name":"messenger-api","event":"http.request","logger":"Http","method":"POST","route":"/api/v1/chats/{chat_id}/messages","status":201,"duration_ms":23.22,"request_id":"078e6db3-d15e-4bdf-bac9-5fe5ea48bd35","trace_id":"67197c10f2dc44bc87840224dec07957","span_id":"a4e80137cbbf9edc"}
|
||||
{"timestamp":"2026-11-20T14:37:24.838Z","severity":"INFO","service.name":"messenger-api","event":"http.request","logger":"Http","method":"POST","route":"/api/v1/chats/{chat_id}/messages","status":201,"duration_ms":20.5,"request_id":"5513c34d-64ea-4250-9cee-a9d9a4b12892","trace_id":"ec0813f680256f525febe0676221457b","span_id":"d20713fac541ea0b"}
|
||||
@@ -0,0 +1,484 @@
|
||||
# Собственные e2e-тесты (PostgreSQL + Redis + MinIO из compose.test.yaml); основной стенд на время тестов остановлен ради памяти Docker
|
||||
# date: 2026-11-20T14:31:15Z
|
||||
# commit: d83876f
|
||||
|
||||
$ docker compose -f compose.test.yaml run --rm tests npx jest --config ./test/jest-e2e.json --verbose
|
||||
Network messenger-lab6-test_default Creating
|
||||
Network messenger-lab6-test_default Created
|
||||
Container messenger-lab6-test-redis-1 Creating
|
||||
Container messenger-lab6-test-s3-1 Creating
|
||||
Container messenger-lab6-test-postgres-1 Creating
|
||||
Container messenger-lab6-test-s3-1 Created
|
||||
Container messenger-lab6-test-redis-1 Created
|
||||
Container messenger-lab6-test-postgres-1 Created
|
||||
Container messenger-lab6-test-s3-1 Starting
|
||||
Container messenger-lab6-test-redis-1 Starting
|
||||
Container messenger-lab6-test-postgres-1 Starting
|
||||
Container messenger-lab6-test-s3-1 Started
|
||||
Container messenger-lab6-test-postgres-1 Started
|
||||
Container messenger-lab6-test-redis-1 Started
|
||||
Prisma schema loaded from prisma/schema.prisma
|
||||
Datasource "db": PostgreSQL database "messenger_test", schema "public" at "postgres:5432"
|
||||
|
||||
5 migrations found in prisma/migrations
|
||||
|
||||
Applying migration `20261021000000_init`
|
||||
Applying migration `20261021010000_auth_sessions`
|
||||
Applying migration `20261101000000_idempotency_keys`
|
||||
Applying migration `20261105000000_attachments`
|
||||
Applying migration `20261115000000_attachment_jobs`
|
||||
|
||||
The following migration(s) have been applied:
|
||||
|
||||
migrations/
|
||||
└─ 20261021000000_init/
|
||||
└─ migration.sql
|
||||
└─ 20261021010000_auth_sessions/
|
||||
└─ migration.sql
|
||||
└─ 20261101000000_idempotency_keys/
|
||||
└─ migration.sql
|
||||
└─ 20261105000000_attachments/
|
||||
└─ migration.sql
|
||||
└─ 20261115000000_attachment_jobs/
|
||||
└─ migration.sql
|
||||
|
||||
All migrations have been successfully applied.
|
||||
[31m[Nest] 102 - [39m11/20/2026, 2:31:59 PM [31m ERROR[39m [38;5;3m[S3Service] [39m[31mget staging/a3987d2f-e0a6-4581-8caa-8670462a94fa failed: NoSuchKey[39m
|
||||
PASS test/api.e2e-spec.ts (17.729 s)
|
||||
API мессенджера (e2e) — граничные случаи сверх публичного smoke
|
||||
POST /api/v1/users
|
||||
✓ отклоняет некорректный JSON с кодом 400 (112 ms)
|
||||
✓ отклоняет тело с лишними полями (39 ms)
|
||||
✓ отклоняет username с заглавными буквами (4 ms)
|
||||
✓ отклоняет display_name из одних пробелов (4 ms)
|
||||
✓ отклоняет дубликат username с кодом 409 (183 ms)
|
||||
✓ никогда не возвращает поле пароля или его хэша (32 ms)
|
||||
✓ отклоняет username короче 3 символов (3 ms)
|
||||
✓ отклоняет username длиннее 32 символов (4 ms)
|
||||
✓ отклоняет username с недопустимыми символами (дефис) (3 ms)
|
||||
✓ принимает username ровно на границах длины 3 и 32 символа (56 ms)
|
||||
✓ отклоняет display_name длиннее 100 Unicode code points (3 ms)
|
||||
✓ принимает display_name ровно из 100 code points, считая эмодзи вне BMP за один символ (27 ms)
|
||||
✓ отклоняет display_name из 101 code point, состоящий из эмодзи вне BMP (3 ms)
|
||||
контроль доступа
|
||||
✓ возвращает 401 раньше 404, если и актор, и ресурс невалидны (5 ms)
|
||||
✓ возвращает 404 (не 403), когда чат запрашивает не его участник (254 ms)
|
||||
✓ отклоняет создание чата, если вызывающий не входит в member_ids (155 ms)
|
||||
✓ отклоняет создание чата с дублирующимися member_ids (83 ms)
|
||||
✓ считает один UUID в разном регистре дубликатом и узнаёт вызывающего в верхнем регистре (112 ms)
|
||||
✓ принимает ровно 100 участников и отклоняет 101 (104 ms)
|
||||
✓ отклоняет создание чата с неизвестным участником (79 ms)
|
||||
✓ отклоняет создание чата с пустым/пробельным title (75 ms)
|
||||
✓ возвращает 400 для синтаксически некорректного chat_id (не UUID) (74 ms)
|
||||
✓ возвращает 404 для корректного по формату, но несуществующего chat_id (75 ms)
|
||||
GET /api/v1/chats (пагинация)
|
||||
✓ использует limit=50 по умолчанию, если limit не передан (85 ms)
|
||||
✓ отклоняет нечисловой limit (75 ms)
|
||||
✓ отклоняет limit=0 (77 ms)
|
||||
✓ отклоняет повторённые limit и cursor (72 ms)
|
||||
✓ отклоняет cursor, не являющийся корректным base64url JSON (67 ms)
|
||||
✓ отклоняет корректно закодированный cursor с id чата, который никогда не существовал (69 ms)
|
||||
✓ обходит весь список чатов по страницам без пропусков и дублей (114 ms)
|
||||
сообщения
|
||||
✓ не принимает sender_id, переданный клиентом (144 ms)
|
||||
✓ считает attachment_ids: [] равным отсутствию поля, неизвестный ID — 404, неверный тип — 400 (241 ms)
|
||||
✓ отклоняет пустое сообщение (148 ms)
|
||||
✓ отклоняет сообщение из одних пробелов (132 ms)
|
||||
✓ отклоняет отправку в чат, где вызывающий не участник (199 ms)
|
||||
✓ отклоняет limit=0 при чтении истории (165 ms)
|
||||
✓ отклоняет limit=101 при чтении истории (148 ms)
|
||||
✓ отклоняет нечисловой limit (169 ms)
|
||||
✓ отклоняет текст сообщения длиннее 4000 Unicode code points (185 ms)
|
||||
✓ принимает текст сообщения ровно из 4000 Unicode code points (218 ms)
|
||||
✓ отклоняет текст сообщения свыше 4000 code points, состоящий из эмодзи вне BMP (158 ms)
|
||||
✓ отклоняет cursor, не являющийся корректным base64url JSON (158 ms)
|
||||
✓ отклоняет корректно закодированный cursor с id сообщения, которое никогда не существовало в этом чате (168 ms)
|
||||
✓ возвращает 400 для синтаксически некорректного chat_id при отправке сообщения (80 ms)
|
||||
✓ возвращает 400 для синтаксически некорректного chat_id при чтении истории (68 ms)
|
||||
✓ отклоняет cursor, взятый из другого чата (276 ms)
|
||||
✓ обходит всю историю по страницам без пропусков и дублей (277 ms)
|
||||
X-Request-Id
|
||||
✓ присутствует и уникален для независимых запросов (4 ms)
|
||||
✓ совпадает с request_id внутри тела ошибки (2 ms)
|
||||
|
||||
[31m[Nest] 102 - [39m11/20/2026, 2:32:05 PM [31m ERROR[39m [38;5;3m[S3Service] [39m[31mget staging/02ca6856-dcd8-4bbd-9fe8-272649f2b87c failed: NoSuchKey[39m
|
||||
[31m[Nest] 102 - [39m11/20/2026, 2:32:05 PM [31m ERROR[39m [38;5;3m[S3Service] [39m[31mget staging/8007bc99-ed6a-4a68-9ff7-1b65dcb1cd82 failed: NoSuchKey[39m
|
||||
PASS test/jobs.e2e-spec.ts (19.57 s)
|
||||
Фоновая обработка вложений (e2e)
|
||||
до обработки
|
||||
✓ queued: метаданные доступны, content любого варианта — 409, сообщение с вложением создаётся (250 ms)
|
||||
результат worker
|
||||
✓ ready: оригинал неизменен, thumbnail 256×160 JPEG, crop 128×128 JPEG, optimized WebP без увеличения (253 ms)
|
||||
✓ маленький исходник не увеличивается в thumbnail/optimized, crop — увеличивается (148 ms)
|
||||
✓ прозрачность компонуется на белый, EXIF orientation применяется (322 ms)
|
||||
✓ worker ограничивает пиксели до распаковки: превышение — InvalidImageError (18 ms)
|
||||
✓ размеры считаются целочисленно: 319×200 → 256×160, перебор совпадает с точной формулой (2289 ms)
|
||||
✓ размеры округляются вниз, минимум 1 px (1 ms)
|
||||
идемпотентность и повторная доставка
|
||||
✓ повтор уже ready-задания — no-op: второго набора вариантов нет, объекты не меняются (249 ms)
|
||||
✓ пока lease действует, второй обработчик того же job не запускается (162 ms)
|
||||
✓ worker упал после записи объектов, до COMMIT/ack: lease истекает, повтор завершает тот же результат (293 ms)
|
||||
✓ worker упал после COMMIT, до ack: повтор — no-op, staging убирается при повторной доставке (210 ms)
|
||||
✓ сообщение потеряно брокером: queued без свежей публикации публикуется повторно один раз (233 ms)
|
||||
✓ устаревшее сообщение, чужой attachment_id и другая pipeline_version игнорируются (62 ms)
|
||||
ошибки, повторы и replay
|
||||
✓ временная ошибка S3: повтор с backoff, до срока задание не берётся, затем ready (208 ms)
|
||||
✓ попытки исчерпаны — failed retry_exhausted; replay явно возвращает в очередь и доводит до ready (470 ms)
|
||||
✓ lease истёк на последней попытке — failed retry_exhausted, а не вечный processing (278 ms)
|
||||
✓ replay и очистка staging не теряют исходник ни при каком порядке (541 ms)
|
||||
✓ replay задания в очереди не создаёт дубль (76 ms)
|
||||
✓ нераспаковываемое изображение — постоянная ошибка: сразу failed invalid_image без повторов (165 ms)
|
||||
✓ staging подменён — failed checksum_mismatch; staging пропал — failed staging_missing (134 ms)
|
||||
✓ превышение времени обработки — временная ошибка с повтором (112 ms)
|
||||
формат сообщения и backoff
|
||||
✓ сообщение соответствует jobs.schema.json, лишние/неверные поля отклоняются (1 ms)
|
||||
✓ backoff растёт экспоненциально с jitter и ограничен сверху (1 ms)
|
||||
|
||||
console.warn
|
||||
An error was encountered in a non-retryable streaming request.
|
||||
|
||||
213 | private async guard<T>(action: string, run: () => Promise<T>): Promise<T> {
|
||||
214 | try {
|
||||
> 215 | return await run();
|
||||
| ^
|
||||
216 | } catch (error) {
|
||||
217 | const reason = error instanceof Error ? error.name : 'unknown error';
|
||||
218 | this.logger.error(`${action} failed: ${reason}`);
|
||||
|
||||
at ../node_modules/@smithy/middleware-retry/dist-cjs/index.js:329:125
|
||||
at ../node_modules/@aws-sdk/middleware-flexible-checksums/dist-cjs/index.js:263:18
|
||||
at ../node_modules/@aws-sdk/middleware-sdk-s3/dist-cjs/index.js:110:22
|
||||
at ../node_modules/@aws-sdk/middleware-sdk-s3/dist-cjs/index.js:138:14
|
||||
at ../node_modules/@aws-sdk/middleware-logger/dist-cjs/index.js:34:22
|
||||
at S3Service.guard (../src/storage/s3.service.ts:215:11)
|
||||
at S3Service.putFile (../src/storage/s3.service.ts:64:3)
|
||||
at AttachmentsService.store (../src/attachments/attachments.service.ts:153:3)
|
||||
at AttachmentsService.upload (../src/attachments/attachments.service.ts:75:11)
|
||||
at AttachmentsController.upload (../src/attachments/attachments.controller.ts:36:22)
|
||||
at ../node_modules/@nestjs/core/router/router-execution-context.js:46:28
|
||||
at Object.<anonymous> (../node_modules/@nestjs/core/router/router-proxy.js:9:17)
|
||||
|
||||
[31m[Nest] 108 - [39m11/20/2026, 2:32:06 PM [31m ERROR[39m [38;5;3m[S3Service] [39m[31mput staging/1071fe6c-2f2f-4f80-a8b5-904ca05009a2 failed: Error[39m
|
||||
[31m[Nest] 108 - [39m11/20/2026, 2:32:06 PM [31m ERROR[39m [38;5;3m[HttpExceptionFilter] [39m[31mObject:[39m
|
||||
{
|
||||
"event": "http.error",
|
||||
"message": "object storage unavailable",
|
||||
"method": "POST",
|
||||
"route": "/api/v1/chats/{chat_id}/attachments",
|
||||
"status": 503,
|
||||
"request_id": "38769a67-411f-41a7-bca3-d3682a083471"
|
||||
}
|
||||
|
||||
[31m[Nest] 108 - [39m11/20/2026, 2:32:06 PM [31m ERROR[39m [38;5;3m[HttpExceptionFilter] [39m[31mObject:[39m
|
||||
{
|
||||
"event": "http.error",
|
||||
"message": "Internal server error",
|
||||
"method": "POST",
|
||||
"route": "/api/v1/chats/{chat_id}/attachments",
|
||||
"status": 500,
|
||||
"request_id": "be749a43-6bee-4629-ac78-3f15b6f77eac"
|
||||
}
|
||||
|
||||
[31m[Nest] 108 - [39m11/20/2026, 2:32:06 PM [31m ERROR[39m [38;5;3m[AttachmentsService] [39m[31mmetadata insert failed for staging/50082f9d-e8dd-4f01-9391-47bf371e52fb[39m
|
||||
[31m[Nest] 108 - [39m11/20/2026, 2:32:06 PM [31m ERROR[39m [38;5;3m[HttpExceptionFilter] [39m[31mObject:[39m
|
||||
{
|
||||
"event": "http.error",
|
||||
"message": "database unavailable",
|
||||
"method": "POST",
|
||||
"route": "/api/v1/chats/{chat_id}/attachments",
|
||||
"status": 503,
|
||||
"request_id": "2a4d5542-1230-4568-acb5-7eb6c4390cdf"
|
||||
}
|
||||
|
||||
[31m[Nest] 108 - [39m11/20/2026, 2:32:06 PM [31m ERROR[39m [38;5;3m[HttpExceptionFilter] [39m[31mObject:[39m
|
||||
{
|
||||
"event": "http.error",
|
||||
"message": "database unavailable",
|
||||
"method": "POST",
|
||||
"route": "/api/v1/chats/{chat_id}/attachments",
|
||||
"status": 503,
|
||||
"request_id": "41953d69-f89d-42ba-9f5c-61bd4fa2e40d"
|
||||
}
|
||||
|
||||
PASS test/attachments.e2e-spec.ts (20.033 s)
|
||||
Загрузка вложений (e2e)
|
||||
успешная загрузка
|
||||
✓ возвращает 202 Attachment queued по контракту, staging и задание сохранены до ответа (307 ms)
|
||||
✓ принимает реальный JPEG (48 ms)
|
||||
✓ игнорирует параметры и регистр Content-Type (41 ms)
|
||||
✓ принимает файл ровно 10 MiB, filename ровно 255 символов и PNG с байтами после IEND (724 ms)
|
||||
✓ принимает ровно 40 млн пикселей (2896 ms)
|
||||
порядок проверок и доступ
|
||||
✓ без токена — 401 раньше любых проверок содержимого (19 ms)
|
||||
✓ посторонний и несуществующий чат — одинаковый 404, чужой файл не сохраняется (17 ms)
|
||||
✓ неверный UUID чата — 400 (5 ms)
|
||||
filename
|
||||
✓ отсутствует — 400 (4 ms)
|
||||
✓ пустой — 400 (4 ms)
|
||||
✓ только пробелы — 400 (5 ms)
|
||||
✓ 256 символов — 400 (5 ms)
|
||||
✓ путь через / — 400 (5 ms)
|
||||
✓ путь через \ — 400 (5 ms)
|
||||
✓ NUL — 400 (8 ms)
|
||||
✓ перевод строки — 400 (5 ms)
|
||||
✓ DEL — 400 (5 ms)
|
||||
✓ C1 U+0085 — 400 (5 ms)
|
||||
✓ разделитель строк U+2028 — 400 (4 ms)
|
||||
✓ повторённый параметр — 400 (8 ms)
|
||||
тип содержимого
|
||||
✓ Content-Type text/plain — 415 (5 ms)
|
||||
✓ Content-Type application/json — 415 (35 ms)
|
||||
✓ Content-Type application/octet-stream — 415 (9 ms)
|
||||
✓ Content-Type multipart/form-data; boundary=x — 415 (13 ms)
|
||||
✓ Content-Type image/gif — 415 (13 ms)
|
||||
✓ Content-Type image/webp — 415 (9 ms)
|
||||
✓ без Content-Type — 415 (11 ms)
|
||||
✓ подмена MIME в обе стороны и чужие форматы — 415 (84 ms)
|
||||
✓ пустое тело — 400 (30 ms)
|
||||
повреждённые изображения
|
||||
✓ PNG: первые 33 байта sample.png — 415 (20 ms)
|
||||
✓ PNG: половина файла — 415 (17 ms)
|
||||
✓ PNG: без последних 20 байт — 415 (20 ms)
|
||||
✓ PNG: без IEND — 415 (24 ms)
|
||||
✓ PNG: неверный CRC — 415 (18 ms)
|
||||
✓ PNG: фильтр строки 5 — 415 (15 ms)
|
||||
✓ PNG: фильтр строки 255 — 415 (10 ms)
|
||||
✓ PNG: неверный фильтр в последней строке — 415 (11 ms)
|
||||
✓ PNG: мусор в IDAT с верным CRC — 415 (10 ms)
|
||||
✓ PNG: данных меньше, чем строк — 415 (8 ms)
|
||||
✓ PNG: палитра на 257 записей — 415 (11 ms)
|
||||
✓ JPEG: все повреждения — 415 (68 ms)
|
||||
лимиты
|
||||
✓ 10 MiB + 1 байт — 413 и без чтения объявленного тела (220 ms)
|
||||
✓ превышение при chunked без Content-Length — 413 (944 ms)
|
||||
✓ превышение лимита пикселей — 413 до декодирования (92 ms)
|
||||
✓ оборванная загрузка не оставляет временных файлов и строк в БД (819 ms)
|
||||
отказы
|
||||
✓ S3 недоступен — 503 с заголовками, метаданные не создаются (185 ms)
|
||||
✓ явный отказ транзакции после PUT удаляет объект staging (56 ms)
|
||||
✓ необработанная ошибка PostgreSQL при INSERT — 503, объект не удаляется (34 ms)
|
||||
✓ при неясном исходе COMMIT объект не удаляется (31 ms)
|
||||
|
||||
[31m[Nest] 101 - [39m11/20/2026, 2:32:07 PM [31m ERROR[39m [38;5;3m[S3Service] [39m[31mget attachments/0b88f570-7a00-4713-bc1f-eb84c29804f1/original failed: Error[39m
|
||||
[31m[Nest] 101 - [39m11/20/2026, 2:32:07 PM [31m ERROR[39m [38;5;3m[HttpExceptionFilter] [39m[31mObject:[39m
|
||||
{
|
||||
"event": "http.error",
|
||||
"message": "object storage unavailable",
|
||||
"method": "GET",
|
||||
"route": "/api/v1/attachments/{attachment_id}/content",
|
||||
"status": 503,
|
||||
"request_id": "c2f13d3f-5110-4cc7-83af-7137dd518fea"
|
||||
}
|
||||
|
||||
[31m[Nest] 101 - [39m11/20/2026, 2:32:07 PM [31m ERROR[39m [38;5;3m[S3Service] [39m[31mget attachments/6ea31e5f-9549-486c-8249-e3ffb6da6413/original failed: NoSuchKey[39m
|
||||
[31m[Nest] 101 - [39m11/20/2026, 2:32:07 PM [31m ERROR[39m [38;5;3m[HttpExceptionFilter] [39m[31mObject:[39m
|
||||
{
|
||||
"event": "http.error",
|
||||
"message": "Internal server error",
|
||||
"method": "GET",
|
||||
"route": "/api/v1/attachments/{attachment_id}/content",
|
||||
"status": 500,
|
||||
"request_id": "e2f0e5e7-e052-4972-8fcc-60b05ae4bb83"
|
||||
}
|
||||
|
||||
[31m[Nest] 102 - [39m11/20/2026, 2:32:08 PM [31m ERROR[39m [38;5;3m[HttpExceptionFilter] [39m[31mObject:[39m
|
||||
{
|
||||
"event": "http.error",
|
||||
"message": "object storage unavailable",
|
||||
"method": "GET",
|
||||
"route": "/health/ready",
|
||||
"status": 503,
|
||||
"request_id": "fce039e2-3f6a-4a82-bcc9-516576969c72"
|
||||
}
|
||||
|
||||
PASS test/request-constraints.e2e-spec.ts
|
||||
Каждое ограничение запроса из контракта отклоняется 400, граница принимается (e2e)
|
||||
✓ POST /users (263 ms)
|
||||
✓ POST /chats (91 ms)
|
||||
✓ POST /chats/{chat_id}/messages (157 ms)
|
||||
✓ query и path параметры (149 ms)
|
||||
✓ POST /chats/{chat_id}/attachments: filename и chat_id (91 ms)
|
||||
|
||||
PASS test/reconciler.e2e-spec.ts
|
||||
Сверка объектов S3 с метаданными (e2e)
|
||||
✓ dry-run сообщает о сироте и ничего не удаляет, apply удаляет, повторный запуск ничего не делает (67 ms)
|
||||
✓ не трогает свежий объект внутри grace period (24 ms)
|
||||
✓ не трогает старый объект, на который есть метаданные (19 ms)
|
||||
✓ при отказе PostgreSQL проход прерывается и ничего не удаляет (27 ms)
|
||||
✓ сбой удаления считается и повторяется на следующем проходе (50 ms)
|
||||
✓ обходит все страницы листинга (112 ms)
|
||||
✓ реальная загрузка остаётся доступной после сверки всего bucket префикса вложений (384 ms)
|
||||
✓ после grace period объекты вложения и staging в очереди не считаются сиротами (282 ms)
|
||||
✓ staging failed-вложения удаляется после срока хранения (88 ms)
|
||||
✓ объекты вариантов без строк у существующего вложения не удаляются: replay после сверки доводит до ready (301 ms)
|
||||
✓ staging ready-вложения, оставшийся после сбоя worker, удаляется без ожидания срока хранения (161 ms)
|
||||
Готовность при отказе S3 (e2e)
|
||||
✓ ready — 503 с заголовками за короткое время, live и core — 200 (64 ms)
|
||||
✓ при рабочем S3 ready — 200 (61 ms)
|
||||
|
||||
PASS test/auth.e2e-spec.ts
|
||||
Аутентификация и сессии (e2e)
|
||||
регистрация
|
||||
✓ требует password (22 ms)
|
||||
✓ отклоняет password короче 12 code points (3 ms)
|
||||
✓ хранит пароль как Argon2id-хеш, а не открытым текстом (147 ms)
|
||||
POST /api/v1/auth/sessions
|
||||
✓ выдаёт непрозрачный Bearer-токен с expires_at и пользователем (117 ms)
|
||||
✓ принимает пароль с двоеточием и не-ASCII символами (85 ms)
|
||||
✓ отвечает одинаковым 401 на неверный пароль и неизвестного пользователя (141 ms)
|
||||
✓ отклоняет JSON body при входе даже с верным Basic (83 ms)
|
||||
✓ отвечает 401 без заголовка Authorization (3 ms)
|
||||
защищённые endpoint
|
||||
✓ не принимают Basic (68 ms)
|
||||
✓ не принимают X-User-Id сам по себе (89 ms)
|
||||
✓ игнорируют X-User-Id рядом с Bearer (150 ms)
|
||||
✓ отвечают 401 на неизвестный токен (7 ms)
|
||||
DELETE /api/v1/auth/sessions/current
|
||||
✓ отзывает только текущий токен: 204, затем 401 на повтор (195 ms)
|
||||
|
||||
PASS test/contract.e2e-spec.ts
|
||||
Все ответы соответствуют контракту: коды, схемы, заголовки (e2e)
|
||||
✓ health, пользователи и сессии (324 ms)
|
||||
✓ чаты и сообщения (471 ms)
|
||||
✓ вложения во всех состояниях (768 ms)
|
||||
✓ каждая операция контракта вызвана хотя бы раз (1 ms)
|
||||
|
||||
PASS test/attachment-access.e2e-spec.ts (7.499 s)
|
||||
Скачивание и привязка вложений (e2e)
|
||||
чтение
|
||||
✓ участник получает метаданные и оригинал побайтно (329 ms)
|
||||
✓ JPEG отдаётся с истинным Content-Type (210 ms)
|
||||
✓ посторонний и неизвестный ID получают одинаковый 404 (265 ms)
|
||||
✓ UUID nil корректного формата — 404, а не 400 (99 ms)
|
||||
✓ UUID v7 корректного формата — 404, а не 400 (46 ms)
|
||||
✓ без токена 401, неверный UUID 400 (193 ms)
|
||||
✓ variant: original явно, отсутствующий 404, неизвестный и повторённый 400 (218 ms)
|
||||
✓ нет доступа к S3 — метаданные 200, content 503 с заголовками (225 ms)
|
||||
✓ объект пропал из S3 — 500 без внутренних деталей (168 ms)
|
||||
attachment_ids в сообщениях
|
||||
✓ text="" с вложением принимается, Message читается вторым участником (164 ms)
|
||||
✓ порядок [B, A] сохраняется в ответе, истории и повторе; перестановка с тем же ключом — 409 (430 ms)
|
||||
✓ отсутствующий attachment_ids и [] равны при повторе, изменение списка — 409 (179 ms)
|
||||
✓ вложение другого чата и неизвестное — одинаковый 404, сообщение не создаётся (159 ms)
|
||||
✓ смесь своего и чужого вложения — 404 (316 ms)
|
||||
✓ посторонний не может отправить вложение в чужой чат (131 ms)
|
||||
✓ любой участник может сослаться на вложение чата, вложение можно использовать повторно (165 ms)
|
||||
✓ ровно 10 вложений принимается, 11 — 400 (767 ms)
|
||||
✓ UUID в другом регистре — тот же ID и дубль (201 ms)
|
||||
✓ text отсутствует — 400 (152 ms)
|
||||
✓ text null — 400 (159 ms)
|
||||
✓ attachment_ids null — 400 (137 ms)
|
||||
✓ attachment_ids строка — 400 (149 ms)
|
||||
✓ элемент не UUID — 400 (122 ms)
|
||||
✓ элемент число — 400 (113 ms)
|
||||
✓ дубликат — 400 (128 ms)
|
||||
✓ пустой текст без вложений — 400 (115 ms)
|
||||
✓ пробельный текст без вложений — 400 (95 ms)
|
||||
|
||||
PASS test/concurrency.e2e-spec.ts
|
||||
Безопасность при конкурентном доступе (e2e)
|
||||
✓ принимает 100 конкурентных сообщений от 10 клиентов и сохраняет ровно 100 уникальных (1147 ms)
|
||||
✓ регистрирует ровно одного пользователя при гонке 10 запросов за один и тот же username (192 ms)
|
||||
|
||||
[31m[Nest] 102 - [39m11/20/2026, 2:32:13 PM [31m ERROR[39m [38;5;3m[HttpExceptionFilter] [39m[31mObject:[39m
|
||||
{
|
||||
"event": "http.error",
|
||||
"message": "session cache unavailable",
|
||||
"method": "DELETE",
|
||||
"route": "/api/v1/auth/sessions/current",
|
||||
"status": 503,
|
||||
"request_id": "f7de03bc-bdf0-42ee-a767-01a1c55c8900"
|
||||
}
|
||||
|
||||
[31m[Nest] 102 - [39m11/20/2026, 2:32:13 PM [31m ERROR[39m [38;5;3m[HttpExceptionFilter] [39m[31mObject:[39m
|
||||
{
|
||||
"event": "http.error",
|
||||
"message": "session cache unavailable",
|
||||
"method": "DELETE",
|
||||
"route": "/api/v1/auth/sessions/current",
|
||||
"status": 503,
|
||||
"request_id": "3cb57701-e14e-4d0c-95b1-74125f609d5f"
|
||||
}
|
||||
|
||||
[31m[Nest] 102 - [39m11/20/2026, 2:32:14 PM [31m ERROR[39m [38;5;3m[HttpExceptionFilter] [39m[31mObject:[39m
|
||||
{
|
||||
"event": "http.error",
|
||||
"message": "database unavailable",
|
||||
"method": "GET",
|
||||
"route": "/health/ready",
|
||||
"status": 503,
|
||||
"request_id": "c84cbd52-47cb-410d-ae40-7028a9bd486f"
|
||||
}
|
||||
|
||||
[31m[Nest] 102 - [39m11/20/2026, 2:32:14 PM [31m ERROR[39m [38;5;3m[HttpExceptionFilter] [39m[31mObject:[39m
|
||||
{
|
||||
"event": "http.error",
|
||||
"message": "database unavailable",
|
||||
"method": "GET",
|
||||
"route": "/api/v1/chats",
|
||||
"status": 503,
|
||||
"request_id": "0f917fec-5b5c-4001-9fad-02c9c41c29a6"
|
||||
}
|
||||
|
||||
[31m[Nest] 102 - [39m11/20/2026, 2:32:14 PM [31m ERROR[39m [38;5;3m[HttpExceptionFilter] [39m[31mObject:[39m
|
||||
{
|
||||
"event": "http.error",
|
||||
"message": "database unavailable",
|
||||
"method": "GET",
|
||||
"route": "/api/v1/chats",
|
||||
"status": 503,
|
||||
"request_id": "ce2feb99-927d-4095-971d-d4a91e5674d3"
|
||||
}
|
||||
|
||||
[31m[Nest] 102 - [39m11/20/2026, 2:32:14 PM [31m ERROR[39m [38;5;3m[HttpExceptionFilter] [39m[31mObject:[39m
|
||||
{
|
||||
"event": "http.error",
|
||||
"message": "database unavailable",
|
||||
"method": "POST",
|
||||
"route": "/api/v1/users",
|
||||
"status": 503,
|
||||
"request_id": "2e2483c7-5f3d-4218-94f0-5dc4813905f4"
|
||||
}
|
||||
|
||||
PASS test/idempotency.e2e-spec.ts
|
||||
Идемпотентность отправки и X-Instance-Id (e2e)
|
||||
✓ требует Idempotency-Key правильного формата (147 ms)
|
||||
✓ повтор с тем же ключом и телом возвращает тот же 201 и то же сообщение (107 ms)
|
||||
✓ тот же ключ с другим телом — 409, сообщение не создаётся (106 ms)
|
||||
✓ 20 параллельных запросов с одним ключом через два экземпляра создают ровно одно сообщение (312 ms)
|
||||
✓ область ключа — пользователь и чат: одинаковый ключ у разных пользователей и чатов независим (238 ms)
|
||||
✓ повтор заново проверяет сессию (219 ms)
|
||||
✓ после IDEMPOTENCY_TTL_SECONDS ключ можно использовать заново (1832 ms)
|
||||
✓ X-Instance-Id есть в успешных ответах, ошибках и health и стабилен в процессе (58 ms)
|
||||
|
||||
[31m[Nest] 102 - [39m11/20/2026, 2:32:14 PM [31m ERROR[39m [38;5;3m[HttpExceptionFilter] [39m[31mObject:[39m
|
||||
{
|
||||
"event": "http.error",
|
||||
"message": "Internal server error",
|
||||
"method": "GET",
|
||||
"route": "/api/v1/chats",
|
||||
"status": 500,
|
||||
"request_id": "dd8bae40-6d02-4739-b4ac-194bb45a416c"
|
||||
}
|
||||
|
||||
PASS test/failures.e2e-spec.ts
|
||||
Уровень 4: expiry, отказы зависимостей, rollback (e2e)
|
||||
✓ сессия истекает при коротком SESSION_TTL_SECONDS, в том числе из кэша (2882 ms)
|
||||
✓ без Redis вход и проверка сессии работают через PostgreSQL, logout — 503 без отзыва (160 ms)
|
||||
✓ logout через реплику без связи с Redis не создаёт расхождения с репликой, у которой сессия в кэше (424 ms)
|
||||
✓ без PostgreSQL отвечает ограниченным 503 и не пускает анонима (263 ms)
|
||||
✓ неожиданное исключение — 500 internal_error без деталей, с X-Request-Id и X-Instance-Id (267 ms)
|
||||
✓ откатывает создание чата целиком, если один из участников не существует (226 ms)
|
||||
|
||||
PASS test/docs.e2e-spec.ts
|
||||
Swagger UI и примеры (e2e)
|
||||
✓ отдаёт Swagger UI и документ контракта со всеми путями (39 ms)
|
||||
✓ каждый пример запроса и ответа проходит JSON Schema контракта (407 ms)
|
||||
|
||||
A worker process has failed to exit gracefully and has been force exited. This is likely caused by tests leaking due to improper teardown. Try running with --detectOpenHandles to find leaks. Active timers can also cause this, ensure that .unref() was called on them.
|
||||
Test Suites: 12 passed, 12 total
|
||||
Tests: 201 passed, 201 total
|
||||
Snapshots: 0 total
|
||||
Time: 30.222 s
|
||||
Ran all test suites.
|
||||
|
||||
@@ -0,0 +1,428 @@
|
||||
{
|
||||
"uid": "messenger-overview",
|
||||
"title": "Messenger overview",
|
||||
"tags": ["messenger", "lab6"],
|
||||
"timezone": "browser",
|
||||
"schemaVersion": 39,
|
||||
"version": 2,
|
||||
"editable": false,
|
||||
"graphTooltip": 1,
|
||||
"refresh": "10s",
|
||||
"time": { "from": "now-30m", "to": "now" },
|
||||
"templating": { "list": [] },
|
||||
"annotations": { "list": [] },
|
||||
"panels": [
|
||||
{ "type": "row", "title": "SLO и alerts", "id": 100, "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 } },
|
||||
{
|
||||
"type": "stat",
|
||||
"title": "Доступность API, 1 ч (SLO ≥ 99% без 5xx)",
|
||||
"id": 20,
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"gridPos": { "h": 5, "w": 6, "x": 0, "y": 1 },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "percentunit", "decimals": 2,
|
||||
"thresholds": { "mode": "absolute", "steps": [{ "color": "red", "value": null }, { "color": "green", "value": 0.99 }] }
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": { "reduceOptions": { "calcs": ["lastNotNull"] }, "colorMode": "background" },
|
||||
"targets": [{ "refId": "A", "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "1 - messenger:http_error_ratio:rate1h" }]
|
||||
},
|
||||
{
|
||||
"type": "stat",
|
||||
"title": "Запросы быстрее 500 мс, 1 ч (SLO ≥ 95%)",
|
||||
"id": 21,
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"gridPos": { "h": 5, "w": 6, "x": 6, "y": 1 },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "percentunit", "decimals": 2,
|
||||
"thresholds": { "mode": "absolute", "steps": [{ "color": "red", "value": null }, { "color": "green", "value": 0.95 }] }
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": { "reduceOptions": { "calcs": ["lastNotNull"] }, "colorMode": "background" },
|
||||
"targets": [{ "refId": "A", "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "messenger:http_fast_ratio:rate1h" }]
|
||||
},
|
||||
{
|
||||
"type": "stat",
|
||||
"title": "Вложения готовы за ≤ 10 с, 1 ч (SLO ≥ 95%)",
|
||||
"id": 22,
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"gridPos": { "h": 5, "w": 6, "x": 12, "y": 1 },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "percentunit", "decimals": 2,
|
||||
"thresholds": { "mode": "absolute", "steps": [{ "color": "red", "value": null }, { "color": "green", "value": 0.95 }] }
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": { "reduceOptions": { "calcs": ["lastNotNull"] }, "colorMode": "background" },
|
||||
"targets": [{ "refId": "A", "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "messenger:time_to_ready_fast_ratio:rate1h" }]
|
||||
},
|
||||
{
|
||||
"type": "state-timeline",
|
||||
"title": "Alerts (pending / firing)",
|
||||
"id": 23,
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"gridPos": { "h": 5, "w": 6, "x": 18, "y": 1 },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": { "mode": "thresholds" },
|
||||
"thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }, { "color": "orange", "value": 1 }, { "color": "red", "value": 2 }] },
|
||||
"mappings": [{ "type": "value", "options": { "1": { "text": "pending" }, "2": { "text": "firing" } } }]
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": { "showValue": "never", "mergeValues": true, "legend": { "showLegend": false } },
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"expr": "max by (alertname) (ALERTS{alertstate=\"firing\"}) * 2 or max by (alertname) (ALERTS{alertstate=\"pending\"})",
|
||||
"legendFormat": "{{alertname}}"
|
||||
}
|
||||
]
|
||||
},
|
||||
|
||||
{ "type": "row", "title": "API", "id": 101, "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 6 } },
|
||||
{
|
||||
"type": "timeseries",
|
||||
"title": "RPS по маршрутам (без health)",
|
||||
"id": 1,
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 7 },
|
||||
"fieldConfig": { "defaults": { "unit": "reqps" }, "overrides": [] },
|
||||
"options": { "legend": { "displayMode": "table", "placement": "right", "calcs": ["lastNotNull"] } },
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"expr": "sum by (http_request_method, http_route) (rate(http_server_requests_total{job=\"messenger-api\", http_route!~\"/health.*\"}[$__rate_interval]))",
|
||||
"legendFormat": "{{http_request_method}} {{http_route}}"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"type": "timeseries",
|
||||
"title": "Ответы по классам статуса (без health)",
|
||||
"id": 2,
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 7 },
|
||||
"fieldConfig": { "defaults": { "unit": "reqps" }, "overrides": [] },
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"expr": "sum by (http_response_status_class) (rate(http_server_requests_total{job=\"messenger-api\", http_route!~\"/health.*\"}[$__rate_interval]))",
|
||||
"legendFormat": "{{http_response_status_class}}"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"type": "timeseries",
|
||||
"title": "Доля 5xx среди бизнес-запросов (окно 5m, порог alert 5%)",
|
||||
"id": 3,
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"gridPos": { "h": 8, "w": 8, "x": 0, "y": 15 },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "percentunit",
|
||||
"min": 0,
|
||||
"thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }, { "color": "red", "value": 0.05 }] },
|
||||
"custom": { "thresholdsStyle": { "mode": "line" } }
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"targets": [
|
||||
{ "refId": "A", "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "messenger:http_error_ratio:rate5m", "legendFormat": "5xx ratio" }
|
||||
]
|
||||
},
|
||||
{
|
||||
"type": "timeseries",
|
||||
"title": "Длительность API p50 / p95 / p99 (все реплики, без health)",
|
||||
"id": 4,
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"gridPos": { "h": 8, "w": 8, "x": 8, "y": 15 },
|
||||
"fieldConfig": { "defaults": { "unit": "s", "min": 0 }, "overrides": [] },
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"expr": "histogram_quantile(0.50, sum by (le) (rate(http_server_request_duration_seconds_bucket{job=\"messenger-api\", http_route!~\"/health.*\"}[$__rate_interval])))",
|
||||
"legendFormat": "p50"
|
||||
},
|
||||
{
|
||||
"refId": "B",
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"expr": "histogram_quantile(0.95, sum by (le) (rate(http_server_request_duration_seconds_bucket{job=\"messenger-api\", http_route!~\"/health.*\"}[$__rate_interval])))",
|
||||
"legendFormat": "p95"
|
||||
},
|
||||
{
|
||||
"refId": "C",
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"expr": "histogram_quantile(0.99, sum by (le) (rate(http_server_request_duration_seconds_bucket{job=\"messenger-api\", http_route!~\"/health.*\"}[$__rate_interval])))",
|
||||
"legendFormat": "p99"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"type": "timeseries",
|
||||
"title": "p95 по маршрутам",
|
||||
"id": 5,
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"gridPos": { "h": 8, "w": 8, "x": 16, "y": 15 },
|
||||
"fieldConfig": { "defaults": { "unit": "s", "min": 0 }, "overrides": [] },
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"expr": "histogram_quantile(0.95, sum by (le, http_request_method, http_route) (rate(http_server_request_duration_seconds_bucket{job=\"messenger-api\", http_route!~\"/health.*\"}[$__rate_interval])))",
|
||||
"legendFormat": "{{http_request_method}} {{http_route}}"
|
||||
}
|
||||
]
|
||||
},
|
||||
|
||||
{ "type": "row", "title": "Очередь и worker", "id": 102, "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 23 } },
|
||||
{
|
||||
"type": "timeseries",
|
||||
"title": "Глубина очереди attachment.jobs (из attachment_jobs)",
|
||||
"id": 30,
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"gridPos": { "h": 8, "w": 8, "x": 0, "y": 24 },
|
||||
"fieldConfig": { "defaults": { "unit": "short", "min": 0 }, "overrides": [] },
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"expr": "max by (state) (jobs_queue_depth{job=\"messenger-relay\"})",
|
||||
"legendFormat": "{{state}}"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"type": "timeseries",
|
||||
"title": "Возраст старейшего готового задания (порог alert 60 s)",
|
||||
"id": 31,
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"gridPos": { "h": 8, "w": 8, "x": 8, "y": 24 },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "s", "min": 0,
|
||||
"thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }, { "color": "red", "value": 60 }] },
|
||||
"custom": { "thresholdsStyle": { "mode": "line" } }
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"targets": [
|
||||
{ "refId": "A", "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "messenger:jobs_queue_oldest_age:max", "legendFormat": "oldest due job age" }
|
||||
]
|
||||
},
|
||||
{
|
||||
"type": "timeseries",
|
||||
"title": "Worker: занято / ёмкость",
|
||||
"id": 32,
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"gridPos": { "h": 8, "w": 8, "x": 16, "y": 24 },
|
||||
"fieldConfig": { "defaults": { "unit": "short", "min": 0 }, "overrides": [] },
|
||||
"targets": [
|
||||
{ "refId": "A", "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "sum(worker_jobs_busy{job=\"messenger-worker\"})", "legendFormat": "busy" },
|
||||
{ "refId": "B", "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "sum(worker_jobs_capacity{job=\"messenger-worker\"})", "legendFormat": "capacity" },
|
||||
{ "refId": "C", "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "sum(avg_over_time(worker_jobs_busy{job=\"messenger-worker\"}[5m])) / sum(worker_jobs_capacity{job=\"messenger-worker\"})", "legendFormat": "saturation 5m" }
|
||||
]
|
||||
},
|
||||
{
|
||||
"type": "timeseries",
|
||||
"title": "Задания по исходу (в секунду)",
|
||||
"id": 6,
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"gridPos": { "h": 8, "w": 8, "x": 0, "y": 32 },
|
||||
"fieldConfig": { "defaults": { "unit": "ops" }, "overrides": [] },
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"expr": "sum by (job_type, outcome) (rate(jobs_processed_total{job=\"messenger-worker\"}[$__rate_interval]))",
|
||||
"legendFormat": "{{job_type}} {{outcome}}"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"type": "timeseries",
|
||||
"title": "Длительность обработки задания p50 / p95 / p99",
|
||||
"id": 7,
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"gridPos": { "h": 8, "w": 8, "x": 8, "y": 32 },
|
||||
"fieldConfig": { "defaults": { "unit": "s", "min": 0 }, "overrides": [] },
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"expr": "histogram_quantile(0.50, sum by (le) (rate(jobs_duration_seconds_bucket{job=\"messenger-worker\", outcome=\"success\"}[$__rate_interval])))",
|
||||
"legendFormat": "p50 success"
|
||||
},
|
||||
{
|
||||
"refId": "B",
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"expr": "histogram_quantile(0.95, sum by (le) (rate(jobs_duration_seconds_bucket{job=\"messenger-worker\", outcome=\"success\"}[$__rate_interval])))",
|
||||
"legendFormat": "p95 success"
|
||||
},
|
||||
{
|
||||
"refId": "C",
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"expr": "histogram_quantile(0.99, sum by (le) (rate(jobs_duration_seconds_bucket{job=\"messenger-worker\", outcome=\"success\"}[$__rate_interval])))",
|
||||
"legendFormat": "p99 success"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"type": "timeseries",
|
||||
"title": "Время от upload до ready p50 / p95 / p99",
|
||||
"id": 33,
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"gridPos": { "h": 8, "w": 8, "x": 16, "y": 32 },
|
||||
"fieldConfig": { "defaults": { "unit": "s", "min": 0 }, "overrides": [] },
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"expr": "histogram_quantile(0.50, sum by (le) (rate(attachments_time_to_ready_seconds_bucket{job=\"messenger-worker\"}[$__rate_interval])))",
|
||||
"legendFormat": "p50"
|
||||
},
|
||||
{
|
||||
"refId": "B",
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"expr": "histogram_quantile(0.95, sum by (le) (rate(attachments_time_to_ready_seconds_bucket{job=\"messenger-worker\"}[$__rate_interval])))",
|
||||
"legendFormat": "p95"
|
||||
},
|
||||
{
|
||||
"refId": "C",
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"expr": "histogram_quantile(0.99, sum by (le) (rate(attachments_time_to_ready_seconds_bucket{job=\"messenger-worker\"}[$__rate_interval])))",
|
||||
"legendFormat": "p99"
|
||||
}
|
||||
]
|
||||
},
|
||||
|
||||
{ "type": "row", "title": "БД и ресурсы", "id": 103, "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 40 } },
|
||||
{
|
||||
"type": "timeseries",
|
||||
"title": "Пул БД: занято / ожидают соединения",
|
||||
"id": 40,
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"gridPos": { "h": 8, "w": 8, "x": 0, "y": 41 },
|
||||
"fieldConfig": { "defaults": { "unit": "short", "min": 0 }, "overrides": [] },
|
||||
"targets": [
|
||||
{ "refId": "A", "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "sum by (job) (db_client_connection_count{db_client_connection_state=\"used\"})", "legendFormat": "{{job}} used" },
|
||||
{ "refId": "B", "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "sum by (job) (db_client_connection_pending_requests)", "legendFormat": "{{job}} waiting" },
|
||||
{ "refId": "C", "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "sum by (job) (db_client_connection_max)", "legendFormat": "{{job}} max" }
|
||||
]
|
||||
},
|
||||
{
|
||||
"type": "timeseries",
|
||||
"title": "CPU процессов (доля ядра)",
|
||||
"id": 41,
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"gridPos": { "h": 8, "w": 8, "x": 8, "y": 41 },
|
||||
"fieldConfig": { "defaults": { "unit": "percentunit", "min": 0 }, "overrides": [] },
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"expr": "sum by (job, instance) (rate(process_cpu_time_seconds_total{job=~\"messenger-.+\"}[$__rate_interval]))",
|
||||
"legendFormat": "{{job}} {{instance}}"
|
||||
},
|
||||
{
|
||||
"refId": "B",
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"expr": "rate(otelcol_process_cpu_seconds{job=\"otel-collector\"}[$__rate_interval])",
|
||||
"legendFormat": "otel-collector"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"type": "timeseries",
|
||||
"title": "RAM процессов (RSS)",
|
||||
"id": 42,
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"gridPos": { "h": 8, "w": 8, "x": 16, "y": 41 },
|
||||
"fieldConfig": { "defaults": { "unit": "bytes", "min": 0 }, "overrides": [] },
|
||||
"targets": [
|
||||
{ "refId": "A", "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "process_memory_usage_bytes{job=~\"messenger-.+\"}", "legendFormat": "{{job}} {{instance}}" },
|
||||
{ "refId": "B", "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "otelcol_process_memory_rss{job=\"otel-collector\"}", "legendFormat": "otel-collector" }
|
||||
]
|
||||
},
|
||||
|
||||
{ "type": "row", "title": "Конвейер телеметрии (Collector)", "id": 104, "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 49 } },
|
||||
{
|
||||
"type": "timeseries",
|
||||
"title": "Принято / отправлено / ошибки отправки (в секунду)",
|
||||
"id": 50,
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 50 },
|
||||
"fieldConfig": { "defaults": { "unit": "short", "min": 0 }, "overrides": [] },
|
||||
"targets": [
|
||||
{ "refId": "A", "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "sum(rate(otelcol_receiver_accepted_spans[$__rate_interval]))", "legendFormat": "spans accepted" },
|
||||
{ "refId": "B", "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "sum(rate(otelcol_exporter_sent_spans[$__rate_interval]))", "legendFormat": "spans sent" },
|
||||
{ "refId": "C", "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "sum(rate(otelcol_exporter_send_failed_spans[$__rate_interval])) or vector(0)", "legendFormat": "spans failed" },
|
||||
{ "refId": "D", "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "sum(rate(otelcol_exporter_sent_log_records[$__rate_interval]))", "legendFormat": "logs sent" },
|
||||
{ "refId": "E", "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "sum(rate(otelcol_exporter_send_failed_log_records[$__rate_interval])) or vector(0)", "legendFormat": "logs failed" },
|
||||
{ "refId": "F", "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "sum(rate(otelcol_receiver_refused_spans[$__rate_interval])) + sum(rate(otelcol_receiver_refused_log_records[$__rate_interval])) or vector(0)", "legendFormat": "refused (memory_limiter)" }
|
||||
]
|
||||
},
|
||||
{
|
||||
"type": "timeseries",
|
||||
"title": "Очередь exporter (ограничена queue_size)",
|
||||
"id": 51,
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 50 },
|
||||
"fieldConfig": { "defaults": { "unit": "short", "min": 0 }, "overrides": [] },
|
||||
"targets": [
|
||||
{ "refId": "A", "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "max by (exporter) (otelcol_exporter_queue_size)", "legendFormat": "{{exporter}} size" },
|
||||
{ "refId": "B", "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "max(otelcol_exporter_queue_capacity)", "legendFormat": "capacity" }
|
||||
]
|
||||
},
|
||||
|
||||
{ "type": "row", "title": "Логи и трассы", "id": 105, "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 58 } },
|
||||
{
|
||||
"type": "logs",
|
||||
"title": "Логи API и worker (WARN и выше)",
|
||||
"id": 8,
|
||||
"datasource": { "type": "loki", "uid": "loki" },
|
||||
"gridPos": { "h": 10, "w": 12, "x": 0, "y": 59 },
|
||||
"options": { "showTime": true, "wrapLogMessage": true, "sortOrder": "Descending", "enableLogDetails": true },
|
||||
"targets": [
|
||||
{ "refId": "A", "datasource": { "type": "loki", "uid": "loki" }, "expr": "{service_name=~\"messenger-.+\"} | severity_text=~\"WARN|ERROR|FATAL\"" }
|
||||
]
|
||||
},
|
||||
{
|
||||
"type": "logs",
|
||||
"title": "Все логи (http.request, job.handled, …)",
|
||||
"id": 9,
|
||||
"datasource": { "type": "loki", "uid": "loki" },
|
||||
"gridPos": { "h": 10, "w": 12, "x": 12, "y": 59 },
|
||||
"options": { "showTime": true, "wrapLogMessage": true, "sortOrder": "Descending", "enableLogDetails": true },
|
||||
"targets": [
|
||||
{ "refId": "A", "datasource": { "type": "loki", "uid": "loki" }, "expr": "{service_name=~\"messenger-.+\"}" }
|
||||
]
|
||||
},
|
||||
{
|
||||
"type": "table",
|
||||
"title": "Последние трассы (HTTP server и consumer spans)",
|
||||
"id": 10,
|
||||
"datasource": { "type": "tempo", "uid": "tempo" },
|
||||
"gridPos": { "h": 10, "w": 24, "x": 0, "y": 69 },
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": { "type": "tempo", "uid": "tempo" },
|
||||
"queryType": "traceql",
|
||||
"query": "{resource.service.name=~\"messenger-.+\" && (kind=server || kind=consumer)}",
|
||||
"limit": 20,
|
||||
"tableType": "traces"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,10 @@
|
||||
apiVersion: 1
|
||||
|
||||
providers:
|
||||
- name: messenger
|
||||
folder: Messenger
|
||||
type: file
|
||||
disableDeletion: true
|
||||
allowUiUpdates: false
|
||||
options:
|
||||
path: /var/lib/grafana/dashboards
|
||||
@@ -0,0 +1,45 @@
|
||||
apiVersion: 1
|
||||
|
||||
datasources:
|
||||
- name: Prometheus
|
||||
uid: prometheus
|
||||
type: prometheus
|
||||
access: proxy
|
||||
url: http://prometheus:9090
|
||||
isDefault: true
|
||||
jsonData:
|
||||
httpMethod: POST
|
||||
exemplarTraceIdDestinations:
|
||||
- name: trace_id
|
||||
datasourceUid: tempo
|
||||
- name: Loki
|
||||
uid: loki
|
||||
type: loki
|
||||
access: proxy
|
||||
url: http://loki:3100
|
||||
jsonData:
|
||||
derivedFields:
|
||||
- name: trace_id
|
||||
matcherType: label
|
||||
matcherRegex: trace_id
|
||||
url: "$${__value.raw}"
|
||||
datasourceUid: tempo
|
||||
- name: Tempo
|
||||
uid: tempo
|
||||
type: tempo
|
||||
access: proxy
|
||||
url: http://tempo:3200
|
||||
jsonData:
|
||||
tracesToLogsV2:
|
||||
datasourceUid: loki
|
||||
spanStartTimeShift: -5m
|
||||
spanEndTimeShift: 5m
|
||||
filterByTraceID: false
|
||||
customQuery: true
|
||||
query: '{service_name=~".+"} | trace_id="$${__span.traceId}"'
|
||||
tracesToMetrics:
|
||||
datasourceUid: prometheus
|
||||
serviceMap:
|
||||
datasourceUid: prometheus
|
||||
nodeGraph:
|
||||
enabled: true
|
||||
@@ -0,0 +1,43 @@
|
||||
global
|
||||
log stdout format raw local0 info
|
||||
maxconn 2000
|
||||
|
||||
defaults
|
||||
mode http
|
||||
log global
|
||||
option httplog
|
||||
option dontlognull
|
||||
timeout connect 1s
|
||||
timeout client 30s
|
||||
timeout server 10s
|
||||
timeout http-request 10s
|
||||
timeout queue 5s
|
||||
timeout check 1s
|
||||
retries 2
|
||||
option redispatch 1
|
||||
retry-on conn-failure
|
||||
unique-id-format %[uuid()]
|
||||
http-error status 502 content-type application/json hdr X-Request-Id "%[unique-id]" lf-string "{\"error\":{\"code\":\"unavailable\",\"message\":\"bad gateway\",\"request_id\":\"%[unique-id]\"}}"
|
||||
http-error status 503 content-type application/json hdr X-Request-Id "%[unique-id]" lf-string "{\"error\":{\"code\":\"unavailable\",\"message\":\"no API replica available\",\"request_id\":\"%[unique-id]\"}}"
|
||||
http-error status 504 content-type application/json hdr X-Request-Id "%[unique-id]" lf-string "{\"error\":{\"code\":\"unavailable\",\"message\":\"gateway timeout\",\"request_id\":\"%[unique-id]\"}}"
|
||||
|
||||
resolvers docker
|
||||
nameserver dns 127.0.0.11:53
|
||||
resolve_retries 3
|
||||
timeout resolve 1s
|
||||
timeout retry 1s
|
||||
hold valid 5s
|
||||
hold obsolete 5s
|
||||
accepted_payload_size 8192
|
||||
|
||||
frontend public
|
||||
bind :8080
|
||||
default_backend api
|
||||
|
||||
backend api
|
||||
balance roundrobin
|
||||
http-request set-timeout server 60s if { method POST } { path_reg ^/api/v1/chats/[^/]+/attachments$ }
|
||||
option httpchk GET /health/core
|
||||
http-check expect status 200
|
||||
default-server check inter 1s fall 2 rise 2 resolvers docker resolve-prefer ipv4 init-addr none
|
||||
server-template api 1-4 api:8080
|
||||
@@ -0,0 +1,43 @@
|
||||
auth_enabled: false
|
||||
|
||||
server:
|
||||
http_listen_address: 0.0.0.0
|
||||
http_listen_port: 3100
|
||||
grpc_listen_port: 9096
|
||||
log_level: warn
|
||||
|
||||
common:
|
||||
instance_addr: 127.0.0.1
|
||||
path_prefix: /loki
|
||||
replication_factor: 1
|
||||
storage:
|
||||
filesystem:
|
||||
chunks_directory: /loki/chunks
|
||||
rules_directory: /loki/rules
|
||||
ring:
|
||||
kvstore:
|
||||
store: inmemory
|
||||
|
||||
schema_config:
|
||||
configs:
|
||||
- from: 2026-01-01
|
||||
store: tsdb
|
||||
object_store: filesystem
|
||||
schema: v13
|
||||
index:
|
||||
prefix: index_
|
||||
period: 24h
|
||||
|
||||
limits_config:
|
||||
allow_structured_metadata: true
|
||||
retention_period: 168h
|
||||
ingestion_rate_mb: 8
|
||||
ingestion_burst_size_mb: 16
|
||||
|
||||
compactor:
|
||||
working_directory: /loki/compactor
|
||||
retention_enabled: true
|
||||
delete_request_store: filesystem
|
||||
|
||||
analytics:
|
||||
reporting_enabled: false
|
||||
@@ -0,0 +1,85 @@
|
||||
extensions:
|
||||
health_check:
|
||||
endpoint: 0.0.0.0:13133
|
||||
|
||||
receivers:
|
||||
otlp:
|
||||
protocols:
|
||||
http:
|
||||
endpoint: 0.0.0.0:4318
|
||||
max_request_body_size: 4194304
|
||||
|
||||
processors:
|
||||
memory_limiter:
|
||||
check_interval: 1s
|
||||
limit_mib: 160
|
||||
spike_limit_mib: 40
|
||||
batch:
|
||||
send_batch_size: 512
|
||||
send_batch_max_size: 1024
|
||||
timeout: 2s
|
||||
|
||||
exporters:
|
||||
otlphttp/prometheus:
|
||||
endpoint: http://prometheus:9090/api/v1/otlp
|
||||
timeout: 5s
|
||||
sending_queue:
|
||||
enabled: true
|
||||
num_consumers: 2
|
||||
queue_size: 200
|
||||
retry_on_failure:
|
||||
enabled: true
|
||||
initial_interval: 1s
|
||||
max_interval: 10s
|
||||
max_elapsed_time: 60s
|
||||
otlphttp/loki:
|
||||
endpoint: http://loki:3100/otlp
|
||||
timeout: 5s
|
||||
sending_queue:
|
||||
enabled: true
|
||||
num_consumers: 2
|
||||
queue_size: 200
|
||||
retry_on_failure:
|
||||
enabled: true
|
||||
initial_interval: 1s
|
||||
max_interval: 10s
|
||||
max_elapsed_time: 60s
|
||||
otlphttp/tempo:
|
||||
endpoint: http://tempo:4318
|
||||
timeout: 5s
|
||||
sending_queue:
|
||||
enabled: true
|
||||
num_consumers: 2
|
||||
queue_size: 200
|
||||
retry_on_failure:
|
||||
enabled: true
|
||||
initial_interval: 1s
|
||||
max_interval: 10s
|
||||
max_elapsed_time: 60s
|
||||
|
||||
service:
|
||||
extensions: [health_check]
|
||||
telemetry:
|
||||
logs:
|
||||
level: warn
|
||||
metrics:
|
||||
level: normal
|
||||
readers:
|
||||
- pull:
|
||||
exporter:
|
||||
prometheus:
|
||||
host: 0.0.0.0
|
||||
port: 8888
|
||||
pipelines:
|
||||
metrics:
|
||||
receivers: [otlp]
|
||||
processors: [memory_limiter, batch]
|
||||
exporters: [otlphttp/prometheus]
|
||||
logs:
|
||||
receivers: [otlp]
|
||||
processors: [memory_limiter, batch]
|
||||
exporters: [otlphttp/loki]
|
||||
traces:
|
||||
receivers: [otlp]
|
||||
processors: [memory_limiter, batch]
|
||||
exporters: [otlphttp/tempo]
|
||||
@@ -0,0 +1,23 @@
|
||||
global:
|
||||
scrape_interval: 15s
|
||||
evaluation_interval: 15s
|
||||
|
||||
otlp:
|
||||
promote_resource_attributes:
|
||||
- deployment.environment.name
|
||||
- service.version
|
||||
|
||||
storage:
|
||||
tsdb:
|
||||
out_of_order_time_window: 30m
|
||||
|
||||
rule_files:
|
||||
- /etc/prometheus/rules/*.yml
|
||||
|
||||
scrape_configs:
|
||||
- job_name: otel-collector
|
||||
static_configs:
|
||||
- targets: ["otel-collector:8888"]
|
||||
- job_name: prometheus
|
||||
static_configs:
|
||||
- targets: ["localhost:9090"]
|
||||
@@ -0,0 +1,77 @@
|
||||
groups:
|
||||
- name: messenger-sli
|
||||
interval: 15s
|
||||
rules:
|
||||
- record: messenger:http_requests:rate5m
|
||||
expr: sum(rate(http_server_requests_total{job="messenger-api", http_route!~"/health.*"}[5m]))
|
||||
- record: messenger:http_errors:rate5m
|
||||
expr: sum(rate(http_server_requests_total{job="messenger-api", http_route!~"/health.*", http_response_status_class="5xx"}[5m])) or vector(0)
|
||||
- record: messenger:http_error_ratio:rate5m
|
||||
expr: messenger:http_errors:rate5m / messenger:http_requests:rate5m
|
||||
- record: messenger:http_error_ratio:rate1h
|
||||
expr: |
|
||||
(sum(rate(http_server_requests_total{job="messenger-api", http_route!~"/health.*", http_response_status_class="5xx"}[1h])) or vector(0))
|
||||
/ sum(rate(http_server_requests_total{job="messenger-api", http_route!~"/health.*"}[1h]))
|
||||
- record: messenger:http_latency_p95:rate5m
|
||||
expr: histogram_quantile(0.95, sum by (le) (rate(http_server_request_duration_seconds_bucket{job="messenger-api", http_route!~"/health.*"}[5m])))
|
||||
- record: messenger:http_fast_ratio:rate1h
|
||||
expr: |
|
||||
sum(rate(http_server_request_duration_seconds_bucket{job="messenger-api", http_route!~"/health.*", le="0.5"}[1h]))
|
||||
/ sum(rate(http_server_request_duration_seconds_count{job="messenger-api", http_route!~"/health.*"}[1h]))
|
||||
- record: messenger:time_to_ready_fast_ratio:rate1h
|
||||
expr: |
|
||||
sum(rate(attachments_time_to_ready_seconds_bucket{job="messenger-worker", le="10"}[1h]))
|
||||
/ sum(rate(attachments_time_to_ready_seconds_count{job="messenger-worker"}[1h]))
|
||||
- record: messenger:jobs_queue_oldest_age:max
|
||||
expr: max(jobs_queue_oldest_age_seconds{job="messenger-relay"})
|
||||
|
||||
- name: messenger-alerts
|
||||
rules:
|
||||
- alert: MessengerHighErrorRate
|
||||
expr: messenger:http_error_ratio:rate5m > 0.05 and messenger:http_requests:rate5m > 0.05
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "Доля 5xx бизнес-запросов API {{ $value | humanizePercentage }} за 5 минут (порог 5%)"
|
||||
description: "SLO доступности 99% за 1 ч: при 5% ошибок бюджет сгорает в 5 раз быстрее допустимого. Health-запросы не учитываются."
|
||||
runbook: "docs/runbooks.md#messengerhigherrorrate"
|
||||
- alert: MessengerApiNotReady
|
||||
expr: sum(increase(http_server_requests_total{job="messenger-api", http_route="/health/core", http_response_status_class="5xx"}[1m])) > 10
|
||||
for: 30s
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "Реплики API отвечают 503 на /health/core — HAProxy выводит их из балансировки"
|
||||
description: "Обычно недоступна БД или реплика останавливается. Пока все реплики не готовы, клиенты получают 503 от LB, и такие ответы не попадают в метрики API (MessengerHighErrorRate их не видит)."
|
||||
runbook: "docs/runbooks.md#messengerapinotready"
|
||||
- alert: MessengerJobQueueLag
|
||||
expr: messenger:jobs_queue_oldest_age:max > 60
|
||||
for: 1m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Старейшее готовое к обработке задание ждёт {{ $value | humanizeDuration }} (порог 60 s)"
|
||||
description: "Задания принимаются, но не обрабатываются: worker остановлен, не справляется или не получает сообщения из брокера."
|
||||
runbook: "docs/runbooks.md#messengerjobqueuelag"
|
||||
- alert: MessengerSlowApi
|
||||
expr: messenger:http_latency_p95:rate5m > 1 and messenger:http_requests:rate5m > 0.05
|
||||
for: 2m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "p95 бизнес-запросов API {{ $value | humanizeDuration }} за 5 минут (порог 1 s)"
|
||||
description: "Проверьте ожидание пула БД (db_client_connection_pending_requests) и spans prisma:engine:db_query в трассах медленных запросов."
|
||||
runbook: "docs/runbooks.md#messengerslowapi"
|
||||
- alert: TelemetryExportFailing
|
||||
expr: |
|
||||
sum(rate(otelcol_exporter_send_failed_spans[2m])) + sum(rate(otelcol_exporter_send_failed_log_records[2m]))
|
||||
+ sum(rate(otelcol_exporter_send_failed_metric_points[2m])) > 0
|
||||
or max(otelcol_exporter_queue_size / otelcol_exporter_queue_capacity) > 0.5
|
||||
for: 1m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Collector не может доставить телеметрию в хранилище"
|
||||
description: "Растёт очередь exporter или есть отказы отправки. Мессенджер работает, но часть трасс/логов теряется после заполнения ограниченной очереди."
|
||||
runbook: "docs/runbooks.md#telemetryexportfailing"
|
||||
@@ -0,0 +1,31 @@
|
||||
stream_over_http_enabled: true
|
||||
|
||||
server:
|
||||
http_listen_address: 0.0.0.0
|
||||
http_listen_port: 3200
|
||||
log_level: warn
|
||||
|
||||
distributor:
|
||||
receivers:
|
||||
otlp:
|
||||
protocols:
|
||||
http:
|
||||
endpoint: 0.0.0.0:4318
|
||||
|
||||
ingester:
|
||||
max_block_duration: 5m
|
||||
|
||||
compactor:
|
||||
compaction:
|
||||
block_retention: 168h
|
||||
|
||||
storage:
|
||||
trace:
|
||||
backend: local
|
||||
wal:
|
||||
path: /var/tempo/wal
|
||||
local:
|
||||
path: /var/tempo/blocks
|
||||
|
||||
usage_report:
|
||||
reporting_enabled: false
|
||||
@@ -0,0 +1,8 @@
|
||||
{
|
||||
"$schema": "https://json.schemastore.org/nest-cli",
|
||||
"collection": "@nestjs/schematics",
|
||||
"sourceRoot": "src",
|
||||
"compilerOptions": {
|
||||
"deleteOutDir": true
|
||||
}
|
||||
}
|
||||
Generated
+13078
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,69 @@
|
||||
{
|
||||
"name": "mtusi-messenger-lab6",
|
||||
"version": "1.0.0",
|
||||
"description": "Lab 6: messenger API with S3 attachments and background image processing (NestJS + Fastify)",
|
||||
"private": true,
|
||||
"license": "UNLICENSED",
|
||||
"scripts": {
|
||||
"build": "nest build",
|
||||
"start": "node dist/main.js",
|
||||
"start:dev": "nest start --watch",
|
||||
"lint": "eslint \"{src,test}/**/*.ts\"",
|
||||
"test": "jest --config ./test/jest-e2e.json",
|
||||
"test:watch": "jest --config ./test/jest-e2e.json --watch"
|
||||
},
|
||||
"dependencies": {
|
||||
"@aws-sdk/client-s3": "3.700.0",
|
||||
"@fastify/static": "7.0.4",
|
||||
"@nestjs/common": "^10.4.15",
|
||||
"@nestjs/core": "^10.4.15",
|
||||
"@nestjs/platform-fastify": "^10.4.15",
|
||||
"@nestjs/swagger": "7.4.2",
|
||||
"@opentelemetry/api": "1.9.0",
|
||||
"@opentelemetry/api-logs": "0.57.2",
|
||||
"@opentelemetry/core": "1.30.1",
|
||||
"@opentelemetry/exporter-logs-otlp-proto": "0.57.2",
|
||||
"@opentelemetry/exporter-metrics-otlp-proto": "0.57.2",
|
||||
"@opentelemetry/exporter-trace-otlp-proto": "0.57.2",
|
||||
"@opentelemetry/instrumentation-aws-sdk": "0.49.1",
|
||||
"@opentelemetry/instrumentation-http": "0.57.2",
|
||||
"@opentelemetry/resources": "1.30.1",
|
||||
"@opentelemetry/sdk-logs": "0.57.2",
|
||||
"@opentelemetry/sdk-metrics": "1.30.1",
|
||||
"@opentelemetry/sdk-node": "0.57.2",
|
||||
"@opentelemetry/sdk-trace-base": "1.30.1",
|
||||
"@opentelemetry/semantic-conventions": "1.28.0",
|
||||
"@prisma/client": "6.1.0",
|
||||
"@prisma/instrumentation": "6.1.0",
|
||||
"amqplib": "0.10.5",
|
||||
"argon2": "0.41.1",
|
||||
"class-transformer": "^0.5.1",
|
||||
"class-validator": "^0.14.1",
|
||||
"ioredis": "5.4.1",
|
||||
"reflect-metadata": "^0.2.2",
|
||||
"rxjs": "^7.8.1",
|
||||
"sharp": "0.33.5"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@nestjs/cli": "^10.4.9",
|
||||
"@nestjs/schematics": "^10.2.3",
|
||||
"@nestjs/testing": "^10.4.15",
|
||||
"@types/amqplib": "0.10.6",
|
||||
"@types/jest": "^29.5.14",
|
||||
"@types/node": "^20.17.9",
|
||||
"@types/supertest": "^6.0.2",
|
||||
"@typescript-eslint/eslint-plugin": "^8.18.1",
|
||||
"@typescript-eslint/parser": "^8.18.1",
|
||||
"ajv": "8.12.0",
|
||||
"eslint": "^8.57.1",
|
||||
"jest": "^29.7.0",
|
||||
"prisma": "6.1.0",
|
||||
"source-map-support": "^0.5.21",
|
||||
"supertest": "^7.0.0",
|
||||
"ts-jest": "^29.2.5",
|
||||
"ts-loader": "^9.5.1",
|
||||
"ts-node": "^10.9.2",
|
||||
"tsconfig-paths": "^4.2.0",
|
||||
"typescript": "^5.7.2"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,50 @@
|
||||
CREATE TABLE "users" (
|
||||
"id" UUID NOT NULL,
|
||||
"username" VARCHAR(32) NOT NULL,
|
||||
"display_name" VARCHAR(100) NOT NULL,
|
||||
"created_at" TIMESTAMPTZ(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
|
||||
CONSTRAINT "users_pkey" PRIMARY KEY ("id"),
|
||||
CONSTRAINT "users_username_format" CHECK ("username" ~ '^[a-z0-9_]{3,32}$'),
|
||||
CONSTRAINT "users_display_name_not_blank" CHECK (btrim("display_name") <> '')
|
||||
);
|
||||
|
||||
CREATE TABLE "chats" (
|
||||
"id" UUID NOT NULL,
|
||||
"title" VARCHAR(100) NOT NULL,
|
||||
"created_at" TIMESTAMPTZ(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
|
||||
CONSTRAINT "chats_pkey" PRIMARY KEY ("id"),
|
||||
CONSTRAINT "chats_title_not_blank" CHECK (btrim("title") <> '')
|
||||
);
|
||||
|
||||
CREATE TABLE "chat_members" (
|
||||
"chat_id" UUID NOT NULL,
|
||||
"user_id" UUID NOT NULL,
|
||||
|
||||
CONSTRAINT "chat_members_pkey" PRIMARY KEY ("chat_id","user_id")
|
||||
);
|
||||
|
||||
CREATE TABLE "messages" (
|
||||
"id" UUID NOT NULL,
|
||||
"chat_id" UUID NOT NULL,
|
||||
"sender_id" UUID NOT NULL,
|
||||
"text" VARCHAR(4000) NOT NULL,
|
||||
"created_at" TIMESTAMPTZ(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
|
||||
CONSTRAINT "messages_pkey" PRIMARY KEY ("id")
|
||||
);
|
||||
|
||||
CREATE UNIQUE INDEX "users_username_key" ON "users"("username");
|
||||
|
||||
CREATE INDEX "chat_members_user_id_idx" ON "chat_members"("user_id");
|
||||
|
||||
CREATE INDEX "messages_chat_id_created_at_id_idx" ON "messages"("chat_id", "created_at", "id");
|
||||
|
||||
ALTER TABLE "chat_members" ADD CONSTRAINT "chat_members_chat_id_fkey" FOREIGN KEY ("chat_id") REFERENCES "chats"("id") ON DELETE CASCADE ON UPDATE CASCADE;
|
||||
|
||||
ALTER TABLE "chat_members" ADD CONSTRAINT "chat_members_user_id_fkey" FOREIGN KEY ("user_id") REFERENCES "users"("id") ON DELETE RESTRICT ON UPDATE CASCADE;
|
||||
|
||||
ALTER TABLE "messages" ADD CONSTRAINT "messages_chat_id_fkey" FOREIGN KEY ("chat_id") REFERENCES "chats"("id") ON DELETE CASCADE ON UPDATE CASCADE;
|
||||
|
||||
ALTER TABLE "messages" ADD CONSTRAINT "messages_sender_id_fkey" FOREIGN KEY ("sender_id") REFERENCES "users"("id") ON DELETE RESTRICT ON UPDATE CASCADE;
|
||||
@@ -0,0 +1,14 @@
|
||||
ALTER TABLE "users" ADD COLUMN "password_hash" TEXT NOT NULL;
|
||||
|
||||
CREATE TABLE "sessions" (
|
||||
"token_hash" CHAR(64) NOT NULL,
|
||||
"user_id" UUID NOT NULL,
|
||||
"expires_at" TIMESTAMPTZ(3) NOT NULL,
|
||||
"created_at" TIMESTAMPTZ(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
|
||||
CONSTRAINT "sessions_pkey" PRIMARY KEY ("token_hash")
|
||||
);
|
||||
|
||||
CREATE INDEX "sessions_user_id_idx" ON "sessions"("user_id");
|
||||
|
||||
ALTER TABLE "sessions" ADD CONSTRAINT "sessions_user_id_fkey" FOREIGN KEY ("user_id") REFERENCES "users"("id") ON DELETE CASCADE ON UPDATE CASCADE;
|
||||
@@ -0,0 +1,13 @@
|
||||
CREATE TABLE "idempotency_keys" (
|
||||
"user_id" UUID NOT NULL,
|
||||
"chat_id" UUID NOT NULL,
|
||||
"key" VARCHAR(128) NOT NULL,
|
||||
"request_hash" CHAR(64) NOT NULL,
|
||||
"message_id" UUID NOT NULL,
|
||||
"created_at" TIMESTAMPTZ(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
|
||||
CONSTRAINT "idempotency_keys_pkey" PRIMARY KEY ("user_id","chat_id","key"),
|
||||
CONSTRAINT "idempotency_keys_key_format" CHECK ("key" ~ '^[A-Za-z0-9._:-]{1,128}$')
|
||||
);
|
||||
|
||||
ALTER TABLE "idempotency_keys" ADD CONSTRAINT "idempotency_keys_message_id_fkey" FOREIGN KEY ("message_id") REFERENCES "messages"("id") ON DELETE CASCADE ON UPDATE CASCADE;
|
||||
@@ -0,0 +1,45 @@
|
||||
CREATE TABLE "attachments" (
|
||||
"id" UUID NOT NULL,
|
||||
"chat_id" UUID NOT NULL,
|
||||
"owner_id" UUID NOT NULL,
|
||||
"filename" VARCHAR(255) NOT NULL,
|
||||
"content_type" VARCHAR(32) NOT NULL,
|
||||
"size_bytes" INTEGER NOT NULL,
|
||||
"sha256" CHAR(64) NOT NULL,
|
||||
"object_key" VARCHAR(128) NOT NULL,
|
||||
"created_at" TIMESTAMPTZ(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
"updated_at" TIMESTAMPTZ(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
|
||||
CONSTRAINT "attachments_pkey" PRIMARY KEY ("id"),
|
||||
CONSTRAINT "attachments_filename_not_blank" CHECK (btrim("filename") <> ''),
|
||||
CONSTRAINT "attachments_content_type_allowed" CHECK ("content_type" IN ('image/png', 'image/jpeg')),
|
||||
CONSTRAINT "attachments_size_range" CHECK ("size_bytes" BETWEEN 1 AND 10485760),
|
||||
CONSTRAINT "attachments_sha256_format" CHECK ("sha256" ~ '^[a-f0-9]{64}$')
|
||||
);
|
||||
|
||||
CREATE TABLE "message_attachments" (
|
||||
"message_id" UUID NOT NULL,
|
||||
"position" SMALLINT NOT NULL,
|
||||
"attachment_id" UUID NOT NULL,
|
||||
|
||||
CONSTRAINT "message_attachments_pkey" PRIMARY KEY ("message_id","position"),
|
||||
CONSTRAINT "message_attachments_position_range" CHECK ("position" BETWEEN 0 AND 9)
|
||||
);
|
||||
|
||||
CREATE UNIQUE INDEX "attachments_object_key_key" ON "attachments"("object_key");
|
||||
|
||||
CREATE INDEX "attachments_chat_id_idx" ON "attachments"("chat_id");
|
||||
|
||||
CREATE INDEX "attachments_owner_id_idx" ON "attachments"("owner_id");
|
||||
|
||||
CREATE UNIQUE INDEX "message_attachments_message_id_attachment_id_key" ON "message_attachments"("message_id", "attachment_id");
|
||||
|
||||
CREATE INDEX "message_attachments_attachment_id_idx" ON "message_attachments"("attachment_id");
|
||||
|
||||
ALTER TABLE "attachments" ADD CONSTRAINT "attachments_chat_id_fkey" FOREIGN KEY ("chat_id") REFERENCES "chats"("id") ON DELETE CASCADE ON UPDATE CASCADE;
|
||||
|
||||
ALTER TABLE "attachments" ADD CONSTRAINT "attachments_owner_id_fkey" FOREIGN KEY ("owner_id") REFERENCES "users"("id") ON DELETE RESTRICT ON UPDATE CASCADE;
|
||||
|
||||
ALTER TABLE "message_attachments" ADD CONSTRAINT "message_attachments_message_id_fkey" FOREIGN KEY ("message_id") REFERENCES "messages"("id") ON DELETE CASCADE ON UPDATE CASCADE;
|
||||
|
||||
ALTER TABLE "message_attachments" ADD CONSTRAINT "message_attachments_attachment_id_fkey" FOREIGN KEY ("attachment_id") REFERENCES "attachments"("id") ON DELETE NO ACTION ON UPDATE CASCADE;
|
||||
@@ -0,0 +1,97 @@
|
||||
ALTER TABLE "attachments"
|
||||
ADD COLUMN "staging_key" VARCHAR(128),
|
||||
ADD COLUMN "status" VARCHAR(16) NOT NULL DEFAULT 'ready',
|
||||
ADD COLUMN "error_code" VARCHAR(64);
|
||||
|
||||
ALTER TABLE "attachments" ALTER COLUMN "status" DROP DEFAULT;
|
||||
|
||||
ALTER TABLE "attachments"
|
||||
ADD CONSTRAINT "attachments_status_allowed" CHECK ("status" IN ('queued', 'processing', 'ready', 'failed')),
|
||||
ADD CONSTRAINT "attachments_error_code_when_failed" CHECK (("status" = 'failed') = ("error_code" IS NOT NULL));
|
||||
|
||||
CREATE UNIQUE INDEX "attachments_staging_key_key" ON "attachments"("staging_key");
|
||||
|
||||
CREATE TABLE "attachment_variants" (
|
||||
"attachment_id" UUID NOT NULL,
|
||||
"pipeline_version" INTEGER NOT NULL,
|
||||
"name" VARCHAR(16) NOT NULL,
|
||||
"content_type" VARCHAR(32) NOT NULL,
|
||||
"object_key" VARCHAR(128) NOT NULL,
|
||||
"size_bytes" INTEGER NOT NULL,
|
||||
"width" INTEGER NOT NULL,
|
||||
"height" INTEGER NOT NULL,
|
||||
"sha256" CHAR(64) NOT NULL,
|
||||
"created_at" TIMESTAMPTZ(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
|
||||
CONSTRAINT "attachment_variants_pkey" PRIMARY KEY ("attachment_id","pipeline_version","name"),
|
||||
CONSTRAINT "attachment_variants_name_allowed" CHECK ("name" IN ('thumbnail', 'crop', 'optimized')),
|
||||
CONSTRAINT "attachment_variants_content_type_allowed" CHECK ("content_type" IN ('image/jpeg', 'image/png', 'image/webp')),
|
||||
CONSTRAINT "attachment_variants_positive" CHECK ("pipeline_version" >= 1 AND "size_bytes" >= 1 AND "width" >= 1 AND "height" >= 1),
|
||||
CONSTRAINT "attachment_variants_sha256_format" CHECK ("sha256" ~ '^[a-f0-9]{64}$')
|
||||
);
|
||||
|
||||
CREATE TABLE "attachment_jobs" (
|
||||
"id" UUID NOT NULL,
|
||||
"attachment_id" UUID NOT NULL,
|
||||
"pipeline_version" INTEGER NOT NULL,
|
||||
"status" VARCHAR(16) NOT NULL,
|
||||
"attempt" INTEGER NOT NULL,
|
||||
"max_attempts" INTEGER NOT NULL,
|
||||
"next_attempt_at" TIMESTAMPTZ(3) NOT NULL,
|
||||
"lease_owner" UUID,
|
||||
"lease_until" TIMESTAMPTZ(3),
|
||||
"error_code" VARCHAR(64),
|
||||
"last_error" VARCHAR(200),
|
||||
"created_at" TIMESTAMPTZ(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
"updated_at" TIMESTAMPTZ(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
"finished_at" TIMESTAMPTZ(3),
|
||||
|
||||
CONSTRAINT "attachment_jobs_pkey" PRIMARY KEY ("id"),
|
||||
CONSTRAINT "attachment_jobs_status_allowed" CHECK ("status" IN ('queued', 'processing', 'ready', 'failed')),
|
||||
CONSTRAINT "attachment_jobs_attempt_range" CHECK ("pipeline_version" >= 1 AND "attempt" >= 1 AND "attempt" <= "max_attempts"),
|
||||
CONSTRAINT "attachment_jobs_lease_when_processing" CHECK ("status" <> 'processing' OR ("lease_owner" IS NOT NULL AND "lease_until" IS NOT NULL)),
|
||||
CONSTRAINT "attachment_jobs_error_code_when_failed" CHECK (("status" = 'failed') = ("error_code" IS NOT NULL))
|
||||
);
|
||||
|
||||
CREATE TABLE "job_events" (
|
||||
"id" BIGSERIAL NOT NULL,
|
||||
"job_id" UUID NOT NULL,
|
||||
"attempt" INTEGER NOT NULL,
|
||||
"event" VARCHAR(32) NOT NULL,
|
||||
"detail" VARCHAR(200),
|
||||
"created_at" TIMESTAMPTZ(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
|
||||
CONSTRAINT "job_events_pkey" PRIMARY KEY ("id")
|
||||
);
|
||||
|
||||
CREATE TABLE "outbox" (
|
||||
"id" BIGSERIAL NOT NULL,
|
||||
"job_id" UUID NOT NULL,
|
||||
"payload" JSONB NOT NULL,
|
||||
"available_at" TIMESTAMPTZ(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
"published_at" TIMESTAMPTZ(3),
|
||||
"publish_attempts" INTEGER NOT NULL DEFAULT 0,
|
||||
"created_at" TIMESTAMPTZ(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
|
||||
CONSTRAINT "outbox_pkey" PRIMARY KEY ("id")
|
||||
);
|
||||
|
||||
CREATE UNIQUE INDEX "attachment_variants_object_key_key" ON "attachment_variants"("object_key");
|
||||
|
||||
CREATE UNIQUE INDEX "attachment_jobs_attachment_id_pipeline_version_key" ON "attachment_jobs"("attachment_id", "pipeline_version");
|
||||
|
||||
CREATE INDEX "attachment_jobs_status_lease_until_idx" ON "attachment_jobs"("status", "lease_until");
|
||||
|
||||
CREATE INDEX "job_events_job_id_idx" ON "job_events"("job_id");
|
||||
|
||||
CREATE INDEX "outbox_published_at_available_at_idx" ON "outbox"("published_at", "available_at");
|
||||
|
||||
CREATE INDEX "outbox_job_id_idx" ON "outbox"("job_id");
|
||||
|
||||
ALTER TABLE "attachment_variants" ADD CONSTRAINT "attachment_variants_attachment_id_fkey" FOREIGN KEY ("attachment_id") REFERENCES "attachments"("id") ON DELETE CASCADE ON UPDATE CASCADE;
|
||||
|
||||
ALTER TABLE "attachment_jobs" ADD CONSTRAINT "attachment_jobs_attachment_id_fkey" FOREIGN KEY ("attachment_id") REFERENCES "attachments"("id") ON DELETE CASCADE ON UPDATE CASCADE;
|
||||
|
||||
ALTER TABLE "job_events" ADD CONSTRAINT "job_events_job_id_fkey" FOREIGN KEY ("job_id") REFERENCES "attachment_jobs"("id") ON DELETE CASCADE ON UPDATE CASCADE;
|
||||
|
||||
ALTER TABLE "outbox" ADD CONSTRAINT "outbox_job_id_fkey" FOREIGN KEY ("job_id") REFERENCES "attachment_jobs"("id") ON DELETE CASCADE ON UPDATE CASCADE;
|
||||
@@ -0,0 +1,3 @@
|
||||
# Please do not edit this file manually
|
||||
# It should be added in your version-control system (i.e. Git)
|
||||
provider = "postgresql"
|
||||
@@ -0,0 +1,191 @@
|
||||
generator client {
|
||||
provider = "prisma-client-js"
|
||||
previewFeatures = ["metrics"]
|
||||
}
|
||||
|
||||
datasource db {
|
||||
provider = "postgresql"
|
||||
url = env("DATABASE_URL")
|
||||
}
|
||||
|
||||
model User {
|
||||
id String @id @default(uuid()) @db.Uuid
|
||||
username String @unique @db.VarChar(32)
|
||||
displayName String @map("display_name") @db.VarChar(100)
|
||||
passwordHash String @map("password_hash")
|
||||
createdAt DateTime @default(now()) @map("created_at") @db.Timestamptz(3)
|
||||
memberships ChatMember[]
|
||||
messages Message[]
|
||||
sessions Session[]
|
||||
attachments Attachment[]
|
||||
|
||||
@@map("users")
|
||||
}
|
||||
|
||||
model Session {
|
||||
tokenHash String @id @map("token_hash") @db.Char(64)
|
||||
userId String @map("user_id") @db.Uuid
|
||||
expiresAt DateTime @map("expires_at") @db.Timestamptz(3)
|
||||
createdAt DateTime @default(now()) @map("created_at") @db.Timestamptz(3)
|
||||
user User @relation(fields: [userId], references: [id], onDelete: Cascade)
|
||||
|
||||
@@index([userId])
|
||||
@@map("sessions")
|
||||
}
|
||||
|
||||
model Chat {
|
||||
id String @id @default(uuid()) @db.Uuid
|
||||
title String @db.VarChar(100)
|
||||
createdAt DateTime @default(now()) @map("created_at") @db.Timestamptz(3)
|
||||
members ChatMember[]
|
||||
messages Message[]
|
||||
attachments Attachment[]
|
||||
|
||||
@@map("chats")
|
||||
}
|
||||
|
||||
model ChatMember {
|
||||
chatId String @map("chat_id") @db.Uuid
|
||||
userId String @map("user_id") @db.Uuid
|
||||
chat Chat @relation(fields: [chatId], references: [id], onDelete: Cascade)
|
||||
user User @relation(fields: [userId], references: [id], onDelete: Restrict)
|
||||
|
||||
@@id([chatId, userId])
|
||||
@@index([userId])
|
||||
@@map("chat_members")
|
||||
}
|
||||
|
||||
model Message {
|
||||
id String @id @default(uuid()) @db.Uuid
|
||||
chatId String @map("chat_id") @db.Uuid
|
||||
senderId String @map("sender_id") @db.Uuid
|
||||
text String @db.VarChar(4000)
|
||||
createdAt DateTime @default(now()) @map("created_at") @db.Timestamptz(3)
|
||||
chat Chat @relation(fields: [chatId], references: [id], onDelete: Cascade)
|
||||
sender User @relation(fields: [senderId], references: [id], onDelete: Restrict)
|
||||
idempotencyKeys IdempotencyKey[]
|
||||
attachments MessageAttachment[]
|
||||
|
||||
@@index([chatId, createdAt, id])
|
||||
@@map("messages")
|
||||
}
|
||||
|
||||
model Attachment {
|
||||
id String @id @default(uuid()) @db.Uuid
|
||||
chatId String @map("chat_id") @db.Uuid
|
||||
ownerId String @map("owner_id") @db.Uuid
|
||||
filename String @db.VarChar(255)
|
||||
contentType String @map("content_type") @db.VarChar(32)
|
||||
sizeBytes Int @map("size_bytes")
|
||||
sha256 String @db.Char(64)
|
||||
objectKey String @unique @map("object_key") @db.VarChar(128)
|
||||
stagingKey String? @unique @map("staging_key") @db.VarChar(128)
|
||||
status String @db.VarChar(16)
|
||||
errorCode String? @map("error_code") @db.VarChar(64)
|
||||
createdAt DateTime @default(now()) @map("created_at") @db.Timestamptz(3)
|
||||
updatedAt DateTime @default(now()) @updatedAt @map("updated_at") @db.Timestamptz(3)
|
||||
chat Chat @relation(fields: [chatId], references: [id], onDelete: Cascade)
|
||||
owner User @relation(fields: [ownerId], references: [id], onDelete: Restrict)
|
||||
messages MessageAttachment[]
|
||||
variants AttachmentVariant[]
|
||||
jobs AttachmentJob[]
|
||||
|
||||
@@index([chatId])
|
||||
@@index([ownerId])
|
||||
@@map("attachments")
|
||||
}
|
||||
|
||||
model AttachmentVariant {
|
||||
attachmentId String @map("attachment_id") @db.Uuid
|
||||
pipelineVersion Int @map("pipeline_version")
|
||||
name String @db.VarChar(16)
|
||||
contentType String @map("content_type") @db.VarChar(32)
|
||||
objectKey String @unique @map("object_key") @db.VarChar(128)
|
||||
sizeBytes Int @map("size_bytes")
|
||||
width Int
|
||||
height Int
|
||||
sha256 String @db.Char(64)
|
||||
createdAt DateTime @default(now()) @map("created_at") @db.Timestamptz(3)
|
||||
attachment Attachment @relation(fields: [attachmentId], references: [id], onDelete: Cascade)
|
||||
|
||||
@@id([attachmentId, pipelineVersion, name])
|
||||
@@map("attachment_variants")
|
||||
}
|
||||
|
||||
model AttachmentJob {
|
||||
id String @id @db.Uuid
|
||||
attachmentId String @map("attachment_id") @db.Uuid
|
||||
pipelineVersion Int @map("pipeline_version")
|
||||
status String @db.VarChar(16)
|
||||
attempt Int
|
||||
maxAttempts Int @map("max_attempts")
|
||||
nextAttemptAt DateTime @map("next_attempt_at") @db.Timestamptz(3)
|
||||
leaseOwner String? @map("lease_owner") @db.Uuid
|
||||
leaseUntil DateTime? @map("lease_until") @db.Timestamptz(3)
|
||||
errorCode String? @map("error_code") @db.VarChar(64)
|
||||
lastError String? @map("last_error") @db.VarChar(200)
|
||||
createdAt DateTime @default(now()) @map("created_at") @db.Timestamptz(3)
|
||||
updatedAt DateTime @default(now()) @updatedAt @map("updated_at") @db.Timestamptz(3)
|
||||
finishedAt DateTime? @map("finished_at") @db.Timestamptz(3)
|
||||
attachment Attachment @relation(fields: [attachmentId], references: [id], onDelete: Cascade)
|
||||
events JobEvent[]
|
||||
outbox OutboxEntry[]
|
||||
|
||||
@@unique([attachmentId, pipelineVersion])
|
||||
@@index([status, leaseUntil])
|
||||
@@map("attachment_jobs")
|
||||
}
|
||||
|
||||
model JobEvent {
|
||||
id BigInt @id @default(autoincrement())
|
||||
jobId String @map("job_id") @db.Uuid
|
||||
attempt Int
|
||||
event String @db.VarChar(32)
|
||||
detail String? @db.VarChar(200)
|
||||
createdAt DateTime @default(now()) @map("created_at") @db.Timestamptz(3)
|
||||
job AttachmentJob @relation(fields: [jobId], references: [id], onDelete: Cascade)
|
||||
|
||||
@@index([jobId])
|
||||
@@map("job_events")
|
||||
}
|
||||
|
||||
model OutboxEntry {
|
||||
id BigInt @id @default(autoincrement())
|
||||
jobId String @map("job_id") @db.Uuid
|
||||
payload Json
|
||||
availableAt DateTime @default(now()) @map("available_at") @db.Timestamptz(3)
|
||||
publishedAt DateTime? @map("published_at") @db.Timestamptz(3)
|
||||
publishAttempts Int @default(0) @map("publish_attempts")
|
||||
createdAt DateTime @default(now()) @map("created_at") @db.Timestamptz(3)
|
||||
job AttachmentJob @relation(fields: [jobId], references: [id], onDelete: Cascade)
|
||||
|
||||
@@index([publishedAt, availableAt])
|
||||
@@index([jobId])
|
||||
@@map("outbox")
|
||||
}
|
||||
|
||||
model MessageAttachment {
|
||||
messageId String @map("message_id") @db.Uuid
|
||||
position Int @db.SmallInt
|
||||
attachmentId String @map("attachment_id") @db.Uuid
|
||||
message Message @relation(fields: [messageId], references: [id], onDelete: Cascade)
|
||||
attachment Attachment @relation(fields: [attachmentId], references: [id], onDelete: NoAction)
|
||||
|
||||
@@id([messageId, position])
|
||||
@@unique([messageId, attachmentId])
|
||||
@@index([attachmentId])
|
||||
@@map("message_attachments")
|
||||
}
|
||||
|
||||
model IdempotencyKey {
|
||||
userId String @map("user_id") @db.Uuid
|
||||
chatId String @map("chat_id") @db.Uuid
|
||||
key String @db.VarChar(128)
|
||||
requestHash String @map("request_hash") @db.Char(64)
|
||||
messageId String @map("message_id") @db.Uuid
|
||||
createdAt DateTime @default(now()) @map("created_at") @db.Timestamptz(3)
|
||||
message Message @relation(fields: [messageId], references: [id], onDelete: Cascade)
|
||||
|
||||
@@id([userId, chatId, key])
|
||||
@@map("idempotency_keys")
|
||||
}
|
||||
Executable
+21
@@ -0,0 +1,21 @@
|
||||
#!/usr/bin/env python3
|
||||
import sys
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
|
||||
bucket, key = sys.argv[1:3]
|
||||
base = 'http://s3:9000/' + bucket
|
||||
attempts = (
|
||||
('GET', '/<object>', base + '/' + key, None),
|
||||
('GET', '/', base + '/', None),
|
||||
('PUT', '/anon.txt', base + '/anon.txt', b'x'),
|
||||
)
|
||||
failed = False
|
||||
for method, label, url, data in attempts:
|
||||
try:
|
||||
code = urllib.request.urlopen(urllib.request.Request(url, data=data, method=method), timeout=5).status
|
||||
except urllib.error.HTTPError as error:
|
||||
code = error.code
|
||||
print(method, bucket + label, '->', code)
|
||||
failed = failed or code != 403
|
||||
sys.exit(1 if failed else 0)
|
||||
Executable
+46
@@ -0,0 +1,46 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
bucket="${S3_BUCKET:-messenger-attachments}"
|
||||
object="check/$(date +%s)-$$.txt"
|
||||
|
||||
as_api() {
|
||||
docker compose run --rm --no-deps -T -e BUCKET="$bucket" -e OBJECT="$object" --entrypoint sh s3-init -c "$1"
|
||||
}
|
||||
|
||||
echo "## S3 не опубликован наружу, консоль выключена"
|
||||
docker compose ps s3 --format '{{.Service}} ports={{.Ports}}'
|
||||
echo "MINIO_BROWSER=$(docker compose exec -T s3 printenv MINIO_BROWSER)"
|
||||
|
||||
echo
|
||||
echo "## Права пользователя API: только свой bucket"
|
||||
as_api '
|
||||
set -e
|
||||
cd /run/secrets
|
||||
export MC_HOST_api="http://$(cat s3_access_key_id):$(cat s3_secret_access_key)@s3:9000"
|
||||
export MC_HOST_root="http://$(cat s3_root_user):$(cat s3_root_password)@s3:9000"
|
||||
expect() { want=$1; label=$2; shift 2; if "$@" >/dev/null 2>&1; then got=allowed; else got=denied; fi; echo "$label -> $got"; [ "$got" = "$want" ]; }
|
||||
echo probe > /tmp/probe.txt
|
||||
mc mb --ignore-existing root/check-other-bucket >/dev/null
|
||||
mc cp /tmp/probe.txt root/check-other-bucket/probe.txt >/dev/null
|
||||
trap "mc rb --force root/check-other-bucket >/dev/null" EXIT
|
||||
expect allowed "PUT в свой bucket" mc cp /tmp/probe.txt "api/$BUCKET/$OBJECT"
|
||||
expect allowed "GET из своего bucket" mc cat "api/$BUCKET/$OBJECT"
|
||||
expect denied "создание другого bucket" mc mb api/forbidden-bucket
|
||||
expect denied "GET из другого bucket" mc cat api/check-other-bucket/probe.txt
|
||||
expect denied "PUT в другой bucket" mc cp /tmp/probe.txt api/check-other-bucket/new.txt
|
||||
expect denied "изменение политики bucket" mc anonymous set public "api/$BUCKET"
|
||||
expect denied "администрирование пользователей" mc admin user list api
|
||||
'
|
||||
|
||||
echo
|
||||
echo "## Анонимный доступ напрямую к S3, мимо API"
|
||||
python3 scripts/check-s3-anonymous.py "$bucket" "$object"
|
||||
|
||||
as_api '
|
||||
cd /run/secrets
|
||||
export MC_HOST_api="http://$(cat s3_access_key_id):$(cat s3_secret_access_key)@s3:9000"
|
||||
mc rm "api/$BUCKET/$OBJECT" >/dev/null
|
||||
'
|
||||
echo
|
||||
echo "PASS: bucket приватный, у API нет прав вне своего bucket"
|
||||
Executable
+161
@@ -0,0 +1,161 @@
|
||||
#!/bin/sh
|
||||
set -u
|
||||
cd "$(dirname "$0")/.."
|
||||
mkdir -p evidence
|
||||
|
||||
COMMIT="$(git rev-parse --short HEAD)$(git diff --quiet HEAD -- src prisma test scripts infra compose.yaml Dockerfile || echo ' (dirty)')"
|
||||
|
||||
header() {
|
||||
printf '# %s\n# date: %s\n# commit: %s\n\n' "$1" "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "$COMMIT"
|
||||
}
|
||||
|
||||
run() {
|
||||
echo "\$ $*"
|
||||
"$@" 2>&1 || echo "EXIT CODE $?"
|
||||
echo
|
||||
}
|
||||
|
||||
started() {
|
||||
docker inspect -f '{{.Name}} {{.State.Status}} started={{.State.StartedAt}}' $(docker compose ps -aq postgres redis s3 rabbitmq api lb)
|
||||
}
|
||||
|
||||
api_states() {
|
||||
docker inspect -f '{{.Name}} id={{slice .Id 0 12}} {{.State.Status}} exit={{.State.ExitCode}} finished={{.State.FinishedAt}}' $(docker compose ps -aq api)
|
||||
}
|
||||
|
||||
restore_api() {
|
||||
for id in $(docker compose ps -aq --status exited api); do
|
||||
docker start "$id" >/dev/null
|
||||
done
|
||||
python3 scripts/probe.py ready >/dev/null
|
||||
sleep 3
|
||||
}
|
||||
|
||||
echo 'container.txt'
|
||||
{
|
||||
header 'Стенд: сервисы, порты, лимиты, секреты, миграции, топология брокера'
|
||||
run docker compose ps -a --format 'table {{.Service}}\t{{.Name}}\t{{.Image}}\t{{.Status}}\t{{.Ports}}'
|
||||
echo '## CPU/RAM-лимиты (NanoCpus: 1e9 = 1 CPU; Memory в байтах)'
|
||||
run sh -c "docker inspect -f '{{.Name}} cpus={{.HostConfig.NanoCpus}} memory={{.HostConfig.Memory}}' \$(docker compose ps -aq)"
|
||||
echo '## Опубликованные порты: только LB и Grafana на 127.0.0.1; OTLP, Prometheus, Loki, Tempo наружу не открыты'
|
||||
run sh -c "docker compose ps --format '{{.Service}} {{.Ports}}' | grep -v '^\\S* *$' || true"
|
||||
echo '## Сети телеметрии: хранилища только во внутренней сети telemetry'
|
||||
run sh -c "for s in otel-collector prometheus loki tempo grafana api worker; do printf '%s: ' \$s; docker inspect -f '{{range \$k, \$v := .NetworkSettings.Networks}}{{\$k}} {{end}}' \$(docker compose ps -q \$s | head -1); done"
|
||||
run docker network inspect -f '{{.Name}} internal={{.Internal}}' messenger-lab6_telemetry
|
||||
echo '## Секреты: каждый сервис видит только свои файлы (содержимое не выводится)'
|
||||
for service in postgres redis s3 api reconciler relay worker grafana; do
|
||||
run docker compose exec -T "$service" ls /run/secrets
|
||||
done
|
||||
run docker compose exec -T rabbitmq ls /etc/rabbitmq/conf.d
|
||||
run docker compose exec -T lb sh -c 'ls /run/secrets 2>/dev/null || echo "no /run/secrets"'
|
||||
echo '## Версии'
|
||||
run docker compose exec -T lb haproxy -v
|
||||
run docker compose exec -T postgres postgres --version
|
||||
run docker compose exec -T redis redis-server --version
|
||||
run docker compose exec -T s3 minio --version
|
||||
run docker compose exec -T rabbitmq rabbitmqctl -q version
|
||||
run sh -c "docker compose images otel-collector prometheus loki tempo grafana"
|
||||
echo '## Топология брокера: durable quorum-очереди, delivery-limit и DLQ'
|
||||
run docker compose exec -T rabbitmq rabbitmqctl -q list_queues name type durable arguments messages
|
||||
run docker compose exec -T rabbitmq rabbitmqctl -q list_consumers queue_name prefetch_count ack_required
|
||||
echo '## Таблицы'
|
||||
run docker compose exec -T postgres psql -U messenger -d messenger -c '\d attachments' -c '\d attachment_variants' -c '\d attachment_jobs' -c '\d outbox' -c '\d job_events'
|
||||
echo '## Миграции: отдельный one-shot сервис, повторный запуск ничего не применяет, схема совпадает с БД'
|
||||
run docker compose run --rm migrate
|
||||
run docker compose run --rm --no-deps migrate sh -c 'export DATABASE_URL="$(cat "$DATABASE_URL_FILE")"; npx prisma migrate diff --from-url "$DATABASE_URL" --to-schema-datamodel prisma/schema.prisma --exit-code'
|
||||
run docker compose exec -T postgres psql -U messenger -d messenger -c 'SELECT migration_name, finished_at FROM _prisma_migrations ORDER BY started_at'
|
||||
} > evidence/container.txt
|
||||
|
||||
echo 'tests.txt'
|
||||
{
|
||||
header 'Собственные e2e-тесты (PostgreSQL + Redis + MinIO из compose.test.yaml); основной стенд на время тестов остановлен ради памяти Docker'
|
||||
docker compose stop lb api worker relay reconciler otel-collector prometheus loki tempo grafana rabbitmq s3 redis postgres >/dev/null 2>&1
|
||||
docker compose -f compose.test.yaml build -q tests
|
||||
run docker compose -f compose.test.yaml run --rm tests npx jest --config ./test/jest-e2e.json --verbose
|
||||
docker compose -f compose.test.yaml down -v >/dev/null 2>&1
|
||||
docker compose start postgres redis s3 rabbitmq api worker relay reconciler lb otel-collector prometheus loki tempo grafana >/dev/null 2>&1
|
||||
python3 scripts/probe.py ready >/dev/null
|
||||
sleep 20
|
||||
} > evidence/tests.txt
|
||||
|
||||
echo 'smoke.txt'
|
||||
{
|
||||
header 'Публичные проверки через LB; разные реплики обслуживают одну сессию'
|
||||
run make check
|
||||
run make test
|
||||
run python3 scripts/probe.py instances 6
|
||||
} > evidence/smoke.txt
|
||||
|
||||
echo 'persistence.txt'
|
||||
{
|
||||
header 'make test-persistence: restart.sh перезапускает postgres, redis, s3, rabbitmq, api, lb с сохранением volumes'
|
||||
echo '## до'
|
||||
started
|
||||
echo
|
||||
run make test-persistence ACTION_SCRIPT=scripts/restart.sh
|
||||
echo '## после'
|
||||
started
|
||||
} > evidence/persistence.txt
|
||||
|
||||
echo 'failover.txt'
|
||||
{
|
||||
header 'make test-failover: stop-one.sh останавливает реплику, обслужившую тест'
|
||||
echo '## до'
|
||||
api_states
|
||||
echo
|
||||
run make test-failover ACTION_SCRIPT=scripts/stop-one.sh
|
||||
echo '## после: остановленная реплика Exited (0) — штатное завершение по SIGTERM'
|
||||
api_states
|
||||
restore_api
|
||||
} > evidence/failover.txt
|
||||
|
||||
echo 's3.txt'
|
||||
{
|
||||
header 'S3 (минимум lab4): идемпотентный bootstrap, приватность мимо API, права пользователя API'
|
||||
run docker compose run --rm --no-deps -T s3-init
|
||||
run docker compose run --rm --no-deps -T s3-init
|
||||
run sh scripts/check-s3.sh
|
||||
} > evidence/s3.txt
|
||||
|
||||
echo 'jobs.txt'
|
||||
{
|
||||
header 'Фоновая обработка: сценарии защиты уровней 3 и 4 (scripts/job-scenario.py)'
|
||||
for scenario in restart crash-after-store crash-after-commit outbox-gap relay-crash retry-replay broker; do
|
||||
echo "## $scenario"
|
||||
run python3 scripts/job-scenario.py "$scenario"
|
||||
done
|
||||
echo '## Итог по всем заданиям стенда'
|
||||
run docker compose exec -T postgres psql -U messenger -d messenger -c 'SELECT status, count(*) FROM attachment_jobs GROUP BY status' -c 'SELECT count(*) AS unpublished_outbox FROM outbox WHERE published_at IS NULL' -c 'SELECT count(*) AS staging_left_on_ready FROM attachments WHERE status = '"'ready'"' AND staging_key IS NOT NULL'
|
||||
echo '## Последний проход reconciler'
|
||||
docker compose logs --no-log-prefix --tail 1 reconciler 2>&1
|
||||
} > evidence/jobs.txt
|
||||
|
||||
echo 'telemetry.txt'
|
||||
{
|
||||
header 'Наблюдаемость: контрольный запрос (trace + логи), dashboard, перцентили, приватность, правила'
|
||||
echo '## Prometheus: правила загружены (promtool и /api/v1/rules)'
|
||||
run docker compose exec -T prometheus promtool check rules /etc/prometheus/rules/messenger.yml
|
||||
run sh -c "wget -qO- http://grafana:3000/api/datasources/proxy/uid/prometheus/api/v1/rules | python3 -c 'import json,sys; [print(r[\"type\"], r[\"name\"], r[\"health\"]) for g in json.load(sys.stdin)[\"data\"][\"groups\"] for r in g[\"rules\"]]'"
|
||||
echo '## Grafana: datasources с фиксированными UID, dashboard из provisioning'
|
||||
run sh -c "for uid in prometheus loki tempo; do wget -qO- http://grafana:3000/api/datasources/uid/\$uid | python3 -c 'import json,sys; d=json.load(sys.stdin); print(d[\"uid\"], d[\"type\"], d[\"url\"])'; done"
|
||||
echo '## Контрольный upload с клиентским traceparent: trace в Tempo и логи в Loki'
|
||||
run python3 scripts/trace-check.py
|
||||
echo '## Секреты и содержимое не попадают в telemetry; в labels метрик нет ID'
|
||||
run python3 scripts/telemetry-privacy.py
|
||||
echo '## Перцентили dashboard против клиентского замера tests/load.py'
|
||||
run python3 scripts/latency-compare.py --duration 30
|
||||
echo '## Все запросы dashboard (пустые панели alerts/WARN-логов допустимы без инцидентов)'
|
||||
run python3 scripts/dashboard-check.py
|
||||
echo '## Пример структурного лога (stdout API)'
|
||||
docker compose logs --no-log-prefix --tail 200 api 2>&1 | grep '"event":"http.request"' | tail -2
|
||||
} > evidence/telemetry.txt
|
||||
|
||||
for scenario in retry-trace worker-stop db telemetry; do
|
||||
echo "incident-$scenario.txt"
|
||||
{
|
||||
header "Инцидент: scripts/incidents.py $scenario"
|
||||
run python3 scripts/incidents.py "$scenario"
|
||||
} > "evidence/incident-$scenario.txt"
|
||||
done
|
||||
|
||||
echo 'done'
|
||||
Executable
+57
@@ -0,0 +1,57 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Runs every query of the provisioned dashboard through Grafana datasource proxies and reports empty panels.
|
||||
|
||||
Run inside the tools container: docker compose run --rm tools python3 scripts/dashboard-check.py
|
||||
"""
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
|
||||
GRAFANA = os.environ.get('GRAFANA_URL', 'http://grafana:3000')
|
||||
DASHBOARD = os.path.join(os.path.dirname(__file__), '..', 'infra', 'grafana', 'dashboards', 'messenger.json')
|
||||
|
||||
|
||||
def get(path, **params):
|
||||
url = GRAFANA + path + ('?' + urllib.parse.urlencode(params) if params else '')
|
||||
with urllib.request.urlopen(url, timeout=15) as response:
|
||||
return json.loads(response.read())
|
||||
|
||||
|
||||
def run(target):
|
||||
uid = target['datasource']['uid']
|
||||
now = int(time.time())
|
||||
if uid == 'prometheus':
|
||||
expr = target['expr'].replace('$__rate_interval', '1m')
|
||||
body = get('/api/datasources/proxy/uid/prometheus/api/v1/query_range', query=expr, start=now - 1800, end=now, step=30)
|
||||
series = body['data']['result']
|
||||
last = [s['values'][-1][1] for s in series if s['values']]
|
||||
return len(series), ', '.join('%s=%s' % (s['metric'].get('outcome') or s['metric'].get('http_response_status_class') or '', v)
|
||||
for s, v in zip(series[:4], last[:4]))
|
||||
if uid == 'loki':
|
||||
body = get('/api/datasources/proxy/uid/loki/loki/api/v1/query_range', query=target['expr'], limit=20, since='30m')
|
||||
return sum(len(s['values']) for s in body['data']['result']), ''
|
||||
if uid == 'tempo':
|
||||
body = get('/api/datasources/proxy/uid/tempo/api/search', q=target['query'], limit=target.get('limit', 20), start=now - 1800, end=now)
|
||||
return len(body.get('traces', [])), ''
|
||||
return 0, 'unknown datasource'
|
||||
|
||||
|
||||
def main():
|
||||
grafana = get('/api/dashboards/uid/messenger-overview')
|
||||
print('Grafana dashboard:', grafana['dashboard']['title'], 'folder:', grafana['meta'].get('folderTitle'))
|
||||
dashboard = json.load(open(DASHBOARD, encoding='utf-8'))
|
||||
empty = 0
|
||||
for panel in dashboard['panels']:
|
||||
for target in panel.get('targets', []):
|
||||
count, sample = run(target)
|
||||
empty += count == 0
|
||||
print('%-4s %-62s %-3s %4d %s' % ('OK' if count else 'EMPTY', panel['title'][:62], target['refId'], count, sample[:70]))
|
||||
print('\n%s: %d empty queries' % ('PASS' if empty == 0 else 'WARN', empty))
|
||||
sys.exit(0 if empty == 0 else 1)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
Executable
+389
@@ -0,0 +1,389 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Controlled incidents for lab6 observability; run inside the tools container.
|
||||
|
||||
docker compose run --rm tools python3 scripts/incidents.py worker-stop|db|telemetry|retry-trace
|
||||
|
||||
Every scenario injects a fault, keeps synthetic traffic going, samples Prometheus/Tempo/Loki through Grafana
|
||||
and waits for the related alert to become firing and then resolved after recovery.
|
||||
"""
|
||||
import base64
|
||||
import importlib.util
|
||||
import json
|
||||
import os
|
||||
import statistics
|
||||
import struct
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
import uuid
|
||||
import zlib
|
||||
|
||||
spec = importlib.util.spec_from_file_location('job_scenario', os.path.join(os.path.dirname(__file__), 'job-scenario.py'))
|
||||
js = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(js)
|
||||
log = js.log
|
||||
|
||||
GRAFANA = os.environ.get('GRAFANA_URL', 'http://grafana:3000')
|
||||
PROM = GRAFANA + '/api/datasources/proxy/uid/prometheus/api/v1/'
|
||||
TEMPO = GRAFANA + '/api/datasources/proxy/uid/tempo/api/'
|
||||
LOKI = GRAFANA + '/api/datasources/proxy/uid/loki/loki/api/v1/'
|
||||
|
||||
|
||||
def get(url, **params):
|
||||
with urllib.request.urlopen(url + ('?' + urllib.parse.urlencode(params) if params else ''), timeout=20) as r:
|
||||
return json.loads(r.read())
|
||||
|
||||
|
||||
def prom(expr):
|
||||
try:
|
||||
result = get(PROM + 'query', query=expr)['data']['result']
|
||||
except OSError:
|
||||
return None
|
||||
return float(result[0]['value'][1]) if result else None
|
||||
|
||||
|
||||
def alert_state(name):
|
||||
try:
|
||||
alerts = get(PROM + 'alerts')['data']['alerts']
|
||||
except OSError:
|
||||
return 'unknown'
|
||||
states = [a['state'] for a in alerts if a['labels'].get('alertname') == name]
|
||||
return 'firing' if 'firing' in states else ('pending' if 'pending' in states else 'inactive')
|
||||
|
||||
|
||||
def fmt(value, unit=''):
|
||||
if value is None:
|
||||
return '-'
|
||||
if unit == '%':
|
||||
return '%.1f%%' % (value * 100)
|
||||
if unit == 'ms':
|
||||
return '%.0fms' % (value * 1000)
|
||||
if unit == 'MiB':
|
||||
return '%.0fMiB' % (value / 2 ** 20)
|
||||
return ('%.1f' % value).rstrip('0').rstrip('.')
|
||||
|
||||
|
||||
def sample(columns, alerts):
|
||||
parts = ['%s=%s' % (label, fmt(prom(expr), unit)) for label, expr, unit in columns]
|
||||
parts += ['%s=%s' % (name, alert_state(name)) for name in alerts]
|
||||
log(' '.join(parts))
|
||||
|
||||
|
||||
def watch(columns, alerts, until, timeout, every=10):
|
||||
deadline = time.monotonic() + timeout
|
||||
while time.monotonic() < deadline:
|
||||
sample(columns, alerts)
|
||||
if until():
|
||||
return True
|
||||
time.sleep(every)
|
||||
sample(columns, alerts)
|
||||
return False
|
||||
|
||||
|
||||
class Traffic:
|
||||
"""Closed-loop synthetic traffic through the LB: POST message + GET history, counts statuses per phase."""
|
||||
|
||||
def __init__(self, auth, chat, threads=4, pause=0.2):
|
||||
self.auth, self.chat, self.threads, self.pause = auth, chat, threads, pause
|
||||
self.stop = threading.Event()
|
||||
self.lock = threading.Lock()
|
||||
self.reset()
|
||||
|
||||
def reset(self):
|
||||
with self.lock:
|
||||
self.codes, self.latencies = {}, []
|
||||
|
||||
def snapshot(self, phase):
|
||||
with self.lock:
|
||||
codes, latencies = dict(self.codes), sorted(self.latencies)
|
||||
p = lambda q: latencies[min(len(latencies) - 1, int(q * len(latencies)))] * 1000 if latencies else 0
|
||||
log('client [%s]: %s; p50=%.0fms p95=%.0fms p99=%.0fms' % (
|
||||
phase, ' '.join('%s:%d' % kv for kv in sorted(codes.items())), p(.5), p(.95), p(.99)))
|
||||
self.reset()
|
||||
|
||||
def run(self):
|
||||
def loop(n):
|
||||
i = 0
|
||||
while not self.stop.is_set():
|
||||
started = time.monotonic()
|
||||
if i % 2 == 0:
|
||||
code = js.call('POST', '/api/v1/chats/%s/messages' % self.chat, {'text': 'incident %d-%d' % (n, i)},
|
||||
{**self.auth, 'Idempotency-Key': uuid.uuid4().hex})[0]
|
||||
else:
|
||||
code = js.call('GET', '/api/v1/chats/%s/messages?limit=20' % self.chat, headers=self.auth)[0]
|
||||
with self.lock:
|
||||
self.codes[code] = self.codes.get(code, 0) + 1
|
||||
if 200 <= code < 300:
|
||||
self.latencies.append(time.monotonic() - started)
|
||||
i += 1
|
||||
time.sleep(self.pause)
|
||||
self.workers = [threading.Thread(target=loop, args=(n,), daemon=True) for n in range(self.threads)]
|
||||
for worker in self.workers:
|
||||
worker.start()
|
||||
return self
|
||||
|
||||
def close(self):
|
||||
self.stop.set()
|
||||
for worker in self.workers:
|
||||
worker.join(20)
|
||||
|
||||
|
||||
def b64hex(value):
|
||||
return base64.b64decode(value).hex() if value else ''
|
||||
|
||||
|
||||
def trace_spans(trace_id):
|
||||
body = get(TEMPO + 'traces/' + trace_id)
|
||||
spans = []
|
||||
for batch in body.get('batches', []):
|
||||
service = {a['key']: a['value'] for a in batch['resource']['attributes']}['service.name']['stringValue']
|
||||
for scope in batch.get('scopeSpans', []):
|
||||
for span in scope.get('spans', []):
|
||||
span['service'] = service
|
||||
span['ms'] = (int(span['endTimeUnixNano']) - int(span['startTimeUnixNano'])) / 1e6
|
||||
span['attrs'] = {a['key']: list(a['value'].values())[0] for a in span.get('attributes', [])}
|
||||
spans.append(span)
|
||||
return spans
|
||||
|
||||
|
||||
def find_traces(query, since, limit=5):
|
||||
try:
|
||||
return get(TEMPO + 'search', q=query, limit=limit, start=int(since), end=int(time.time()) + 60).get('traces', [])
|
||||
except OSError:
|
||||
return []
|
||||
|
||||
|
||||
def breakdown(trace_id, title):
|
||||
spans = trace_spans(trace_id)
|
||||
server = next((s for s in spans if s.get('kind') == 'SPAN_KIND_SERVER'), None)
|
||||
if not server:
|
||||
log('trace %s: no server span' % trace_id)
|
||||
return
|
||||
|
||||
names = {s['spanId']: s['name'] for s in spans}
|
||||
|
||||
def total(prefix, top_level=False):
|
||||
return sum(s['ms'] for s in spans if s['name'].startswith(prefix)
|
||||
and not (top_level and names.get(s.get('parentSpanId'), '').startswith('prisma:')))
|
||||
db, s3 = total('prisma:', top_level=True), total('S3.')
|
||||
log('%s: trace %s %s -> %s, %.0f ms' % (title, trace_id, server['name'], server['attrs'].get('http.status_code'), server['ms']))
|
||||
log(' DB calls %.0f ms (pool wait %.0f ms, SQL %.0f ms); S3 %.0f ms; handler itself %.0f ms' % (
|
||||
db, total('prisma:engine:connection'), total('prisma:engine:db_query'), s3, max(0, server['ms'] - db - s3)))
|
||||
errors = sorted({s['name'] for s in spans if s.get('status', {}).get('code') == 'STATUS_CODE_ERROR'})
|
||||
if errors:
|
||||
log(' spans with error status:', ', '.join(errors))
|
||||
|
||||
|
||||
def noise_png(width, height):
|
||||
raw = b''.join(b'\0' + os.urandom(width * 3) for _ in range(height))
|
||||
|
||||
def chunk(kind, data):
|
||||
return struct.pack('>I', len(data)) + kind + data + struct.pack('>I', zlib.crc32(kind + data))
|
||||
header = struct.pack('>IIBBBBB', width, height, 8, 2, 0, 0, 0)
|
||||
return b'\x89PNG\r\n\x1a\n' + chunk(b'IHDR', header) + chunk(b'IDAT', zlib.compress(raw, 1)) + chunk(b'IEND', b'')
|
||||
|
||||
|
||||
def docker(*args):
|
||||
log('$ docker', ' '.join(args))
|
||||
subprocess.run(['docker', *args], check=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
|
||||
|
||||
|
||||
def container(service):
|
||||
return subprocess.run(['docker', 'compose', 'ps', '-q', service], check=True, capture_output=True, text=True).stdout.split()
|
||||
|
||||
|
||||
def scenario_worker_stop():
|
||||
"""Incident 1: worker stopped -> jobs age, MessengerJobQueueLag fires -> worker started -> backlog drains, resolved."""
|
||||
js.wait_ready()
|
||||
auth, chat = js.session()
|
||||
columns = [
|
||||
('due', 'max(jobs_queue_depth{state="due"})', ''),
|
||||
('processing', 'max(jobs_queue_depth{state="processing"})', ''),
|
||||
('oldest_age_s', 'messenger:jobs_queue_oldest_age:max', ''),
|
||||
('worker_busy', 'sum(worker_jobs_busy)', ''),
|
||||
('worker_up', 'count(timestamp(worker_jobs_capacity) > time() - 30)', ''),
|
||||
('ready_per_s', 'sum(rate(jobs_processed_total{outcome="success"}[1m]))', ''),
|
||||
]
|
||||
alerts = ['MessengerJobQueueLag']
|
||||
js.compose('stop', 'worker')
|
||||
stopped = time.time()
|
||||
uploads = []
|
||||
|
||||
def upload_more():
|
||||
if len(uploads) < 12 and time.time() - stopped < 150:
|
||||
uploads.append(js.upload(auth, chat))
|
||||
return alert_state('MessengerJobQueueLag') == 'firing'
|
||||
fired = watch(columns, alerts, upload_more, timeout=300)
|
||||
log('alert MessengerJobQueueLag firing after %.0f s of worker downtime: %s' % (time.time() - stopped, fired))
|
||||
js.start_only('worker')
|
||||
started = time.time()
|
||||
resolved = watch(columns, alerts, lambda: alert_state('MessengerJobQueueLag') == 'inactive', timeout=240)
|
||||
log('backlog drained and alert resolved %.0f s after worker start: %s' % (time.time() - started, resolved))
|
||||
ready = sum(1 for a in uploads if js.metadata(auth, a)[1]['status'] == 'ready')
|
||||
log('uploads during downtime: %d, ready now: %d' % (len(uploads), ready))
|
||||
time.sleep(15)
|
||||
log('time to ready, histogram over 5m: p50=%s s, p99=%s s' % tuple(fmt(prom(
|
||||
'histogram_quantile(%s, sum by (le) (increase(attachments_time_to_ready_seconds_bucket[5m])))' % q)) for q in (0.5, 0.99)))
|
||||
log('time to ready from the database (finished_at - created_at), s:', js.psql(
|
||||
"SELECT round(min(extract(epoch FROM j.finished_at - a.created_at))::numeric, 1) || ' .. ' || "
|
||||
"round(max(extract(epoch FROM j.finished_at - a.created_at))::numeric, 1) FROM attachments a "
|
||||
"JOIN attachment_jobs j ON j.attachment_id = a.id WHERE a.id IN (%s)" % ','.join("'%s'" % a for a in uploads)))
|
||||
if not (fired and resolved and ready == len(uploads)):
|
||||
sys.exit('FAIL')
|
||||
log('PASS')
|
||||
|
||||
|
||||
def scenario_db():
|
||||
"""Incident 2: slow DB (CPU-throttled postgres) vs slow handler (large upload); then DB unavailable (paused)."""
|
||||
js.wait_ready()
|
||||
auth, chat = js.session()
|
||||
columns = [
|
||||
('p95_1m', 'histogram_quantile(0.95, sum by (le) (rate(http_server_request_duration_seconds_bucket{job="messenger-api",http_route!~"/health.*"}[1m])))', 'ms'),
|
||||
('5xx_1m', '(sum(rate(http_server_requests_total{job="messenger-api",http_route!~"/health.*",http_response_status_class="5xx"}[1m])) or vector(0)) / sum(rate(http_server_requests_total{job="messenger-api",http_route!~"/health.*"}[1m]))', '%'),
|
||||
('pool_wait', 'sum(db_client_connection_pending_requests{job="messenger-api"})', ''),
|
||||
('pool_used', 'sum(db_client_connection_count{job="messenger-api",db_client_connection_state="used"})', ''),
|
||||
('core_5xx_1m', 'sum(increase(http_server_requests_total{http_route="/health/core",http_response_status_class="5xx"}[1m]))', ''),
|
||||
('api_cpu', 'sum(rate(process_cpu_time_seconds_total{job="messenger-api"}[1m]))', ''),
|
||||
]
|
||||
alerts = ['MessengerSlowApi', 'MessengerHighErrorRate', 'MessengerApiNotReady']
|
||||
traffic = Traffic(auth, chat).run()
|
||||
try:
|
||||
log('== baseline: 60 s of traffic')
|
||||
watch(columns, alerts, lambda: False, timeout=60, every=20)
|
||||
traffic.snapshot('baseline')
|
||||
|
||||
log('== slow handler (not DB): 1700x1700 noise PNG upload — decoding and S3 dominate')
|
||||
big = noise_png(1700, 1700)
|
||||
since = time.time()
|
||||
code, raw, _ = js.call('POST', '/api/v1/chats/%s/attachments?filename=big.png' % chat,
|
||||
headers={**auth, 'Content-Type': 'image/png'}, raw=big)
|
||||
log('POST %.1f MiB attachment -> %s' % (len(big) / 2 ** 20, code))
|
||||
time.sleep(15)
|
||||
for t in find_traces('{resource.service.name="messenger-api" && kind=server && span.http.route="/api/v1/chats/{chat_id}/attachments"}', since, 1):
|
||||
breakdown(t['traceID'], 'slow handler')
|
||||
|
||||
log('== slow DB: postgres limited to 0.05 CPU for 150 s')
|
||||
postgres = container('postgres')
|
||||
since = time.time()
|
||||
docker('update', '--cpus', '0.05', *postgres)
|
||||
watch(columns, alerts, lambda: alert_state('MessengerSlowApi') == 'firing', timeout=150, every=15)
|
||||
traffic.snapshot('slow DB')
|
||||
slow = find_traces('{resource.service.name="messenger-api" && kind=server && duration > 500ms}', since, 3)
|
||||
for t in slow[:2]:
|
||||
breakdown(t['traceID'], 'slow DB')
|
||||
docker('update', '--cpus', '1.0', *postgres)
|
||||
watch(columns, alerts, lambda: alert_state('MessengerSlowApi') != 'firing', timeout=240, every=15)
|
||||
traffic.snapshot('after slow DB')
|
||||
|
||||
log('== DB unavailable: postgres paused for 120 s')
|
||||
since = time.time()
|
||||
docker('pause', *postgres)
|
||||
not_ready = watch(columns, alerts, lambda: alert_state('MessengerApiNotReady') == 'firing', timeout=120, every=15)
|
||||
traffic.snapshot('DB paused')
|
||||
for t in find_traces('{resource.service.name="messenger-api" && kind=server && status=error}', since, 2):
|
||||
breakdown(t['traceID'], 'DB unavailable')
|
||||
logs = get(LOKI + 'query_range', query='{service_name="messenger-api"} | event="http.error"', limit=3, since='3m')
|
||||
for stream in logs['data']['result'][:3]:
|
||||
log(' loki:', stream['values'][0][1][:200])
|
||||
docker('unpause', *postgres)
|
||||
recovered = watch(columns, alerts, lambda: alert_state('MessengerApiNotReady') == 'inactive', timeout=240, every=15)
|
||||
traffic.snapshot('after DB unpaused')
|
||||
finally:
|
||||
traffic.close()
|
||||
subprocess.run(['docker', 'unpause', *container('postgres')], capture_output=True)
|
||||
subprocess.run(['docker', 'update', '--cpus', '1.0', *container('postgres')], capture_output=True)
|
||||
log('MessengerApiNotReady fired: %s, resolved after recovery: %s' % (not_ready, recovered))
|
||||
log('PASS' if not_ready and recovered else 'FAIL')
|
||||
|
||||
|
||||
def scenario_telemetry():
|
||||
"""Incident 3: Loki and Tempo stopped, then the Collector stopped; API keeps serving, memory stays bounded."""
|
||||
js.wait_ready()
|
||||
auth, chat = js.session()
|
||||
columns = [
|
||||
('spans_failed_1m', 'sum(increase(otelcol_exporter_send_failed_spans[1m]))', ''),
|
||||
('logs_failed_1m', 'sum(increase(otelcol_exporter_send_failed_log_records[1m]))', ''),
|
||||
('queue', 'max(otelcol_exporter_queue_size)', ''),
|
||||
('queue_cap', 'max(otelcol_exporter_queue_capacity)', ''),
|
||||
('refused_1m', 'sum(increase(otelcol_receiver_refused_spans[1m])) + sum(increase(otelcol_receiver_refused_log_records[1m]))', ''),
|
||||
('collector_rss', 'max(otelcol_process_memory_rss)', 'MiB'),
|
||||
('api_rss', 'max(process_memory_usage_bytes{job="messenger-api"})', 'MiB'),
|
||||
]
|
||||
alerts = ['TelemetryExportFailing', 'MessengerHighErrorRate']
|
||||
traffic = Traffic(auth, chat, threads=6, pause=0.05).run()
|
||||
try:
|
||||
watch(columns, alerts, lambda: False, timeout=30, every=15)
|
||||
traffic.snapshot('baseline')
|
||||
log('== Loki and Tempo stopped until TelemetryExportFailing fires (max 240 s)')
|
||||
js.compose('stop', 'loki', 'tempo')
|
||||
fired = watch(columns, alerts, lambda: alert_state('TelemetryExportFailing') == 'firing', timeout=240, every=15)
|
||||
traffic.snapshot('Loki+Tempo down')
|
||||
js.compose('start', 'loki', 'tempo')
|
||||
resolved = watch(columns, alerts, lambda: alert_state('TelemetryExportFailing') == 'inactive', timeout=240, every=15)
|
||||
traffic.snapshot('after start')
|
||||
|
||||
log('== Collector stopped for 300 s: SDK export fails fast (5 s timeout, bounded queues); RSS saw-tooth until GC')
|
||||
js.compose('stop', 'otel-collector')
|
||||
for _ in range(15):
|
||||
time.sleep(20)
|
||||
log('docker stats:', subprocess.run(
|
||||
['docker', 'stats', '--no-stream', '--format', '{{.Name}} {{.MemUsage}}', *container('api'), *container('worker')],
|
||||
capture_output=True, text=True).stdout.replace('\n', '; '))
|
||||
traffic.snapshot('collector down')
|
||||
js.compose('start', 'otel-collector')
|
||||
time.sleep(30)
|
||||
traffic.snapshot('collector back')
|
||||
finally:
|
||||
traffic.close()
|
||||
subprocess.run(['docker', 'compose', 'start', 'loki', 'tempo', 'otel-collector'], capture_output=True)
|
||||
restarts = subprocess.run(['docker', 'inspect', '-f', '{{.Name}} restarts={{.RestartCount}} oom={{.State.OOMKilled}}',
|
||||
*container('api'), *container('worker')], capture_output=True, text=True).stdout.split('\n')
|
||||
log('containers:', '; '.join(r for r in restarts if r))
|
||||
log('TelemetryExportFailing fired: %s, resolved: %s' % (fired, resolved))
|
||||
log('PASS' if fired and resolved else 'FAIL')
|
||||
|
||||
|
||||
def scenario_retry_trace():
|
||||
"""Level 4 trace: one upload whose first attempt fails on S3, then succeeds; all attempts in one trace."""
|
||||
js.wait_ready()
|
||||
auth, chat = js.session()
|
||||
trace_id = os.urandom(16).hex()
|
||||
js.compose('stop', 'worker')
|
||||
code, raw, _ = js.call('POST', '/api/v1/chats/%s/attachments?filename=sample.png' % chat,
|
||||
headers={**auth, 'Content-Type': 'image/png', 'traceparent': '00-%s-%s-01' % (trace_id, os.urandom(8).hex())},
|
||||
raw=js.SAMPLE)
|
||||
attachment = js.json_of(raw)['id']
|
||||
log('upload ->', code, attachment, 'trace', trace_id)
|
||||
js.compose('stop', 's3')
|
||||
js.start_only('worker')
|
||||
time.sleep(5)
|
||||
js.start_only('s3')
|
||||
js.wait_status(auth, attachment, ('ready', 'failed'))
|
||||
js.job_state(attachment)
|
||||
time.sleep(15)
|
||||
spans = trace_spans(trace_id)
|
||||
for span in sorted(spans, key=lambda s: int(s['startTimeUnixNano'])):
|
||||
if span.get('kind') in ('SPAN_KIND_SERVER', 'SPAN_KIND_PRODUCER', 'SPAN_KIND_CONSUMER') or span['name'].startswith('S3.'):
|
||||
log(' %-17s %-20s %-36s attempt=%-2s %8.1f ms %s' % (
|
||||
span['service'], span.get('kind', '')[10:], span['name'], span['attrs'].get('messaging.job.attempt', ''),
|
||||
span['ms'], span.get('status', {}).get('code', '')[12:]))
|
||||
consumers = [s for s in spans if s.get('kind') == 'SPAN_KIND_CONSUMER']
|
||||
log('consumer spans in one trace: %d (attempts %s)' % (
|
||||
len(consumers), sorted({s['attrs'].get('messaging.job.attempt') for s in consumers})))
|
||||
log('PASS' if len(consumers) >= 2 else 'FAIL')
|
||||
|
||||
|
||||
SCENARIOS = {
|
||||
'worker-stop': scenario_worker_stop,
|
||||
'db': scenario_db,
|
||||
'telemetry': scenario_telemetry,
|
||||
'retry-trace': scenario_retry_trace,
|
||||
}
|
||||
|
||||
if __name__ == '__main__':
|
||||
if len(sys.argv) != 2 or sys.argv[1] not in SCENARIOS:
|
||||
sys.exit('usage: incidents.py %s' % '|'.join(SCENARIOS))
|
||||
SCENARIOS[sys.argv[1]]()
|
||||
Executable
+394
@@ -0,0 +1,394 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Defense scenarios for background attachment processing; run inside the tools container."""
|
||||
import atexit
|
||||
import base64
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
import uuid
|
||||
|
||||
BASE = os.environ.get('BASE_URL', 'http://localhost:8080')
|
||||
SAMPLE = open(os.path.join(os.path.dirname(__file__), '..', 'tests', 'fixtures', 'sample.png'), 'rb').read()
|
||||
STARTED = time.monotonic()
|
||||
ANSI = re.compile(chr(27) + re.escape('[') + '[0-9;]*m')
|
||||
|
||||
|
||||
def log(*parts):
|
||||
print('[%6.1fs]' % (time.monotonic() - STARTED), *parts, flush=True)
|
||||
|
||||
|
||||
def call(method, path, body=None, headers=None, raw=None):
|
||||
data = raw if raw is not None else (json.dumps(body).encode() if body is not None else None)
|
||||
all_headers = dict(headers or {})
|
||||
if body is not None:
|
||||
all_headers['Content-Type'] = 'application/json'
|
||||
request = urllib.request.Request(BASE + path, data=data, method=method, headers=all_headers)
|
||||
try:
|
||||
with urllib.request.urlopen(request, timeout=15) as response:
|
||||
return response.status, response.read(), response.headers
|
||||
except urllib.error.HTTPError as error:
|
||||
return error.code, error.read(), error.headers
|
||||
except OSError as error:
|
||||
return 0, str(error).encode(), {}
|
||||
|
||||
|
||||
def json_of(raw):
|
||||
try:
|
||||
return json.loads(raw or b'null')
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
|
||||
def compose(*args):
|
||||
log('$ docker compose', ' '.join(args))
|
||||
subprocess.run(['docker', 'compose', *args], check=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
|
||||
|
||||
|
||||
def start_only(service):
|
||||
"""docker compose start also starts dependencies (worker -> s3-init -> s3); start just this container."""
|
||||
log('$ docker start', service, '(without dependencies)')
|
||||
ids = subprocess.run(['docker', 'compose', 'ps', '-aq', service], check=True, capture_output=True, text=True).stdout.split()
|
||||
subprocess.run(['docker', 'start', *ids], check=True, stdout=subprocess.DEVNULL)
|
||||
|
||||
|
||||
def psql(sql):
|
||||
out = subprocess.run(
|
||||
['docker', 'compose', 'exec', '-T', 'postgres', 'psql', '-U', 'messenger', '-d', 'messenger', '-tA', '-F', ' | ', '-c', sql],
|
||||
check=True, capture_output=True, text=True,
|
||||
)
|
||||
return out.stdout.strip()
|
||||
|
||||
|
||||
def queue_depth():
|
||||
time.sleep(6)
|
||||
out = subprocess.run(
|
||||
['docker', 'compose', 'exec', '-T', 'rabbitmq', 'rabbitmqctl', '-q', 'list_queues', 'name', 'type', 'durable', 'messages_ready', 'messages_unacknowledged'],
|
||||
capture_output=True, text=True,
|
||||
)
|
||||
return out.stdout.strip() or out.stderr.strip()
|
||||
|
||||
|
||||
def wait_ready(timeout=90):
|
||||
deadline = time.monotonic() + timeout
|
||||
while time.monotonic() < deadline:
|
||||
if call('GET', '/health/ready')[0] == 200:
|
||||
return
|
||||
time.sleep(1)
|
||||
sys.exit('API did not become ready')
|
||||
|
||||
|
||||
def session():
|
||||
username = 'job_' + uuid.uuid4().hex[:8]
|
||||
password = 'Job-scenario-' + uuid.uuid4().hex
|
||||
code, raw, _ = call('POST', '/api/v1/users', {'username': username, 'display_name': 'Job scenario', 'password': password})
|
||||
if code != 201:
|
||||
sys.exit('register failed: %s' % code)
|
||||
user = json_of(raw)
|
||||
credential = base64.b64encode((username + ':' + password).encode()).decode()
|
||||
code, raw, _ = call('POST', '/api/v1/auth/sessions', headers={'Authorization': 'Basic ' + credential})
|
||||
if code != 201:
|
||||
sys.exit('login failed: %s' % code)
|
||||
auth = {'Authorization': 'Bearer ' + json_of(raw)['token']}
|
||||
code, raw, _ = call('POST', '/api/v1/chats', {'title': 'Job scenario', 'member_ids': [user['id']]}, auth)
|
||||
if code != 201:
|
||||
sys.exit('chat failed: %s' % code)
|
||||
chat = json_of(raw)['id']
|
||||
log('user', user['id'], 'chat', chat)
|
||||
return auth, chat
|
||||
|
||||
|
||||
def upload(auth, chat, body=SAMPLE):
|
||||
code, raw, headers = call(
|
||||
'POST', '/api/v1/chats/%s/attachments?filename=sample.png' % chat,
|
||||
headers={**auth, 'Content-Type': 'image/png'}, raw=body,
|
||||
)
|
||||
attachment = json_of(raw)
|
||||
log('POST attachment ->', code, 'status=%s' % attachment.get('status'), 'id=%s' % attachment.get('id'),
|
||||
'instance=%s' % headers.get('X-Instance-Id'))
|
||||
if code != 202:
|
||||
sys.exit('upload failed')
|
||||
return attachment['id']
|
||||
|
||||
|
||||
def message(auth, chat, text, attachments=()):
|
||||
code, raw, _ = call(
|
||||
'POST', '/api/v1/chats/%s/messages' % chat, {'text': text, 'attachment_ids': list(attachments)},
|
||||
{**auth, 'Idempotency-Key': uuid.uuid4().hex},
|
||||
)
|
||||
log('POST message', 'with %d attachment(s)' % len(attachments), '->', code, 'id=%s' % (json_of(raw) or {}).get('id'))
|
||||
return code
|
||||
|
||||
|
||||
def metadata(auth, attachment):
|
||||
code, raw, _ = call('GET', '/api/v1/attachments/' + attachment, headers=auth)
|
||||
return code, json_of(raw)
|
||||
|
||||
|
||||
def content_status(auth, attachment, variant='original'):
|
||||
return call('GET', '/api/v1/attachments/%s/content?variant=%s' % (attachment, variant), headers=auth)[0]
|
||||
|
||||
|
||||
def wait_status(auth, attachment, wanted, timeout=150):
|
||||
deadline = time.monotonic() + timeout
|
||||
last = None
|
||||
while time.monotonic() < deadline:
|
||||
code, meta = metadata(auth, attachment)
|
||||
if code == 200 and meta['status'] != last:
|
||||
last = meta['status']
|
||||
log('GET attachment -> status=%s error_code=%s' % (last, meta['error_code']))
|
||||
if code == 200 and meta['status'] in wanted:
|
||||
return meta
|
||||
time.sleep(1)
|
||||
sys.exit('attachment did not reach %s' % '/'.join(wanted))
|
||||
|
||||
|
||||
def check_ready(auth, attachment, meta):
|
||||
code, raw, _ = call('GET', '/api/v1/attachments/%s/content' % attachment, headers=auth)
|
||||
log('original ->', code, 'sha256 matches upload:', code == 200 and hashlib.sha256(raw).hexdigest() == hashlib.sha256(SAMPLE).hexdigest())
|
||||
for variant in meta['variants']:
|
||||
code, raw, headers = call('GET', '/api/v1/attachments/%s/content?variant=%s' % (attachment, variant['name']), headers=auth)
|
||||
log('%-9s -> %s %s %dx%d sha256 matches metadata: %s' % (
|
||||
variant['name'], code, headers.get('Content-Type'), variant['width'], variant['height'],
|
||||
hashlib.sha256(raw).hexdigest() == variant['sha256']))
|
||||
|
||||
|
||||
def job_state(attachment):
|
||||
log('job:', psql("SELECT id, status, attempt, max_attempts, error_code FROM attachment_jobs WHERE attachment_id = '%s'" % attachment))
|
||||
log('variants rows:', psql("SELECT count(*) FROM attachment_variants WHERE attachment_id = '%s'" % attachment))
|
||||
log('events:', psql("SELECT string_agg(e.event, ' -> ' ORDER BY e.id) FROM job_events e JOIN attachment_jobs j ON j.id = e.job_id WHERE j.attachment_id = '%s'" % attachment))
|
||||
|
||||
|
||||
def scenario_restart():
|
||||
"""Level 3: worker down -> upload 202 -> restart API and broker -> worker up -> ready."""
|
||||
wait_ready()
|
||||
auth, chat = session()
|
||||
compose('stop', 'worker')
|
||||
attachment = upload(auth, chat)
|
||||
log('content while queued ->', content_status(auth, attachment))
|
||||
message(auth, chat, '', [attachment])
|
||||
message(auth, chat, 'other messages keep working')
|
||||
log('queues:', queue_depth())
|
||||
compose('restart', 'api', 'rabbitmq')
|
||||
wait_ready()
|
||||
log('queues after broker restart:', queue_depth())
|
||||
log('status after restarts:', metadata(auth, attachment)[1]['status'])
|
||||
compose('start', 'worker')
|
||||
meta = wait_status(auth, attachment, ['ready', 'failed'])
|
||||
check_ready(auth, attachment, meta)
|
||||
job_state(attachment)
|
||||
|
||||
|
||||
def remove_container(name):
|
||||
subprocess.run(['docker', 'rm', '-f', name], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
|
||||
|
||||
|
||||
def plain(line):
|
||||
return ANSI.sub('', line).split('] ')[-1].strip()
|
||||
|
||||
|
||||
def faulty(service, env):
|
||||
"""Run a one-off copy of the service with fault injection; no restart policy, so it dies once."""
|
||||
name = 'lab6-fault-%s-%s' % (service, uuid.uuid4().hex[:8])
|
||||
atexit.register(remove_container, name)
|
||||
args = ['docker', 'compose', 'run', '-d', '--no-deps', '--name', name]
|
||||
for key, value in env.items():
|
||||
args += ['-e', '%s=%s' % (key, value)]
|
||||
log('$ docker compose run --no-deps', ' '.join('-e %s=%s' % item for item in env.items()), service)
|
||||
subprocess.run(args + [service], check=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
|
||||
deadline = time.monotonic() + 30
|
||||
while time.monotonic() < deadline:
|
||||
if 'connected to broker' in subprocess.run(['docker', 'logs', name], capture_output=True, text=True).stdout:
|
||||
return name
|
||||
time.sleep(0.5)
|
||||
sys.exit('%s with fault did not start' % service)
|
||||
|
||||
|
||||
def wait_exit(name, timeout=90):
|
||||
try:
|
||||
code = subprocess.run(['docker', 'wait', name], capture_output=True, text=True, timeout=timeout).stdout.strip()
|
||||
except subprocess.TimeoutExpired:
|
||||
code = 'timeout'
|
||||
for line in subprocess.run(['docker', 'logs', name], capture_output=True, text=True).stdout.splitlines():
|
||||
if 'fault injection' in line or 'published' in line or 'job ' in line:
|
||||
log(' one-off:', plain(line))
|
||||
remove_container(name)
|
||||
log('one-off process exit code', code, '(137 = killed without ack)')
|
||||
|
||||
|
||||
def service_logs(service, since, needle):
|
||||
out = subprocess.run(['docker', 'compose', 'logs', '--no-log-prefix', '--no-color', '--since', since, service],
|
||||
capture_output=True, text=True).stdout
|
||||
for line in out.splitlines():
|
||||
if needle in line:
|
||||
log(' %s:' % service, plain(line))
|
||||
|
||||
|
||||
def utc_now():
|
||||
return time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime())
|
||||
|
||||
|
||||
def staging_key(attachment):
|
||||
return psql("SELECT coalesce(staging_key, 'NULL') FROM attachments WHERE id = '%s'" % attachment)
|
||||
|
||||
|
||||
def job_id(attachment):
|
||||
return psql("SELECT id FROM attachment_jobs WHERE attachment_id = '%s'" % attachment)
|
||||
|
||||
|
||||
def scenario_crash_after_store():
|
||||
"""Level 4: worker killed after writing S3 objects, before COMMIT and ack; lease expires; same result."""
|
||||
wait_ready()
|
||||
since = utc_now()
|
||||
auth, chat = session()
|
||||
compose('stop', 'worker')
|
||||
proc = faulty('worker', {'WORKER_FAULT': 'exit-after-store', 'JOB_LEASE_SECONDS': '15'})
|
||||
attachment = upload(auth, chat)
|
||||
wait_exit(proc)
|
||||
log('status after crash:', metadata(auth, attachment)[1]['status'])
|
||||
log('lease:', psql("SELECT status, attempt, lease_until > now() AS lease_active FROM attachment_jobs WHERE attachment_id = '%s'" % attachment))
|
||||
log('queues (unacked message returned):', queue_depth())
|
||||
compose('start', 'worker')
|
||||
meta = wait_status(auth, attachment, ['ready', 'failed'])
|
||||
check_ready(auth, attachment, meta)
|
||||
job_state(attachment)
|
||||
service_logs('worker', since, job_id(attachment))
|
||||
service_logs('relay', since, 'lease expired')
|
||||
|
||||
|
||||
def scenario_crash_after_commit():
|
||||
"""Level 4: worker killed after COMMIT ready, before ack; redelivery is a no-op and cleans staging."""
|
||||
wait_ready()
|
||||
since = utc_now()
|
||||
auth, chat = session()
|
||||
compose('stop', 'worker')
|
||||
proc = faulty('worker', {'WORKER_FAULT': 'exit-after-commit'})
|
||||
attachment = upload(auth, chat)
|
||||
wait_exit(proc)
|
||||
log('status after crash:', metadata(auth, attachment)[1]['status'], 'staging_key:', staging_key(attachment))
|
||||
log('queues (unacked message returned):', queue_depth())
|
||||
compose('start', 'worker')
|
||||
deadline = time.monotonic() + 30
|
||||
while staging_key(attachment) != 'NULL' and time.monotonic() < deadline:
|
||||
time.sleep(1)
|
||||
log('staging_key after redelivery:', staging_key(attachment))
|
||||
check_ready(auth, attachment, metadata(auth, attachment)[1])
|
||||
job_state(attachment)
|
||||
service_logs('worker', since, job_id(attachment))
|
||||
|
||||
|
||||
def scenario_outbox_gap():
|
||||
"""Level 4: crash between COMMIT and publish; the outbox row is published after recovery."""
|
||||
wait_ready()
|
||||
auth, chat = session()
|
||||
compose('stop', 'relay')
|
||||
attachment = upload(auth, chat)
|
||||
outbox = "SELECT o.id, coalesce(o.published_at::text, 'NOT PUBLISHED'), o.publish_attempts FROM outbox o JOIN attachment_jobs j ON j.id = o.job_id WHERE j.attachment_id = '%s'" % attachment
|
||||
log('outbox after COMMIT:', psql(outbox))
|
||||
log('queues:', queue_depth())
|
||||
compose('kill', 'api')
|
||||
log('API killed (SIGKILL) after COMMIT, nothing was published')
|
||||
compose('start', 'api')
|
||||
wait_ready()
|
||||
log('status after API restart:', metadata(auth, attachment)[1]['status'])
|
||||
compose('start', 'relay')
|
||||
meta = wait_status(auth, attachment, ['ready', 'failed'])
|
||||
log('outbox after relay recovery:', psql(outbox))
|
||||
check_ready(auth, attachment, meta)
|
||||
job_state(attachment)
|
||||
|
||||
|
||||
def scenario_relay_crash():
|
||||
"""Level 4: relay killed after broker confirm, before marking outbox; duplicate delivery is a no-op."""
|
||||
wait_ready()
|
||||
since = utc_now()
|
||||
auth, chat = session()
|
||||
compose('stop', 'relay')
|
||||
proc = faulty('relay', {'RELAY_FAULT': 'exit-after-confirm'})
|
||||
attachment = upload(auth, chat)
|
||||
wait_exit(proc)
|
||||
outbox = "SELECT o.id, coalesce(o.published_at::text, 'NOT MARKED'), o.publish_attempts FROM outbox o JOIN attachment_jobs j ON j.id = o.job_id WHERE j.attachment_id = '%s'" % attachment
|
||||
log('outbox after relay crash:', psql(outbox))
|
||||
compose('start', 'relay')
|
||||
meta = wait_status(auth, attachment, ['ready', 'failed'])
|
||||
time.sleep(3)
|
||||
log('outbox after relay restart:', psql(outbox))
|
||||
check_ready(auth, attachment, meta)
|
||||
job_state(attachment)
|
||||
service_logs('worker', since, job_id(attachment))
|
||||
|
||||
|
||||
def replay(*args):
|
||||
out = subprocess.run(['docker', 'compose', 'exec', '-T', '-e', 'OPERATOR=defense-demo', 'worker', 'node', 'dist/replay.js', *args],
|
||||
capture_output=True, text=True)
|
||||
for line in (out.stdout.strip() or out.stderr.strip()).splitlines():
|
||||
log(' replay:', line)
|
||||
|
||||
|
||||
def scenario_retry_replay():
|
||||
"""Level 4: S3 down -> retries with backoff and jitter -> failed retry_exhausted -> explicit replay -> ready."""
|
||||
wait_ready()
|
||||
auth, chat = session()
|
||||
compose('stop', 'worker')
|
||||
attachment = upload(auth, chat)
|
||||
compose('stop', 's3')
|
||||
start_only('worker')
|
||||
meta = wait_status(auth, attachment, ['failed'], timeout=120)
|
||||
log('failed attachment in a new message ->', message(auth, chat, '', [attachment]))
|
||||
log('retries:')
|
||||
for line in psql("SELECT e.attempt, e.event, coalesce(e.detail, ''), to_char(e.created_at, 'HH24:MI:SS.MS') FROM job_events e JOIN attachment_jobs j ON j.id = e.job_id WHERE j.attachment_id = '%s' ORDER BY e.id" % attachment).splitlines():
|
||||
log(' ', line)
|
||||
log('$ replay.js --list')
|
||||
replay('--list')
|
||||
compose('start', 's3')
|
||||
wait_ready()
|
||||
log('$ replay.js', attachment)
|
||||
replay(attachment)
|
||||
meta = wait_status(auth, attachment, ['ready', 'failed'])
|
||||
check_ready(auth, attachment, meta)
|
||||
log('$ replay.js', attachment, '(already ready)')
|
||||
replay(attachment)
|
||||
job_state(attachment)
|
||||
|
||||
|
||||
def scenario_broker():
|
||||
"""Level 4: prefetch/concurrency, limits and dead letter queue for a malformed message."""
|
||||
wait_ready()
|
||||
since = utc_now()
|
||||
log('consumers:', subprocess.run(
|
||||
['docker', 'compose', 'exec', '-T', 'rabbitmq', 'rabbitmqctl', '-q', 'list_consumers', 'queue_name', 'prefetch_count', 'ack_required'],
|
||||
capture_output=True, text=True).stdout.strip())
|
||||
log('worker limits:', subprocess.run(
|
||||
['sh', '-c', "docker inspect -f 'cpus={{.HostConfig.NanoCpus}} memory={{.HostConfig.Memory}}' $(docker compose ps -q worker)"],
|
||||
capture_output=True, text=True).stdout.strip())
|
||||
publish = (
|
||||
"const amqp=require('amqplib');const fs=require('fs');(async()=>{"
|
||||
"const c=await amqp.connect(fs.readFileSync(process.env.BROKER_URL_FILE,'utf8').trim());"
|
||||
"const ch=await c.createConfirmChannel();"
|
||||
"ch.sendToQueue('attachment.jobs',Buffer.from(JSON.stringify({job_id:'not-a-uuid',url:'http://example.invalid/x'})),{persistent:true});"
|
||||
"await ch.waitForConfirms();await c.close();console.log('malformed message published');})()"
|
||||
)
|
||||
log(subprocess.run(['docker', 'compose', 'exec', '-T', 'worker', 'node', '-e', publish], capture_output=True, text=True).stdout.strip())
|
||||
log('queues:', queue_depth())
|
||||
service_logs('worker', since, 'malformed')
|
||||
|
||||
|
||||
SCENARIOS = {
|
||||
'restart': scenario_restart,
|
||||
'crash-after-store': scenario_crash_after_store,
|
||||
'crash-after-commit': scenario_crash_after_commit,
|
||||
'outbox-gap': scenario_outbox_gap,
|
||||
'relay-crash': scenario_relay_crash,
|
||||
'retry-replay': scenario_retry_replay,
|
||||
'broker': scenario_broker,
|
||||
}
|
||||
|
||||
if __name__ == '__main__':
|
||||
if len(sys.argv) != 2 or sys.argv[1] not in SCENARIOS:
|
||||
sys.exit('usage: job-scenario.py %s' % '|'.join(SCENARIOS))
|
||||
SCENARIOS[sys.argv[1]]()
|
||||
Executable
+52
@@ -0,0 +1,52 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Compares client-side percentiles of tests/load.py with histogram_quantile over the same window in Prometheus.
|
||||
|
||||
Run inside the tools container: docker compose run --rm tools python3 scripts/latency-compare.py [--duration 30]
|
||||
"""
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import time
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
|
||||
GRAFANA = os.environ.get('GRAFANA_URL', 'http://grafana:3000')
|
||||
BASE = os.environ.get('BASE_URL', 'http://lb:8080')
|
||||
SERIES = 'http_server_request_duration_seconds_bucket{job="messenger-api",http_request_method="POST",http_route="/api/v1/chats/{chat_id}/messages"}'
|
||||
|
||||
|
||||
def prom(expr, at):
|
||||
params = urllib.parse.urlencode({'query': expr, 'time': at})
|
||||
with urllib.request.urlopen(GRAFANA + '/api/datasources/proxy/uid/prometheus/api/v1/query?' + params, timeout=15) as r:
|
||||
result = json.loads(r.read())['data']['result']
|
||||
return float(result[0]['value'][1]) if result else None
|
||||
|
||||
|
||||
def main():
|
||||
duration = sys.argv[sys.argv.index('--duration') + 1] if '--duration' in sys.argv else '30'
|
||||
output = os.path.join(tempfile.mkdtemp(), 'load.json')
|
||||
started = time.time()
|
||||
subprocess.run([sys.executable, 'tests/load.py', '--base-url', BASE, '--duration', duration, '--output', output],
|
||||
check=True, stdout=subprocess.DEVNULL)
|
||||
report = json.load(open(output))
|
||||
print('waiting 25 s for the metric export interval')
|
||||
time.sleep(25)
|
||||
at = time.time()
|
||||
window = int(at - started) + 5
|
||||
print('client (tests/load.py): %d requests, %.1f rps, errors %d' % (
|
||||
report['successful_requests'], report['successful_rps'], report['failed_requests']))
|
||||
count = prom('sum(increase(%s[%ds]))' % (SERIES.replace('_bucket{', '_count{'), window), at)
|
||||
print('prometheus: %.0f requests in POST /api/v1/chats/{chat_id}/messages over %d s' % (count or 0, window))
|
||||
print('%-5s %12s %16s' % ('', 'client, ms', 'prometheus, ms'))
|
||||
for p in ('p50', 'p95', 'p99'):
|
||||
q = int(p[1:]) / 100
|
||||
server = prom('histogram_quantile(%s, sum by (le) (increase(%s[%ds])))' % (q, SERIES, window), at)
|
||||
print('%-5s %12.2f %16.2f' % (p, report['successful_latency_ms'][p], (server or 0) * 1000))
|
||||
print('server-side histogram excludes network/client overhead; values are interpolated inside buckets '
|
||||
'(…, 5, 10, 25, 50, 100 ms, …).')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
Executable
+88
@@ -0,0 +1,88 @@
|
||||
#!/usr/bin/env python3
|
||||
import base64
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
import uuid
|
||||
|
||||
BASE = os.environ.get('BASE_URL', 'http://localhost:8080')
|
||||
|
||||
|
||||
def call(method, path, body=None, headers=None, with_headers=False):
|
||||
data = json.dumps(body).encode() if body is not None else None
|
||||
content_type = {'Content-Type': 'application/json'} if data else {}
|
||||
request = urllib.request.Request(BASE + path, data=data, method=method, headers={**content_type, **(headers or {})})
|
||||
try:
|
||||
with urllib.request.urlopen(request, timeout=15) as response:
|
||||
result = response.status, json.loads(response.read() or b'null'), response.headers
|
||||
except urllib.error.HTTPError as error:
|
||||
result = error.code, json.loads(error.read() or b'null'), error.headers
|
||||
except OSError as error:
|
||||
result = 0, str(error), {}
|
||||
return result if with_headers else result[:2]
|
||||
|
||||
|
||||
def login():
|
||||
username = 'probe_' + uuid.uuid4().hex[:8]
|
||||
password = 'Probe-check-' + uuid.uuid4().hex
|
||||
print('register ->', call('POST', '/api/v1/users', {'username': username, 'display_name': 'Probe', 'password': password})[0])
|
||||
credential = base64.b64encode((username + ':' + password).encode()).decode()
|
||||
code, session = call('POST', '/api/v1/auth/sessions', headers={'Authorization': 'Basic ' + credential})
|
||||
if code != 201:
|
||||
sys.exit('login failed: %s %s' % (code, session))
|
||||
print('login ->', code, 'expires_at:', session['expires_at'])
|
||||
return {'Authorization': 'Bearer ' + session['token']}
|
||||
|
||||
|
||||
def instances(count):
|
||||
auth = login()
|
||||
for _ in range(count):
|
||||
code, _, headers = call('GET', '/api/v1/users/me', headers=auth, with_headers=True)
|
||||
print('GET /api/v1/users/me ->', code, 'X-Instance-Id:', headers.get('X-Instance-Id'))
|
||||
|
||||
|
||||
def logout():
|
||||
auth = login()
|
||||
print('GET /api/v1/users/me ->', call('GET', '/api/v1/users/me', headers=auth)[0])
|
||||
code, body = call('DELETE', '/api/v1/auth/sessions/current', headers=auth)
|
||||
print('DELETE /api/v1/auth/sessions/current ->', code, json.dumps(body, ensure_ascii=False) if body else '')
|
||||
print('GET /api/v1/users/me ->', call('GET', '/api/v1/users/me', headers=auth)[0])
|
||||
|
||||
|
||||
def ready(timeout=90):
|
||||
deadline = time.monotonic() + timeout
|
||||
while time.monotonic() < deadline:
|
||||
if call('GET', '/health/ready')[0] == 200:
|
||||
print('ready')
|
||||
return
|
||||
time.sleep(0.5)
|
||||
sys.exit('not ready within %ds' % timeout)
|
||||
|
||||
|
||||
def status(method, path, authorization=None):
|
||||
code, body = call(method, path, headers={'Authorization': authorization} if authorization else None)
|
||||
print(method, path, '(%s)' % (authorization or 'anonymous'), '->', code, json.dumps(body, ensure_ascii=False))
|
||||
|
||||
|
||||
def expiry(ttl):
|
||||
auth = login()
|
||||
print('GET /api/v1/users/me (fresh) ->', call('GET', '/api/v1/users/me', headers=auth)[0])
|
||||
time.sleep(ttl + 1)
|
||||
print('GET /api/v1/users/me (after %ds) ->' % (ttl + 1), call('GET', '/api/v1/users/me', headers=auth)[0])
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
command = sys.argv[1]
|
||||
if command == 'ready':
|
||||
ready()
|
||||
elif command == 'status':
|
||||
status(*sys.argv[2:5])
|
||||
elif command == 'expiry':
|
||||
expiry(int(sys.argv[2]))
|
||||
elif command == 'instances':
|
||||
instances(int(sys.argv[2]))
|
||||
elif command == 'logout':
|
||||
logout()
|
||||
Executable
+9
@@ -0,0 +1,9 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
started() {
|
||||
docker inspect -f '{{.Name}} {{.State.Status}} started={{.State.StartedAt}}' $(docker compose ps -aq postgres redis s3 rabbitmq api lb)
|
||||
}
|
||||
started
|
||||
docker compose restart postgres redis s3 rabbitmq api lb
|
||||
started
|
||||
Executable
+11
@@ -0,0 +1,11 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
: "${TARGET_INSTANCE_ID:?test provides target API instance alias}"
|
||||
project="${COMPOSE_PROJECT_NAME:-messenger-lab6}"
|
||||
labels=$(docker inspect -f '{{index .Config.Labels "com.docker.compose.project"}}/{{index .Config.Labels "com.docker.compose.service"}}' "$TARGET_INSTANCE_ID")
|
||||
if [ "$labels" != "$project/api" ]; then
|
||||
echo "refusing to stop $TARGET_INSTANCE_ID: not an api replica of $project ($labels)" >&2
|
||||
exit 1
|
||||
fi
|
||||
docker stop -t 8 "$TARGET_INSTANCE_ID" >/dev/null
|
||||
docker inspect -f '{{.Name}} {{.State.Status}} finished={{.State.FinishedAt}}' "$TARGET_INSTANCE_ID"
|
||||
Executable
+102
@@ -0,0 +1,102 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Checks that passwords, tokens, message text and filenames do not reach Loki/Tempo and that metric labels hold no IDs.
|
||||
|
||||
Run inside the tools container: docker compose run --rm tools python3 scripts/telemetry-privacy.py
|
||||
"""
|
||||
import base64
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import time
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
import uuid
|
||||
|
||||
BASE = os.environ.get('BASE_URL', 'http://lb:8080')
|
||||
GRAFANA = os.environ.get('GRAFANA_URL', 'http://grafana:3000')
|
||||
SAMPLE = open(os.path.join(os.path.dirname(__file__), '..', 'tests', 'fixtures', 'sample.png'), 'rb').read()
|
||||
UUID = re.compile(r'[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}')
|
||||
|
||||
|
||||
def call(method, path, body=None, headers=None, raw=None):
|
||||
data = raw if raw is not None else (json.dumps(body).encode() if body is not None else None)
|
||||
headers = dict(headers or {})
|
||||
if body is not None:
|
||||
headers['Content-Type'] = 'application/json'
|
||||
with urllib.request.urlopen(urllib.request.Request(BASE + path, data=data, method=method, headers=headers), timeout=15) as r:
|
||||
return json.loads(r.read() or b'null')
|
||||
|
||||
|
||||
def grafana(path, **params):
|
||||
with urllib.request.urlopen(GRAFANA + path + '?' + urllib.parse.urlencode(params), timeout=30) as r:
|
||||
return r.read().decode()
|
||||
|
||||
|
||||
def main():
|
||||
marker = uuid.uuid4().hex[:12]
|
||||
secrets = {
|
||||
'password': 'Privacy-password-' + marker,
|
||||
'message text': 'privacy message body ' + marker,
|
||||
'filename': 'privacy-file-%s.png' % marker,
|
||||
}
|
||||
trace_ids = []
|
||||
|
||||
def traced():
|
||||
trace_id = os.urandom(16).hex()
|
||||
trace_ids.append(trace_id)
|
||||
return {'traceparent': '00-%s-%s-01' % (trace_id, os.urandom(8).hex())}
|
||||
|
||||
username = 'privacy_' + marker
|
||||
user = call('POST', '/api/v1/users', {'username': username, 'display_name': 'Privacy', 'password': secrets['password']}, traced())
|
||||
basic = base64.b64encode(('%s:%s' % (username, secrets['password'])).encode()).decode()
|
||||
token = call('POST', '/api/v1/auth/sessions', headers={'Authorization': 'Basic ' + basic, **traced()})['token']
|
||||
secrets['session token'] = token
|
||||
secrets['basic credentials'] = basic
|
||||
auth = {'Authorization': 'Bearer ' + token}
|
||||
chat = call('POST', '/api/v1/chats', {'title': 'Privacy', 'member_ids': [user['id']]}, {**auth, **traced()})['id']
|
||||
call('POST', '/api/v1/chats/%s/messages' % chat, {'text': secrets['message text']},
|
||||
{**auth, 'Idempotency-Key': marker, **traced()})
|
||||
call('POST', '/api/v1/chats/%s/attachments?filename=%s' % (chat, secrets['filename']), headers={**auth, 'Content-Type': 'image/png', **traced()},
|
||||
raw=SAMPLE)
|
||||
call('GET', '/api/v1/chats/%s/messages?limit=5' % chat, headers={**auth, **traced()})
|
||||
print('sent %d traced requests with known password, token, message text and filename; waiting 25 s for export' % len(trace_ids))
|
||||
time.sleep(25)
|
||||
|
||||
traces = ''
|
||||
found_traces = 0
|
||||
for trace_id in trace_ids:
|
||||
try:
|
||||
traces += grafana('/api/datasources/proxy/uid/tempo/api/traces/' + trace_id)
|
||||
found_traces += 1
|
||||
except OSError:
|
||||
pass
|
||||
def loki_lines(query):
|
||||
body = grafana('/api/datasources/proxy/uid/loki/loki/api/v1/query_range', query=query, limit=100, since='5m')
|
||||
return sum(len(s['values']) for s in json.loads(body)['data']['result'])
|
||||
|
||||
own = loki_lines('{service_name=~"messenger-.+"} |~ "%s"' % '|'.join(trace_ids))
|
||||
print('Tempo: %d/%d traces fetched (%d bytes); Loki: %d log lines carry these trace_ids (control)' % (
|
||||
found_traces, len(trace_ids), len(traces), own))
|
||||
leaks = 0
|
||||
for name, value in list(secrets.items()) + [('Authorization hdr', 'authorization')]:
|
||||
in_traces = value.lower() in traces.lower()
|
||||
in_logs = loki_lines('{service_name=~"messenger-.+"} |~ "(?i)%s"' % re.escape(value).replace('\\', '\\\\'))
|
||||
leaks += in_traces + in_logs
|
||||
print(' %-18s in traces: %-5s log lines in Loki (5 min): %d' % (name, in_traces, in_logs))
|
||||
|
||||
series = json.loads(grafana('/api/datasources/proxy/uid/prometheus/api/v1/series', **{'match[]': '{job=~"messenger-.+"}'}))['data']
|
||||
names = {s['__name__'] for s in series}
|
||||
with_ids = [s for s in series if any(UUID.search(v) for k, v in s.items() if k != 'instance')]
|
||||
labels = sorted({k for s in series for k in s} - {'__name__'})
|
||||
print('Prometheus: %d series, %d metric names from messenger services; label keys: %s' % (len(series), len(names), ', '.join(labels)))
|
||||
print(' series with a UUID in any label value: %d' % len(with_ids))
|
||||
routes = sorted({s.get('http_route') for s in series if s.get('http_route')})
|
||||
print(' http_route values: %s' % ', '.join(routes))
|
||||
ok = found_traces == len(trace_ids) and own > 0 and leaks == 0 and not with_ids
|
||||
print('%s: no secrets/content in logs and traces, no IDs in metric labels' % ('PASS' if ok else 'FAIL'))
|
||||
sys.exit(0 if ok else 1)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
Executable
+134
@@ -0,0 +1,134 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Control request for lab6: upload with a client traceparent, then find its trace in Tempo and logs in Loki.
|
||||
|
||||
Run inside the tools container: docker compose run --rm tools python3 scripts/trace-check.py
|
||||
"""
|
||||
import base64
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
import uuid
|
||||
|
||||
BASE = os.environ.get('BASE_URL', 'http://localhost:8080')
|
||||
GRAFANA = os.environ.get('GRAFANA_URL', 'http://grafana:3000')
|
||||
SAMPLE = open(os.path.join(os.path.dirname(__file__), '..', 'tests', 'fixtures', 'sample.png'), 'rb').read()
|
||||
|
||||
|
||||
def call(method, url, body=None, headers=None, raw=None):
|
||||
data = raw if raw is not None else (json.dumps(body).encode() if body is not None else None)
|
||||
all_headers = dict(headers or {})
|
||||
if body is not None:
|
||||
all_headers['Content-Type'] = 'application/json'
|
||||
request = urllib.request.Request(url, data=data, method=method, headers=all_headers)
|
||||
try:
|
||||
with urllib.request.urlopen(request, timeout=15) as response:
|
||||
return response.status, json.loads(response.read() or b'null')
|
||||
except urllib.error.HTTPError as error:
|
||||
return error.code, None
|
||||
|
||||
|
||||
def session():
|
||||
username = 'trace_' + uuid.uuid4().hex[:8]
|
||||
password = 'Trace-check-' + uuid.uuid4().hex
|
||||
_, user = call('POST', BASE + '/api/v1/users', {'username': username, 'display_name': 'Trace check', 'password': password})
|
||||
credential = base64.b64encode((username + ':' + password).encode()).decode()
|
||||
_, login = call('POST', BASE + '/api/v1/auth/sessions', headers={'Authorization': 'Basic ' + credential})
|
||||
auth = {'Authorization': 'Bearer ' + login['token']}
|
||||
_, chat = call('POST', BASE + '/api/v1/chats', {'title': 'Trace check', 'member_ids': [user['id']]}, auth)
|
||||
return auth, chat['id']
|
||||
|
||||
|
||||
def wait_trace(trace_id, services, timeout=60):
|
||||
deadline = time.monotonic() + timeout
|
||||
spans = []
|
||||
while time.monotonic() < deadline:
|
||||
code, body = call('GET', GRAFANA + '/api/datasources/proxy/uid/tempo/api/traces/' + trace_id)
|
||||
if code == 200 and body:
|
||||
spans = []
|
||||
for batch in body.get('batches', []):
|
||||
attrs = {a['key']: a['value'] for a in batch['resource']['attributes']}
|
||||
service = attrs['service.name']['stringValue']
|
||||
for scope in batch.get('scopeSpans', []):
|
||||
for span in scope.get('spans', []):
|
||||
spans.append((service, span))
|
||||
if services <= {service for service, _ in spans}:
|
||||
return spans
|
||||
time.sleep(3)
|
||||
return spans
|
||||
|
||||
|
||||
def b64hex(value):
|
||||
return base64.b64decode(value).hex() if value else ''
|
||||
|
||||
|
||||
def main():
|
||||
trace_id = os.urandom(16).hex()
|
||||
client_span = os.urandom(8).hex()
|
||||
traceparent = '00-%s-%s-01' % (trace_id, client_span)
|
||||
auth, chat = session()
|
||||
started = time.monotonic()
|
||||
code, attachment = call(
|
||||
'POST', BASE + '/api/v1/chats/%s/attachments?filename=sample.png' % chat,
|
||||
headers={**auth, 'Content-Type': 'image/png', 'traceparent': traceparent}, raw=SAMPLE,
|
||||
)
|
||||
print('upload ->', code, attachment and attachment['status'], 'traceparent', traceparent)
|
||||
status = None
|
||||
while time.monotonic() - started < 60:
|
||||
_, body = call('GET', BASE + '/api/v1/attachments/' + attachment['id'], headers=auth)
|
||||
status = body['status']
|
||||
if status in ('ready', 'failed'):
|
||||
break
|
||||
time.sleep(0.5)
|
||||
print('attachment', attachment['id'], 'status', status, 'after %.2f s' % (time.monotonic() - started))
|
||||
|
||||
spans = wait_trace(trace_id, {'messenger-api', 'messenger-relay', 'messenger-worker'})
|
||||
by_id = {b64hex(s['spanId']): (svc, s) for svc, s in spans}
|
||||
print('\ntrace %s: %d spans (prisma:* internals except db_query omitted below)' % (trace_id, len(spans)))
|
||||
for service, span in sorted(spans, key=lambda item: int(item[1]['startTimeUnixNano'])):
|
||||
if span['name'].startswith('prisma:') and span['name'] not in ('prisma:engine:db_query', 'prisma:client:transaction'):
|
||||
continue
|
||||
parent = b64hex(span.get('parentSpanId'))
|
||||
parent_name = 'client' if parent == client_span else by_id.get(parent, ('', {'name': '-'}))[1]['name']
|
||||
duration = (int(span['endTimeUnixNano']) - int(span['startTimeUnixNano'])) / 1e6
|
||||
print(' %-17s %-22s %-48s %8.2f ms parent=%s' % (
|
||||
service, span.get('kind', ''), span['name'][:48], duration, parent_name))
|
||||
|
||||
query = '{service_name=~"messenger-.+"} | trace_id="%s"' % trace_id
|
||||
params = urllib.parse.urlencode({'query': query, 'limit': 50, 'since': '15m'})
|
||||
streams = []
|
||||
deadline = time.monotonic() + 30
|
||||
while time.monotonic() < deadline:
|
||||
_, logs = call('GET', GRAFANA + '/api/datasources/proxy/uid/loki/loki/api/v1/query_range?' + params)
|
||||
streams = (logs or {}).get('data', {}).get('result', [])
|
||||
if {'messenger-api', 'messenger-worker'} <= {s['stream'].get('service_name') for s in streams}:
|
||||
break
|
||||
time.sleep(3)
|
||||
print('\nLoki %s' % query)
|
||||
lines = 0
|
||||
for stream in streams:
|
||||
labels = stream['stream']
|
||||
for _, line in stream['values']:
|
||||
lines += 1
|
||||
print(' %-17s %-8s %-13s span_id=%s %s' % (
|
||||
labels.get('service_name'), labels.get('severity_text', ''), labels.get('event', ''),
|
||||
labels.get('span_id', ''), line[:60]))
|
||||
|
||||
kinds = {(service, span.get('kind')) for service, span in spans}
|
||||
ok = (
|
||||
status == 'ready'
|
||||
and ('messenger-api', 'SPAN_KIND_SERVER') in kinds
|
||||
and ('messenger-relay', 'SPAN_KIND_PRODUCER') in kinds
|
||||
and ('messenger-worker', 'SPAN_KIND_CONSUMER') in kinds
|
||||
and any(b64hex(s.get('parentSpanId')) == client_span for _, s in spans)
|
||||
and {'messenger-api', 'messenger-worker'} <= {s['stream'].get('service_name') for s in streams}
|
||||
)
|
||||
print('\n%s: client trace continued through api -> relay -> worker, logs linked by trace_id' % ('PASS' if ok else 'FAIL'))
|
||||
sys.exit(0 if ok else 1)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,25 @@
|
||||
import { Module } from '@nestjs/common';
|
||||
import { AttachmentsModule } from './attachments/attachments.module';
|
||||
import { AuthModule } from './auth/auth.module';
|
||||
import { CacheModule } from './cache/cache.module';
|
||||
import { ChatsModule } from './chats/chats.module';
|
||||
import { DbModule } from './db/db.module';
|
||||
import { HealthModule } from './health/health.module';
|
||||
import { MessagesModule } from './messages/messages.module';
|
||||
import { StorageModule } from './storage/storage.module';
|
||||
import { UsersModule } from './users/users.module';
|
||||
|
||||
@Module({
|
||||
imports: [
|
||||
DbModule,
|
||||
CacheModule,
|
||||
StorageModule,
|
||||
AuthModule,
|
||||
HealthModule,
|
||||
UsersModule,
|
||||
ChatsModule,
|
||||
MessagesModule,
|
||||
AttachmentsModule,
|
||||
],
|
||||
})
|
||||
export class AppModule {}
|
||||
@@ -0,0 +1,79 @@
|
||||
import {
|
||||
Controller,
|
||||
Get,
|
||||
HttpCode,
|
||||
Param,
|
||||
Post,
|
||||
Query,
|
||||
Req,
|
||||
Res,
|
||||
UseGuards,
|
||||
} from '@nestjs/common';
|
||||
import { FastifyReply, FastifyRequest } from 'fastify';
|
||||
import { Readable } from 'stream';
|
||||
import { AuthGuard } from '../auth/auth.guard';
|
||||
import { CurrentUser } from '../auth/current-user.decorator';
|
||||
import { User } from '../common/domain';
|
||||
import { AttachmentsService } from './attachments.service';
|
||||
import {
|
||||
AttachmentResponse,
|
||||
toAttachmentResponse,
|
||||
} from './dto/attachment.response';
|
||||
|
||||
@Controller()
|
||||
@UseGuards(AuthGuard)
|
||||
export class AttachmentsController {
|
||||
constructor(private readonly attachments: AttachmentsService) {}
|
||||
|
||||
@Post('api/v1/chats/:chatId/attachments')
|
||||
@HttpCode(202)
|
||||
async upload(
|
||||
@Param('chatId') chatId: string,
|
||||
@Query('filename') filename: unknown,
|
||||
@Req() request: FastifyRequest,
|
||||
@CurrentUser() user: User,
|
||||
): Promise<AttachmentResponse> {
|
||||
const attachment = await this.attachments.upload({
|
||||
chatId,
|
||||
ownerId: user.id,
|
||||
filename,
|
||||
contentType: request.uploadContentType,
|
||||
contentLength: request.headers['content-length'],
|
||||
body: request.body as Readable | undefined,
|
||||
});
|
||||
return toAttachmentResponse(attachment);
|
||||
}
|
||||
|
||||
@Get('api/v1/attachments/:attachmentId')
|
||||
async metadata(
|
||||
@Param('attachmentId') attachmentId: string,
|
||||
@CurrentUser() user: User,
|
||||
): Promise<AttachmentResponse> {
|
||||
return toAttachmentResponse(
|
||||
await this.attachments.getVisible(attachmentId, user.id),
|
||||
);
|
||||
}
|
||||
|
||||
@Get('api/v1/attachments/:attachmentId/content')
|
||||
async content(
|
||||
@Param('attachmentId') attachmentId: string,
|
||||
@Query('variant') variant: unknown,
|
||||
@CurrentUser() user: User,
|
||||
@Res() reply: FastifyReply,
|
||||
): Promise<void> {
|
||||
const download = await this.attachments.openContent(
|
||||
attachmentId,
|
||||
user.id,
|
||||
variant,
|
||||
);
|
||||
await reply
|
||||
.header('content-type', download.contentType)
|
||||
.header('content-length', download.sizeBytes)
|
||||
.header('x-content-type-options', 'nosniff')
|
||||
.header(
|
||||
'content-disposition',
|
||||
`inline; filename*=UTF-8''${encodeURIComponent(download.filename)}`,
|
||||
)
|
||||
.send(download.content);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
import { Module } from '@nestjs/common';
|
||||
import { AttachmentsController } from './attachments.controller';
|
||||
import { AttachmentsService } from './attachments.service';
|
||||
|
||||
@Module({
|
||||
controllers: [AttachmentsController],
|
||||
providers: [AttachmentsService],
|
||||
})
|
||||
export class AttachmentsModule {}
|
||||
@@ -0,0 +1,244 @@
|
||||
import { Injectable, Logger, OnModuleInit } from '@nestjs/common';
|
||||
import { Attachment, AttachmentVariant, Prisma } from '@prisma/client';
|
||||
import { randomUUID } from 'crypto';
|
||||
import { mkdtemp, readdir, rm, stat } from 'fs/promises';
|
||||
import { tmpdir } from 'os';
|
||||
import { join } from 'path';
|
||||
import { Readable } from 'stream';
|
||||
import { assertChatMember } from '../chats/chat-access';
|
||||
import { codePointLength } from '../common/dto/code-point-length.validator';
|
||||
import { ApiError } from '../common/filters/api-error';
|
||||
import { validate as isUuid } from '../common/uuid';
|
||||
import { isDatabaseUnavailable, PrismaService } from '../db/prisma.service';
|
||||
import { PIPELINE_VERSION } from '../jobs/job-message';
|
||||
import { enqueueJob } from '../jobs/job-store';
|
||||
import { S3Service } from '../storage/s3.service';
|
||||
import { assertValidImage, isSupportedContentType } from './image-validation';
|
||||
import { receiveBody } from './upload-body';
|
||||
|
||||
const TEMP_PREFIX = 'messenger-upload-';
|
||||
const STALE_TEMP_MS = 60 * 60 * 1000;
|
||||
const MAX_FILENAME_CODE_POINTS = 255;
|
||||
const FORBIDDEN_FILENAME = /[/\\\p{Cc}\p{Zl}\p{Zp}]/u;
|
||||
const KNOWN_VARIANTS = new Set(['original', 'thumbnail', 'crop', 'optimized']);
|
||||
|
||||
export type AttachmentWithVariants = Attachment & { variants: AttachmentVariant[] };
|
||||
|
||||
export interface DownloadableContent {
|
||||
filename: string;
|
||||
contentType: string;
|
||||
sizeBytes: number;
|
||||
content: Readable;
|
||||
}
|
||||
|
||||
export interface UploadInput {
|
||||
chatId: string;
|
||||
ownerId: string;
|
||||
filename: unknown;
|
||||
contentType: string | undefined;
|
||||
contentLength: string | undefined;
|
||||
body: Readable | undefined;
|
||||
}
|
||||
|
||||
@Injectable()
|
||||
export class AttachmentsService implements OnModuleInit {
|
||||
private readonly logger = new Logger('AttachmentsService');
|
||||
private readonly maxBytes = Number(process.env.MAX_UPLOAD_BYTES ?? 10485760);
|
||||
private readonly maxPixels = Number(process.env.MAX_IMAGE_PIXELS ?? 40000000);
|
||||
private readonly maxConcurrent = Number(process.env.MAX_CONCURRENT_UPLOADS ?? 16);
|
||||
private readonly bodyTimeoutMs = Number(process.env.UPLOAD_BODY_TIMEOUT_MS ?? 60000);
|
||||
private readonly tempRoot = process.env.UPLOAD_TMP_DIR ?? tmpdir();
|
||||
private activeUploads = 0;
|
||||
|
||||
constructor(
|
||||
private readonly prisma: PrismaService,
|
||||
private readonly s3: S3Service,
|
||||
) {}
|
||||
|
||||
async onModuleInit(): Promise<void> {
|
||||
await this.removeStaleTempDirs();
|
||||
}
|
||||
|
||||
async upload(input: UploadInput): Promise<AttachmentWithVariants> {
|
||||
await assertChatMember(this.prisma, input.chatId, input.ownerId);
|
||||
const filename = parseFilename(input.filename);
|
||||
const contentType = parseContentType(input.contentType);
|
||||
if (Number(input.contentLength) > this.maxBytes) {
|
||||
throw new ApiError('payload_too_large', 'attachment exceeds size limit');
|
||||
}
|
||||
if (this.activeUploads >= this.maxConcurrent) {
|
||||
throw new ApiError('unavailable', 'too many concurrent uploads');
|
||||
}
|
||||
this.activeUploads++;
|
||||
const dir = await mkdtemp(join(this.tempRoot, TEMP_PREFIX));
|
||||
try {
|
||||
return await this.store(input, filename, contentType, join(dir, 'body'));
|
||||
} finally {
|
||||
this.activeUploads--;
|
||||
await rm(dir, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
|
||||
async getVisible(
|
||||
attachmentId: string,
|
||||
userId: string,
|
||||
): Promise<AttachmentWithVariants> {
|
||||
if (!isUuid(attachmentId)) {
|
||||
throw new ApiError('invalid_request', 'attachmentId must be a valid UUID');
|
||||
}
|
||||
const row = await this.prisma.attachment.findFirst({
|
||||
where: {
|
||||
id: attachmentId,
|
||||
chat: { members: { some: { userId } } },
|
||||
},
|
||||
include: { variants: { where: { pipelineVersion: PIPELINE_VERSION } } },
|
||||
});
|
||||
if (!row) {
|
||||
throw new ApiError('not_found', 'attachment not found');
|
||||
}
|
||||
return row;
|
||||
}
|
||||
|
||||
async openContent(
|
||||
attachmentId: string,
|
||||
userId: string,
|
||||
variant: unknown,
|
||||
): Promise<DownloadableContent> {
|
||||
const attachment = await this.getVisible(attachmentId, userId);
|
||||
const name = parseVariant(variant);
|
||||
if (attachment.status !== 'ready') {
|
||||
throw new ApiError('conflict', `attachment is ${attachment.status}`);
|
||||
}
|
||||
if (name === 'original') {
|
||||
return {
|
||||
filename: attachment.filename,
|
||||
contentType: attachment.contentType,
|
||||
sizeBytes: attachment.sizeBytes,
|
||||
content: await this.s3.getStream(attachment.objectKey),
|
||||
};
|
||||
}
|
||||
const stored = attachment.variants.find((v) => v.name === name);
|
||||
if (!stored) {
|
||||
throw new ApiError('not_found', 'attachment variant not found');
|
||||
}
|
||||
return {
|
||||
filename: attachment.filename,
|
||||
contentType: stored.contentType,
|
||||
sizeBytes: stored.sizeBytes,
|
||||
content: await this.s3.getStream(stored.objectKey),
|
||||
};
|
||||
}
|
||||
|
||||
private async store(
|
||||
input: UploadInput,
|
||||
filename: string,
|
||||
contentType: string,
|
||||
path: string,
|
||||
): Promise<AttachmentWithVariants> {
|
||||
if (!input.body) {
|
||||
throw new ApiError('invalid_request', 'request body must not be empty');
|
||||
}
|
||||
const { size, sha256 } = await receiveBody(
|
||||
input.body,
|
||||
path,
|
||||
this.maxBytes,
|
||||
this.bodyTimeoutMs,
|
||||
);
|
||||
if (size === 0) {
|
||||
throw new ApiError('invalid_request', 'request body must not be empty');
|
||||
}
|
||||
await assertValidImage(path, size, contentType, this.maxPixels);
|
||||
const id = randomUUID();
|
||||
const stagingKey = `staging/${id}`;
|
||||
await this.s3.putFile({ key: stagingKey, path, size, contentType, sha256 });
|
||||
const now = new Date();
|
||||
try {
|
||||
return await this.prisma.$transaction(async (tx) => {
|
||||
const attachment = await tx.attachment.create({
|
||||
data: {
|
||||
id,
|
||||
chatId: input.chatId,
|
||||
ownerId: input.ownerId,
|
||||
filename,
|
||||
contentType,
|
||||
sizeBytes: size,
|
||||
sha256,
|
||||
objectKey: `attachments/${id}/original`,
|
||||
stagingKey,
|
||||
status: 'queued',
|
||||
createdAt: now,
|
||||
updatedAt: now,
|
||||
},
|
||||
});
|
||||
await enqueueJob(tx, id, now);
|
||||
return { ...attachment, variants: [] };
|
||||
});
|
||||
} catch (error) {
|
||||
if (isDefiniteRejection(error)) {
|
||||
await this.s3.delete(stagingKey).catch(() => {
|
||||
this.logger.warn(`compensation failed for ${stagingKey}`);
|
||||
});
|
||||
}
|
||||
if (error instanceof Prisma.PrismaClientUnknownRequestError) {
|
||||
this.logger.error(`metadata insert failed for ${stagingKey}`);
|
||||
throw new ApiError('unavailable', 'database unavailable');
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
private async removeStaleTempDirs(): Promise<void> {
|
||||
const root = this.tempRoot;
|
||||
const names = await readdir(root);
|
||||
for (const name of names.filter((n) => n.startsWith(TEMP_PREFIX))) {
|
||||
const path = join(root, name);
|
||||
const info = await stat(path).catch(() => null);
|
||||
if (info && Date.now() - info.mtimeMs > STALE_TEMP_MS) {
|
||||
await rm(path, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function parseVariant(raw: unknown): string {
|
||||
if (raw === undefined) {
|
||||
return 'original';
|
||||
}
|
||||
if (typeof raw === 'string' && KNOWN_VARIANTS.has(raw)) {
|
||||
return raw;
|
||||
}
|
||||
throw new ApiError('invalid_request', 'unknown attachment variant');
|
||||
}
|
||||
|
||||
function isDefiniteRejection(error: unknown): boolean {
|
||||
return (
|
||||
error instanceof Prisma.PrismaClientKnownRequestError &&
|
||||
!isDatabaseUnavailable(error)
|
||||
);
|
||||
}
|
||||
|
||||
function parseFilename(raw: unknown): string {
|
||||
if (
|
||||
typeof raw !== 'string' ||
|
||||
raw.trim() === '' ||
|
||||
codePointLength(raw) > MAX_FILENAME_CODE_POINTS ||
|
||||
FORBIDDEN_FILENAME.test(raw)
|
||||
) {
|
||||
throw new ApiError(
|
||||
'invalid_request',
|
||||
'filename must be 1-255 characters without path separators or control characters',
|
||||
);
|
||||
}
|
||||
return raw;
|
||||
}
|
||||
|
||||
function parseContentType(raw: string | undefined): string {
|
||||
const contentType = raw?.split(';')[0].trim().toLowerCase() ?? '';
|
||||
if (!isSupportedContentType(contentType)) {
|
||||
throw new ApiError(
|
||||
'unsupported_media_type',
|
||||
'Content-Type must be image/png or image/jpeg',
|
||||
);
|
||||
}
|
||||
return contentType;
|
||||
}
|
||||
@@ -0,0 +1,55 @@
|
||||
import { Attachment, AttachmentVariant } from '@prisma/client';
|
||||
|
||||
const VARIANT_ORDER = ['thumbnail', 'crop', 'optimized'];
|
||||
|
||||
export interface VariantResponse {
|
||||
name: string;
|
||||
content_type: string;
|
||||
size_bytes: number;
|
||||
width: number;
|
||||
height: number;
|
||||
sha256: string;
|
||||
}
|
||||
|
||||
export interface AttachmentResponse {
|
||||
id: string;
|
||||
chat_id: string;
|
||||
owner_id: string;
|
||||
filename: string;
|
||||
content_type: string;
|
||||
size_bytes: number;
|
||||
sha256: string;
|
||||
status: string;
|
||||
variants: VariantResponse[];
|
||||
error_code: string | null;
|
||||
created_at: string;
|
||||
updated_at: string;
|
||||
}
|
||||
|
||||
export function toAttachmentResponse(
|
||||
row: Attachment & { variants: AttachmentVariant[] },
|
||||
): AttachmentResponse {
|
||||
const variants = row.status === 'ready' ? [...row.variants] : [];
|
||||
variants.sort((a, b) => VARIANT_ORDER.indexOf(a.name) - VARIANT_ORDER.indexOf(b.name));
|
||||
return {
|
||||
id: row.id,
|
||||
chat_id: row.chatId,
|
||||
owner_id: row.ownerId,
|
||||
filename: row.filename,
|
||||
content_type: row.contentType,
|
||||
size_bytes: row.sizeBytes,
|
||||
sha256: row.sha256,
|
||||
status: row.status,
|
||||
variants: variants.map((variant) => ({
|
||||
name: variant.name,
|
||||
content_type: variant.contentType,
|
||||
size_bytes: variant.sizeBytes,
|
||||
width: variant.width,
|
||||
height: variant.height,
|
||||
sha256: variant.sha256,
|
||||
})),
|
||||
error_code: row.errorCode,
|
||||
created_at: row.createdAt.toISOString(),
|
||||
updated_at: row.updatedAt.toISOString(),
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,105 @@
|
||||
import { open } from 'fs/promises';
|
||||
import sharp from 'sharp';
|
||||
import { ApiError } from '../common/filters/api-error';
|
||||
|
||||
sharp.cache(false);
|
||||
sharp.concurrency(1);
|
||||
|
||||
const PNG_SIGNATURE = Buffer.from('89504e470d0a1a0a', 'hex');
|
||||
const JPEG_SIGNATURE = Buffer.from('ffd8ff', 'hex');
|
||||
const PNG_END_CHUNK = Buffer.from('0000000049454e44ae426082', 'hex');
|
||||
const PNG_TAIL_BYTES = 64 * 1024;
|
||||
const MAX_CONCURRENT_DECODES = 2;
|
||||
|
||||
const FORMAT_BY_CONTENT_TYPE: Record<string, string> = {
|
||||
'image/png': 'png',
|
||||
'image/jpeg': 'jpeg',
|
||||
};
|
||||
|
||||
let activeDecodes = 0;
|
||||
const decodeQueue: (() => void)[] = [];
|
||||
|
||||
export function isSupportedContentType(contentType: string): boolean {
|
||||
return contentType in FORMAT_BY_CONTENT_TYPE;
|
||||
}
|
||||
|
||||
export async function assertValidImage(
|
||||
path: string,
|
||||
size: number,
|
||||
contentType: string,
|
||||
maxPixels: number,
|
||||
): Promise<void> {
|
||||
const invalid = new ApiError(
|
||||
'unsupported_media_type',
|
||||
'file is not a valid image of the declared type',
|
||||
);
|
||||
const format = FORMAT_BY_CONTENT_TYPE[contentType];
|
||||
if (!(await hasExpectedFraming(path, size, format))) {
|
||||
throw invalid;
|
||||
}
|
||||
const { width, height, format: actual } = await sharp(path, {
|
||||
limitInputPixels: false,
|
||||
})
|
||||
.metadata()
|
||||
.catch(() => {
|
||||
throw invalid;
|
||||
});
|
||||
if (!width || !height || actual !== format) {
|
||||
throw invalid;
|
||||
}
|
||||
if (width * height > maxPixels) {
|
||||
throw new ApiError('payload_too_large', 'image exceeds pixel limit');
|
||||
}
|
||||
await acquireDecodeSlot();
|
||||
try {
|
||||
await sharp(path, {
|
||||
failOn: 'warning',
|
||||
sequentialRead: true,
|
||||
limitInputPixels: maxPixels,
|
||||
}).stats();
|
||||
} catch {
|
||||
throw invalid;
|
||||
} finally {
|
||||
releaseDecodeSlot();
|
||||
}
|
||||
}
|
||||
|
||||
async function hasExpectedFraming(
|
||||
path: string,
|
||||
size: number,
|
||||
format: string,
|
||||
): Promise<boolean> {
|
||||
const file = await open(path, 'r');
|
||||
try {
|
||||
const head = Buffer.alloc(PNG_SIGNATURE.length);
|
||||
await file.read(head, 0, head.length, 0);
|
||||
if (format === 'jpeg') {
|
||||
return head.subarray(0, JPEG_SIGNATURE.length).equals(JPEG_SIGNATURE);
|
||||
}
|
||||
if (!head.equals(PNG_SIGNATURE)) {
|
||||
return false;
|
||||
}
|
||||
const tail = Buffer.alloc(Math.min(size, PNG_TAIL_BYTES));
|
||||
await file.read(tail, 0, tail.length, size - tail.length);
|
||||
return tail.includes(PNG_END_CHUNK);
|
||||
} finally {
|
||||
await file.close();
|
||||
}
|
||||
}
|
||||
|
||||
function acquireDecodeSlot(): Promise<void> {
|
||||
if (activeDecodes < MAX_CONCURRENT_DECODES) {
|
||||
activeDecodes++;
|
||||
return Promise.resolve();
|
||||
}
|
||||
return new Promise((resolve) => decodeQueue.push(resolve));
|
||||
}
|
||||
|
||||
function releaseDecodeSlot(): void {
|
||||
const next = decodeQueue.shift();
|
||||
if (next) {
|
||||
next();
|
||||
} else {
|
||||
activeDecodes--;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,163 @@
|
||||
import { PrismaClient } from '@prisma/client';
|
||||
import { S3Service } from '../storage/s3.service';
|
||||
|
||||
const MAX_REPORTED_KEYS = 20;
|
||||
const DEFAULT_PREFIXES = ['attachments/', 'staging/'];
|
||||
|
||||
type ReconcilePrisma = Pick<PrismaClient, 'attachment' | 'attachmentVariant'>;
|
||||
|
||||
export interface ReconcileOptions {
|
||||
apply: boolean;
|
||||
graceMs: number;
|
||||
stagingRetentionMs?: number;
|
||||
prefix?: string;
|
||||
now?: Date;
|
||||
pageSize?: number;
|
||||
}
|
||||
|
||||
export interface ReconcileReport {
|
||||
mode: 'apply' | 'dry-run';
|
||||
grace_seconds: number;
|
||||
staging_expired: number;
|
||||
scanned: number;
|
||||
fresh_skipped: number;
|
||||
referenced: number;
|
||||
orphans: number;
|
||||
deleted: number;
|
||||
failed: number;
|
||||
orphan_keys: string[];
|
||||
duration_ms: number;
|
||||
}
|
||||
|
||||
export async function reconcile(
|
||||
prisma: ReconcilePrisma,
|
||||
s3: S3Service,
|
||||
options: ReconcileOptions,
|
||||
): Promise<ReconcileReport> {
|
||||
const started = Date.now();
|
||||
const cutoff = (options.now ?? new Date()).getTime() - options.graceMs;
|
||||
const report: ReconcileReport = {
|
||||
mode: options.apply ? 'apply' : 'dry-run',
|
||||
grace_seconds: options.graceMs / 1000,
|
||||
staging_expired: 0,
|
||||
scanned: 0,
|
||||
fresh_skipped: 0,
|
||||
referenced: 0,
|
||||
orphans: 0,
|
||||
deleted: 0,
|
||||
failed: 0,
|
||||
orphan_keys: [],
|
||||
duration_ms: 0,
|
||||
};
|
||||
if (options.stagingRetentionMs !== undefined) {
|
||||
report.staging_expired = await expireFailedStaging(
|
||||
prisma,
|
||||
s3,
|
||||
(options.now ?? new Date()).getTime() - options.stagingRetentionMs,
|
||||
options.apply,
|
||||
options.prefix,
|
||||
);
|
||||
}
|
||||
for (const prefix of options.prefix ? [options.prefix] : DEFAULT_PREFIXES) {
|
||||
await reconcilePrefix(prisma, s3, prefix, cutoff, options, report);
|
||||
}
|
||||
report.duration_ms = Date.now() - started;
|
||||
return report;
|
||||
}
|
||||
|
||||
async function expireFailedStaging(
|
||||
prisma: ReconcilePrisma,
|
||||
s3: S3Service,
|
||||
before: number,
|
||||
apply: boolean,
|
||||
prefix?: string,
|
||||
): Promise<number> {
|
||||
const rows = await prisma.attachment.findMany({
|
||||
where: {
|
||||
stagingKey: { not: null, startsWith: prefix },
|
||||
OR: [{ status: 'ready' }, { status: 'failed', updatedAt: { lt: new Date(before) } }],
|
||||
},
|
||||
select: { id: true, stagingKey: true },
|
||||
take: 500,
|
||||
});
|
||||
if (!apply) {
|
||||
return rows.length;
|
||||
}
|
||||
let expired = 0;
|
||||
for (const row of rows) {
|
||||
const released = await prisma.attachment.updateMany({
|
||||
where: {
|
||||
id: row.id,
|
||||
stagingKey: row.stagingKey,
|
||||
OR: [{ status: 'ready' }, { status: 'failed', updatedAt: { lt: new Date(before) } }],
|
||||
},
|
||||
data: { stagingKey: null },
|
||||
});
|
||||
if (released.count === 0) {
|
||||
continue;
|
||||
}
|
||||
await s3.delete(row.stagingKey!).then(
|
||||
() => expired++,
|
||||
() => undefined,
|
||||
);
|
||||
}
|
||||
return expired;
|
||||
}
|
||||
|
||||
function attachmentIdOf(key: string): string | null {
|
||||
const match = /^attachments\/([0-9a-f-]{36})(\/|$)/.exec(key);
|
||||
return match ? match[1] : null;
|
||||
}
|
||||
|
||||
async function reconcilePrefix(
|
||||
prisma: ReconcilePrisma,
|
||||
s3: S3Service,
|
||||
prefix: string,
|
||||
cutoff: number,
|
||||
options: ReconcileOptions,
|
||||
report: ReconcileReport,
|
||||
): Promise<void> {
|
||||
let token: string | undefined;
|
||||
do {
|
||||
const page = await s3.listObjects(prefix, options.pageSize ?? 500, token);
|
||||
token = page.nextToken;
|
||||
report.scanned += page.objects.length;
|
||||
const old = page.objects.filter((object) => object.lastModified.getTime() < cutoff);
|
||||
report.fresh_skipped += page.objects.length - old.length;
|
||||
const keys = old.map((object) => object.key);
|
||||
const rows = await prisma.attachment.findMany({
|
||||
where: { OR: [{ objectKey: { in: keys } }, { stagingKey: { in: keys } }] },
|
||||
select: { objectKey: true, stagingKey: true },
|
||||
});
|
||||
const variants = await prisma.attachmentVariant.findMany({
|
||||
where: { objectKey: { in: keys } },
|
||||
select: { objectKey: true },
|
||||
});
|
||||
const ids = [...new Set(keys.map(attachmentIdOf).filter((id): id is string => id !== null))];
|
||||
const owners = ids.length
|
||||
? await prisma.attachment.findMany({ where: { id: { in: ids } }, select: { id: true } })
|
||||
: [];
|
||||
const owned = new Set(owners.map((row) => row.id));
|
||||
const known = new Set([
|
||||
...rows.flatMap((row) => [row.objectKey, row.stagingKey]),
|
||||
...variants.map((row) => row.objectKey),
|
||||
]);
|
||||
for (const { key } of old) {
|
||||
const owner = attachmentIdOf(key);
|
||||
if (known.has(key) || (owner !== null && owned.has(owner))) {
|
||||
report.referenced++;
|
||||
continue;
|
||||
}
|
||||
report.orphans++;
|
||||
if (report.orphan_keys.length < MAX_REPORTED_KEYS) {
|
||||
report.orphan_keys.push(key);
|
||||
}
|
||||
if (options.apply) {
|
||||
await s3.delete(key).then(
|
||||
() => report.deleted++,
|
||||
() => report.failed++,
|
||||
);
|
||||
}
|
||||
}
|
||||
} while (token);
|
||||
}
|
||||
@@ -0,0 +1,98 @@
|
||||
import { createHash } from 'crypto';
|
||||
import { FastifyInstance, FastifyRequest } from 'fastify';
|
||||
import { createWriteStream } from 'fs';
|
||||
import { Readable } from 'stream';
|
||||
import { ApiError } from '../common/filters/api-error';
|
||||
|
||||
const UPLOAD_ROUTE = '/api/v1/chats/:chatId/attachments';
|
||||
const PASS_THROUGH_TYPE = 'application/x-messenger-upload';
|
||||
|
||||
declare module 'fastify' {
|
||||
interface FastifyRequest {
|
||||
uploadContentType?: string;
|
||||
}
|
||||
}
|
||||
|
||||
export function registerUploadBody(instance: FastifyInstance): void {
|
||||
instance.addContentTypeParser(PASS_THROUGH_TYPE, (_request, payload, done) => {
|
||||
done(null, payload);
|
||||
});
|
||||
instance.addHook('onRequest', (request: FastifyRequest, _reply, done) => {
|
||||
if (request.method === 'POST' && request.routeOptions.url === UPLOAD_ROUTE) {
|
||||
request.uploadContentType = request.headers['content-type'];
|
||||
request.headers['content-type'] = PASS_THROUGH_TYPE;
|
||||
}
|
||||
done();
|
||||
});
|
||||
}
|
||||
|
||||
export interface ReceivedBody {
|
||||
size: number;
|
||||
sha256: string;
|
||||
}
|
||||
|
||||
export function receiveBody(
|
||||
body: Readable,
|
||||
path: string,
|
||||
maxBytes: number,
|
||||
timeoutMs: number,
|
||||
): Promise<ReceivedBody> {
|
||||
return new Promise((resolve, reject) => {
|
||||
const hash = createHash('sha256');
|
||||
const out = createWriteStream(path, { flags: 'wx', mode: 0o600 });
|
||||
let size = 0;
|
||||
let settled = false;
|
||||
|
||||
const timer = setTimeout(() => body.destroy(), timeoutMs);
|
||||
const settle = (done: () => void): void => {
|
||||
if (settled) {
|
||||
return;
|
||||
}
|
||||
settled = true;
|
||||
clearTimeout(timer);
|
||||
body.off('data', onData);
|
||||
body.off('end', onEnd);
|
||||
body.off('close', onClose);
|
||||
done();
|
||||
};
|
||||
const fail = (error: ApiError): void => {
|
||||
settle(() => {
|
||||
out.destroy();
|
||||
reject(error);
|
||||
});
|
||||
};
|
||||
const onData = (chunk: Buffer): void => {
|
||||
size += chunk.length;
|
||||
if (size > maxBytes) {
|
||||
fail(new ApiError('payload_too_large', 'attachment exceeds size limit'));
|
||||
return;
|
||||
}
|
||||
hash.update(chunk);
|
||||
if (!out.write(chunk)) {
|
||||
body.pause();
|
||||
out.once('drain', () => {
|
||||
if (!settled) {
|
||||
body.resume();
|
||||
}
|
||||
});
|
||||
}
|
||||
};
|
||||
const onEnd = (): void => {
|
||||
settle(() => {
|
||||
out.end(() =>
|
||||
resolve({ size, sha256: hash.digest('hex') }),
|
||||
);
|
||||
});
|
||||
};
|
||||
const onClose = (): void => {
|
||||
fail(new ApiError('invalid_request', 'upload was interrupted'));
|
||||
};
|
||||
|
||||
out.on('error', () => {
|
||||
fail(new ApiError('unavailable', 'temporary storage unavailable'));
|
||||
});
|
||||
body.on('data', onData);
|
||||
body.on('end', onEnd);
|
||||
body.on('close', onClose);
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,51 @@
|
||||
import {
|
||||
Controller,
|
||||
Delete,
|
||||
Headers,
|
||||
HttpCode,
|
||||
Post,
|
||||
Req,
|
||||
UseGuards,
|
||||
} from '@nestjs/common';
|
||||
import { FastifyRequest } from 'fastify';
|
||||
import { ApiError } from '../common/filters/api-error';
|
||||
import { toUserResponse, UserResponse } from '../users/dto/user.response';
|
||||
import { AuthGuard } from './auth.guard';
|
||||
import { AuthService } from './auth.service';
|
||||
|
||||
interface SessionResponse {
|
||||
token: string;
|
||||
token_type: 'Bearer';
|
||||
expires_at: string;
|
||||
user: UserResponse;
|
||||
}
|
||||
|
||||
@Controller('api/v1/auth/sessions')
|
||||
export class AuthController {
|
||||
constructor(private readonly auth: AuthService) {}
|
||||
|
||||
@Post()
|
||||
@HttpCode(201)
|
||||
async create(
|
||||
@Headers('authorization') authorization: string | undefined,
|
||||
@Req() request: FastifyRequest,
|
||||
): Promise<SessionResponse> {
|
||||
if (request.body !== undefined) {
|
||||
throw new ApiError('invalid_request', 'request body is not allowed');
|
||||
}
|
||||
const session = await this.auth.login(authorization);
|
||||
return {
|
||||
token: session.token,
|
||||
token_type: 'Bearer',
|
||||
expires_at: session.expiresAt.toISOString(),
|
||||
user: toUserResponse(session.user),
|
||||
};
|
||||
}
|
||||
|
||||
@Delete('current')
|
||||
@HttpCode(204)
|
||||
@UseGuards(AuthGuard)
|
||||
async revoke(@Req() request: FastifyRequest): Promise<void> {
|
||||
await this.auth.revoke(request.sessionTokenHash!);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,25 @@
|
||||
import { CanActivate, ExecutionContext, Injectable } from '@nestjs/common';
|
||||
import { FastifyRequest } from 'fastify';
|
||||
import { ApiError } from '../common/filters/api-error';
|
||||
import { AuthService } from './auth.service';
|
||||
|
||||
const BEARER_CHALLENGE = 'Bearer realm="messenger"';
|
||||
|
||||
@Injectable()
|
||||
export class AuthGuard implements CanActivate {
|
||||
constructor(private readonly auth: AuthService) {}
|
||||
|
||||
async canActivate(context: ExecutionContext): Promise<boolean> {
|
||||
const request = context.switchToHttp().getRequest<FastifyRequest>();
|
||||
const match = /^Bearer (\S+)$/i.exec(request.headers.authorization ?? '');
|
||||
const session = match ? await this.auth.resolve(match[1]) : null;
|
||||
if (!session) {
|
||||
throw new ApiError('unauthorized', 'missing or invalid session', {
|
||||
'WWW-Authenticate': BEARER_CHALLENGE,
|
||||
});
|
||||
}
|
||||
request.currentUser = session.user;
|
||||
request.sessionTokenHash = session.tokenHash;
|
||||
return true;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,12 @@
|
||||
import { Global, Module } from '@nestjs/common';
|
||||
import { AuthController } from './auth.controller';
|
||||
import { AuthGuard } from './auth.guard';
|
||||
import { AuthService } from './auth.service';
|
||||
|
||||
@Global()
|
||||
@Module({
|
||||
controllers: [AuthController],
|
||||
providers: [AuthService, AuthGuard],
|
||||
exports: [AuthService, AuthGuard],
|
||||
})
|
||||
export class AuthModule {}
|
||||
@@ -0,0 +1,155 @@
|
||||
import { Injectable } from '@nestjs/common';
|
||||
import { createHash, randomBytes } from 'crypto';
|
||||
import { CacheService } from '../cache/cache.service';
|
||||
import { codePointLength } from '../common/dto/code-point-length.validator';
|
||||
import { User } from '../common/domain';
|
||||
import { ApiError } from '../common/filters/api-error';
|
||||
import { PrismaService } from '../db/prisma.service';
|
||||
import { toUser } from '../users/users.service';
|
||||
import { verifyPassword } from './password';
|
||||
|
||||
const BASIC_CHALLENGE = 'Basic realm="messenger", charset="UTF-8"';
|
||||
const USERNAME_PATTERN = /^[a-z0-9_]{3,32}$/;
|
||||
const CACHE_TTL_MS = 60_000;
|
||||
const REVOKED = 'revoked';
|
||||
|
||||
interface CachedSession {
|
||||
user: User;
|
||||
expiresAt: number;
|
||||
}
|
||||
|
||||
export interface CreatedSession {
|
||||
token: string;
|
||||
expiresAt: Date;
|
||||
user: User;
|
||||
}
|
||||
|
||||
export interface ResolvedSession {
|
||||
tokenHash: string;
|
||||
user: User;
|
||||
}
|
||||
|
||||
@Injectable()
|
||||
export class AuthService {
|
||||
private readonly ttlSeconds = readSessionTtl();
|
||||
|
||||
constructor(
|
||||
private readonly prisma: PrismaService,
|
||||
private readonly cache: CacheService,
|
||||
) {}
|
||||
|
||||
async login(authorization: string | undefined): Promise<CreatedSession> {
|
||||
const credentials = parseBasic(authorization);
|
||||
if (!credentials) {
|
||||
throw invalidCredentials();
|
||||
}
|
||||
const row = await this.prisma.user.findUnique({
|
||||
where: { username: credentials.username },
|
||||
});
|
||||
const valid = await verifyPassword(
|
||||
row?.passwordHash ?? null,
|
||||
credentials.password,
|
||||
);
|
||||
if (!row || !valid) {
|
||||
throw invalidCredentials();
|
||||
}
|
||||
const token = randomBytes(32).toString('base64url');
|
||||
const expiresAt = new Date(Date.now() + this.ttlSeconds * 1000);
|
||||
await this.prisma.session.create({
|
||||
data: { tokenHash: hashToken(token), userId: row.id, expiresAt },
|
||||
});
|
||||
return { token, expiresAt, user: toUser(row) };
|
||||
}
|
||||
|
||||
async resolve(token: string): Promise<ResolvedSession | null> {
|
||||
const tokenHash = hashToken(token);
|
||||
const key = cacheKey(tokenHash);
|
||||
const cached = await this.cache.get(key);
|
||||
if (cached === REVOKED) {
|
||||
return null;
|
||||
}
|
||||
if (cached) {
|
||||
const entry = JSON.parse(cached) as CachedSession;
|
||||
if (entry.expiresAt > Date.now()) {
|
||||
return { tokenHash, user: entry.user };
|
||||
}
|
||||
}
|
||||
const session = await this.prisma.session.findUnique({
|
||||
where: { tokenHash },
|
||||
include: { user: true },
|
||||
});
|
||||
const remainingMs = session
|
||||
? session.expiresAt.getTime() - Date.now()
|
||||
: 0;
|
||||
if (!session || remainingMs <= 0) {
|
||||
return null;
|
||||
}
|
||||
const user = toUser(session.user);
|
||||
const entry: CachedSession = {
|
||||
user,
|
||||
expiresAt: session.expiresAt.getTime(),
|
||||
};
|
||||
await this.cache.set(
|
||||
key,
|
||||
JSON.stringify(entry),
|
||||
Math.min(remainingMs, CACHE_TTL_MS),
|
||||
true,
|
||||
);
|
||||
return { tokenHash, user };
|
||||
}
|
||||
|
||||
async revoke(tokenHash: string): Promise<void> {
|
||||
const marked = await this.cache.setStrict(
|
||||
cacheKey(tokenHash),
|
||||
REVOKED,
|
||||
CACHE_TTL_MS,
|
||||
);
|
||||
if (!marked) {
|
||||
throw new ApiError('unavailable', 'session cache unavailable');
|
||||
}
|
||||
await this.prisma.session.deleteMany({ where: { tokenHash } });
|
||||
}
|
||||
}
|
||||
|
||||
function cacheKey(tokenHash: string): string {
|
||||
return `session:v1:${tokenHash}`;
|
||||
}
|
||||
|
||||
function parseBasic(
|
||||
header: string | undefined,
|
||||
): { username: string; password: string } | null {
|
||||
const match = /^Basic ([A-Za-z0-9+/]+={0,2})$/i.exec(header ?? '');
|
||||
if (!match) {
|
||||
return null;
|
||||
}
|
||||
const decoded = Buffer.from(match[1], 'base64').toString('utf8');
|
||||
const separator = decoded.indexOf(':');
|
||||
if (separator < 0) {
|
||||
return null;
|
||||
}
|
||||
const username = decoded.slice(0, separator);
|
||||
const password = decoded.slice(separator + 1);
|
||||
const length = codePointLength(password);
|
||||
if (!USERNAME_PATTERN.test(username) || length < 12 || length > 128) {
|
||||
return null;
|
||||
}
|
||||
return { username, password };
|
||||
}
|
||||
|
||||
function hashToken(token: string): string {
|
||||
return createHash('sha256').update(token).digest('hex');
|
||||
}
|
||||
|
||||
function invalidCredentials(): ApiError {
|
||||
return new ApiError('unauthorized', 'invalid credentials', {
|
||||
'WWW-Authenticate': BASIC_CHALLENGE,
|
||||
});
|
||||
}
|
||||
|
||||
function readSessionTtl(): number {
|
||||
const ttl = Number(process.env.SESSION_TTL_SECONDS ?? 3600);
|
||||
if (!Number.isInteger(ttl) || ttl <= 0) {
|
||||
throw new Error('SESSION_TTL_SECONDS must be a positive integer');
|
||||
}
|
||||
return ttl;
|
||||
}
|
||||
@@ -0,0 +1,20 @@
|
||||
import { createParamDecorator, ExecutionContext } from '@nestjs/common';
|
||||
import { FastifyRequest } from 'fastify';
|
||||
import { User } from '../common/domain';
|
||||
|
||||
declare module 'fastify' {
|
||||
interface FastifyRequest {
|
||||
currentUser?: User;
|
||||
sessionTokenHash?: string;
|
||||
}
|
||||
}
|
||||
|
||||
export const CurrentUser = createParamDecorator(
|
||||
(_data: unknown, ctx: ExecutionContext): User => {
|
||||
const request = ctx.switchToHttp().getRequest<FastifyRequest>();
|
||||
if (!request.currentUser) {
|
||||
throw new Error('CurrentUser used without AuthGuard');
|
||||
}
|
||||
return request.currentUser;
|
||||
},
|
||||
);
|
||||
@@ -0,0 +1,17 @@
|
||||
import { argon2id, hash, verify } from 'argon2';
|
||||
|
||||
const OPTIONS = { type: argon2id, memoryCost: 19456, timeCost: 2, parallelism: 1 };
|
||||
|
||||
const dummyHash = hash('timing-equalizer-password', OPTIONS);
|
||||
|
||||
export function hashPassword(password: string): Promise<string> {
|
||||
return hash(password, OPTIONS);
|
||||
}
|
||||
|
||||
export async function verifyPassword(
|
||||
passwordHash: string | null,
|
||||
password: string,
|
||||
): Promise<boolean> {
|
||||
const valid = await verify(passwordHash ?? (await dummyHash), password);
|
||||
return passwordHash !== null && valid;
|
||||
}
|
||||
@@ -0,0 +1,57 @@
|
||||
import { Logger } from '@nestjs/common';
|
||||
import amqp, { Channel, Connection } from 'amqplib';
|
||||
import { readFileSync } from 'fs';
|
||||
|
||||
export const JOB_QUEUE = 'attachment.jobs';
|
||||
export const DEAD_LETTER_QUEUE = 'attachment.jobs.dlq';
|
||||
const DELIVERY_LIMIT = 20;
|
||||
const RECONNECT_MAX_MS = 10000;
|
||||
|
||||
export function brokerUrl(): string {
|
||||
const file = process.env.BROKER_URL_FILE;
|
||||
const url = file ? readFileSync(file, 'utf8').trim() : process.env.BROKER_URL;
|
||||
if (!url) {
|
||||
throw new Error('BROKER_URL_FILE or BROKER_URL must be set');
|
||||
}
|
||||
return url;
|
||||
}
|
||||
|
||||
export async function assertTopology(channel: Channel): Promise<void> {
|
||||
await channel.assertQueue(DEAD_LETTER_QUEUE, {
|
||||
durable: true,
|
||||
arguments: { 'x-queue-type': 'quorum' },
|
||||
});
|
||||
await channel.assertQueue(JOB_QUEUE, {
|
||||
durable: true,
|
||||
arguments: {
|
||||
'x-queue-type': 'quorum',
|
||||
'x-delivery-limit': DELIVERY_LIMIT,
|
||||
'x-dead-letter-exchange': '',
|
||||
'x-dead-letter-routing-key': DEAD_LETTER_QUEUE,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
export async function connectWithRetry(
|
||||
logger: Logger,
|
||||
stopping: () => boolean,
|
||||
): Promise<Connection | null> {
|
||||
let delay = 500;
|
||||
while (!stopping()) {
|
||||
try {
|
||||
const connection = await amqp.connect(brokerUrl(), { timeout: 5000 });
|
||||
logger.log('connected to broker');
|
||||
return connection;
|
||||
} catch (error) {
|
||||
const reason = error instanceof Error ? error.message.split(' ')[0] : 'unknown';
|
||||
logger.warn(`broker unavailable (${reason}), retry in ${delay} ms`);
|
||||
await sleep(delay);
|
||||
delay = Math.min(RECONNECT_MAX_MS, delay * 2);
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
export function sleep(ms: number): Promise<void> {
|
||||
return new Promise((resolve) => setTimeout(resolve, ms));
|
||||
}
|
||||
Vendored
+9
@@ -0,0 +1,9 @@
|
||||
import { Global, Module } from '@nestjs/common';
|
||||
import { CacheService } from './cache.service';
|
||||
|
||||
@Global()
|
||||
@Module({
|
||||
providers: [CacheService],
|
||||
exports: [CacheService],
|
||||
})
|
||||
export class CacheModule {}
|
||||
Vendored
+98
@@ -0,0 +1,98 @@
|
||||
import { Injectable, Logger, OnModuleDestroy } from '@nestjs/common';
|
||||
import { readFileSync } from 'fs';
|
||||
import Redis from 'ioredis';
|
||||
|
||||
@Injectable()
|
||||
export class CacheService implements OnModuleDestroy {
|
||||
private readonly logger = new Logger('CacheService');
|
||||
private readonly client: Redis | null;
|
||||
private trusted = false;
|
||||
|
||||
constructor() {
|
||||
const url = readCacheUrl();
|
||||
this.client = url
|
||||
? new Redis(url, {
|
||||
enableOfflineQueue: false,
|
||||
maxRetriesPerRequest: 0,
|
||||
connectTimeout: 1000,
|
||||
commandTimeout: 200,
|
||||
})
|
||||
: null;
|
||||
this.client?.on('error', (error) =>
|
||||
this.logger.warn(`redis unavailable: ${error.message}`),
|
||||
);
|
||||
this.client?.on('close', () => {
|
||||
this.trusted = false;
|
||||
});
|
||||
this.client?.on('ready', () => {
|
||||
this.client!.flushdb().then(
|
||||
() => {
|
||||
this.trusted = true;
|
||||
},
|
||||
() => this.distrust(),
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
async get(key: string): Promise<string | null> {
|
||||
if (!this.client || !this.trusted) {
|
||||
return null;
|
||||
}
|
||||
try {
|
||||
return await this.client.get(key);
|
||||
} catch {
|
||||
this.distrust();
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
async set(
|
||||
key: string,
|
||||
value: string,
|
||||
ttlMs: number,
|
||||
onlyIfAbsent = false,
|
||||
): Promise<void> {
|
||||
if (!this.client || !this.trusted) {
|
||||
return;
|
||||
}
|
||||
try {
|
||||
if (onlyIfAbsent) {
|
||||
await this.client.set(key, value, 'PX', ttlMs, 'NX');
|
||||
} else {
|
||||
await this.client.set(key, value, 'PX', ttlMs);
|
||||
}
|
||||
} catch {
|
||||
this.distrust();
|
||||
}
|
||||
}
|
||||
|
||||
async setStrict(key: string, value: string, ttlMs: number): Promise<boolean> {
|
||||
if (!this.client) {
|
||||
return true;
|
||||
}
|
||||
if (!this.trusted) {
|
||||
return false;
|
||||
}
|
||||
try {
|
||||
await this.client.set(key, value, 'PX', ttlMs);
|
||||
return true;
|
||||
} catch {
|
||||
this.distrust();
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
onModuleDestroy(): void {
|
||||
this.client?.disconnect();
|
||||
}
|
||||
|
||||
private distrust(): void {
|
||||
this.trusted = false;
|
||||
this.client?.disconnect(true);
|
||||
}
|
||||
}
|
||||
|
||||
function readCacheUrl(): string | undefined {
|
||||
const file = process.env.CACHE_URL_FILE;
|
||||
return file ? readFileSync(file, 'utf8').trim() : process.env.CACHE_URL;
|
||||
}
|
||||
@@ -0,0 +1,19 @@
|
||||
import { ApiError } from '../common/filters/api-error';
|
||||
import { validate as isUuid } from '../common/uuid';
|
||||
import { PrismaService } from '../db/prisma.service';
|
||||
|
||||
export async function assertChatMember(
|
||||
prisma: PrismaService,
|
||||
chatId: string,
|
||||
userId: string,
|
||||
): Promise<void> {
|
||||
if (!isUuid(chatId)) {
|
||||
throw new ApiError('invalid_request', 'chatId must be a valid UUID');
|
||||
}
|
||||
const member = await prisma.chatMember.findUnique({
|
||||
where: { chatId_userId: { chatId, userId } },
|
||||
});
|
||||
if (!member) {
|
||||
throw new ApiError('not_found', 'chat not found');
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,57 @@
|
||||
import {
|
||||
Body,
|
||||
Controller,
|
||||
Get,
|
||||
HttpCode,
|
||||
Param,
|
||||
Post,
|
||||
Query,
|
||||
UseGuards,
|
||||
} from '@nestjs/common';
|
||||
import { AuthGuard } from '../auth/auth.guard';
|
||||
import { CurrentUser } from '../auth/current-user.decorator';
|
||||
import { parseLimit, rawCursorParam } from '../common/dto/pagination-query.dto';
|
||||
import { User } from '../common/domain';
|
||||
import { ChatsService } from './chats.service';
|
||||
import {
|
||||
ChatPageResponse,
|
||||
ChatResponse,
|
||||
toChatResponse,
|
||||
} from './dto/chat.response';
|
||||
import { CreateChatDto } from './dto/create-chat.dto';
|
||||
|
||||
@Controller('api/v1/chats')
|
||||
@UseGuards(AuthGuard)
|
||||
export class ChatsController {
|
||||
constructor(private readonly chats: ChatsService) {}
|
||||
|
||||
@Post()
|
||||
@HttpCode(201)
|
||||
async create(
|
||||
@Body() dto: CreateChatDto,
|
||||
@CurrentUser() user: User,
|
||||
): Promise<ChatResponse> {
|
||||
const chat = await this.chats.create(dto.title, dto.member_ids, user.id);
|
||||
return toChatResponse(chat);
|
||||
}
|
||||
|
||||
@Get()
|
||||
async list(
|
||||
@CurrentUser() user: User,
|
||||
@Query('limit') limitRaw: unknown,
|
||||
@Query('cursor') cursorRaw: unknown,
|
||||
): Promise<ChatPageResponse> {
|
||||
const limit = parseLimit(limitRaw);
|
||||
const cursor = rawCursorParam(cursorRaw);
|
||||
return this.chats.listForMember(user.id, limit, cursor);
|
||||
}
|
||||
|
||||
@Get(':chatId')
|
||||
async getOne(
|
||||
@Param('chatId') chatId: string,
|
||||
@CurrentUser() user: User,
|
||||
): Promise<ChatResponse> {
|
||||
const chat = await this.chats.getForMember(chatId, user.id);
|
||||
return toChatResponse(chat);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,10 @@
|
||||
import { Module } from '@nestjs/common';
|
||||
import { ChatsController } from './chats.controller';
|
||||
import { ChatsService } from './chats.service';
|
||||
|
||||
@Module({
|
||||
controllers: [ChatsController],
|
||||
providers: [ChatsService],
|
||||
exports: [ChatsService],
|
||||
})
|
||||
export class ChatsModule {}
|
||||
@@ -0,0 +1,131 @@
|
||||
import { Injectable } from '@nestjs/common';
|
||||
import { Prisma } from '@prisma/client';
|
||||
import { decodeChatCursor, encodeChatCursor } from '../common/cursor';
|
||||
import { Chat } from '../common/domain';
|
||||
import { ApiError } from '../common/filters/api-error';
|
||||
import { validate as isUuid } from '../common/uuid';
|
||||
import { PrismaService, isForeignKeyViolation } from '../db/prisma.service';
|
||||
import { ChatPageResponse, toChatResponse } from './dto/chat.response';
|
||||
|
||||
interface ChatRow {
|
||||
id: string;
|
||||
title: string;
|
||||
createdAt: Date;
|
||||
memberIds: string[];
|
||||
}
|
||||
|
||||
@Injectable()
|
||||
export class ChatsService {
|
||||
constructor(private readonly prisma: PrismaService) {}
|
||||
|
||||
async create(
|
||||
title: string,
|
||||
rawMemberIds: string[],
|
||||
callerId: string,
|
||||
): Promise<Chat> {
|
||||
const invalid = rawMemberIds.find((id) => !isUuid(id));
|
||||
if (invalid) {
|
||||
throw new ApiError('invalid_request', `unknown member_id: ${invalid}`);
|
||||
}
|
||||
const memberIds = rawMemberIds.map((id) => id.toLowerCase());
|
||||
if (new Set(memberIds).size !== memberIds.length) {
|
||||
throw new ApiError(
|
||||
'invalid_request',
|
||||
"All member_ids's elements must be unique",
|
||||
);
|
||||
}
|
||||
if (!memberIds.includes(callerId)) {
|
||||
throw new ApiError(
|
||||
'invalid_request',
|
||||
'member_ids must include the caller',
|
||||
);
|
||||
}
|
||||
try {
|
||||
const chat = await this.prisma.$transaction(async (tx) => {
|
||||
const created = await tx.chat.create({
|
||||
data: { title: title.trim() },
|
||||
});
|
||||
await tx.chatMember.createMany({
|
||||
data: memberIds.map((userId) => ({
|
||||
chatId: created.id,
|
||||
userId,
|
||||
})),
|
||||
});
|
||||
return created;
|
||||
});
|
||||
return toChat({ ...chat, memberIds });
|
||||
} catch (error) {
|
||||
if (isForeignKeyViolation(error)) {
|
||||
throw new ApiError(
|
||||
'invalid_request',
|
||||
'member_ids contain an unknown user',
|
||||
);
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
async getForMember(chatId: string, callerId: string): Promise<Chat> {
|
||||
if (!isUuid(chatId)) {
|
||||
throw new ApiError('invalid_request', 'chatId must be a valid UUID');
|
||||
}
|
||||
const chat = await this.prisma.chat.findFirst({
|
||||
where: { id: chatId, members: { some: { userId: callerId } } },
|
||||
include: { members: { select: { userId: true } } },
|
||||
});
|
||||
if (!chat) {
|
||||
throw new ApiError('not_found', 'chat not found');
|
||||
}
|
||||
return toChat({
|
||||
...chat,
|
||||
memberIds: chat.members.map((m) => m.userId),
|
||||
});
|
||||
}
|
||||
|
||||
async listForMember(
|
||||
callerId: string,
|
||||
limit: number,
|
||||
rawCursor: string | null,
|
||||
): Promise<ChatPageResponse> {
|
||||
let after = Prisma.empty;
|
||||
if (rawCursor !== null) {
|
||||
const cursor = decodeChatCursor(rawCursor);
|
||||
const member = cursor
|
||||
? await this.prisma.chatMember.findUnique({
|
||||
where: {
|
||||
chatId_userId: { chatId: cursor.id, userId: callerId },
|
||||
},
|
||||
include: { chat: { select: { createdAt: true } } },
|
||||
})
|
||||
: null;
|
||||
if (!member) {
|
||||
throw new ApiError('invalid_request', 'invalid cursor');
|
||||
}
|
||||
after = Prisma.sql`AND (c.created_at, c.id) > (${member.chat.createdAt}::timestamptz, ${member.chatId}::uuid)`;
|
||||
}
|
||||
const rows = await this.prisma.$queryRaw<ChatRow[]>`
|
||||
SELECT c.id, c.title, c.created_at AS "createdAt",
|
||||
ARRAY(SELECT m.user_id::text FROM chat_members m WHERE m.chat_id = c.id) AS "memberIds"
|
||||
FROM chat_members cm
|
||||
JOIN chats c ON c.id = cm.chat_id
|
||||
WHERE cm.user_id = ${callerId}::uuid ${after}
|
||||
ORDER BY c.created_at, c.id
|
||||
LIMIT ${limit + 1}`;
|
||||
const items = rows.slice(0, limit).map(toChat);
|
||||
const last = items[items.length - 1];
|
||||
return {
|
||||
items: items.map(toChatResponse),
|
||||
next_cursor:
|
||||
rows.length > limit ? encodeChatCursor({ id: last.id }) : null,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
function toChat(row: ChatRow): Chat {
|
||||
return {
|
||||
id: row.id,
|
||||
title: row.title,
|
||||
memberIds: row.memberIds,
|
||||
createdAt: row.createdAt.toISOString(),
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,22 @@
|
||||
import { Chat } from '../../common/domain';
|
||||
|
||||
export interface ChatResponse {
|
||||
id: string;
|
||||
title: string;
|
||||
member_ids: string[];
|
||||
created_at: string;
|
||||
}
|
||||
|
||||
export interface ChatPageResponse {
|
||||
items: ChatResponse[];
|
||||
next_cursor: string | null;
|
||||
}
|
||||
|
||||
export function toChatResponse(chat: Chat): ChatResponse {
|
||||
return {
|
||||
id: chat.id,
|
||||
title: chat.title,
|
||||
member_ids: chat.memberIds,
|
||||
created_at: chat.createdAt,
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,22 @@
|
||||
import {
|
||||
ArrayMaxSize,
|
||||
ArrayMinSize,
|
||||
ArrayUnique,
|
||||
IsArray,
|
||||
IsString,
|
||||
} from 'class-validator';
|
||||
import { CodePointLength } from '../../common/dto/code-point-length.validator';
|
||||
import { NotBlank } from '../../common/dto/not-blank.validator';
|
||||
|
||||
export class CreateChatDto {
|
||||
@NotBlank()
|
||||
@CodePointLength(1, 100)
|
||||
title!: string;
|
||||
|
||||
@IsArray()
|
||||
@ArrayMinSize(1)
|
||||
@ArrayMaxSize(100)
|
||||
@ArrayUnique()
|
||||
@IsString({ each: true })
|
||||
member_ids!: string[];
|
||||
}
|
||||
@@ -0,0 +1,49 @@
|
||||
import { validate as isUuid } from './uuid';
|
||||
|
||||
export interface ChatCursor {
|
||||
id: string;
|
||||
}
|
||||
|
||||
export interface MessageCursor {
|
||||
chatId: string;
|
||||
id: string;
|
||||
}
|
||||
|
||||
export function encodeChatCursor(cursor: ChatCursor): string {
|
||||
return encode(cursor);
|
||||
}
|
||||
|
||||
export function decodeChatCursor(raw: string): ChatCursor | null {
|
||||
const decoded = decode(raw);
|
||||
return decoded && isUuidField(decoded.id) ? { id: decoded.id } : null;
|
||||
}
|
||||
|
||||
export function encodeMessageCursor(cursor: MessageCursor): string {
|
||||
return encode(cursor);
|
||||
}
|
||||
|
||||
export function decodeMessageCursor(raw: string): MessageCursor | null {
|
||||
const decoded = decode(raw);
|
||||
return decoded && isUuidField(decoded.chatId) && isUuidField(decoded.id)
|
||||
? { chatId: decoded.chatId, id: decoded.id }
|
||||
: null;
|
||||
}
|
||||
|
||||
function isUuidField(value: unknown): value is string {
|
||||
return typeof value === 'string' && isUuid(value);
|
||||
}
|
||||
|
||||
function encode(value: object): string {
|
||||
return Buffer.from(JSON.stringify(value), 'utf8').toString('base64url');
|
||||
}
|
||||
|
||||
function decode(raw: string): Record<string, unknown> | null {
|
||||
try {
|
||||
const parsed = JSON.parse(
|
||||
Buffer.from(raw, 'base64url').toString('utf8'),
|
||||
);
|
||||
return typeof parsed === 'object' && parsed !== null ? parsed : null;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,22 @@
|
||||
export interface User {
|
||||
id: string;
|
||||
username: string;
|
||||
displayName: string;
|
||||
createdAt: string;
|
||||
}
|
||||
|
||||
export interface Chat {
|
||||
id: string;
|
||||
title: string;
|
||||
memberIds: string[];
|
||||
createdAt: string;
|
||||
}
|
||||
|
||||
export interface Message {
|
||||
id: string;
|
||||
chatId: string;
|
||||
senderId: string;
|
||||
text: string;
|
||||
createdAt: string;
|
||||
attachmentIds: string[];
|
||||
}
|
||||
@@ -0,0 +1,33 @@
|
||||
import { registerDecorator, ValidationOptions } from 'class-validator';
|
||||
|
||||
export function codePointLength(value: string): number {
|
||||
return [...value].length;
|
||||
}
|
||||
|
||||
export function CodePointLength(
|
||||
min: number,
|
||||
max: number,
|
||||
options?: ValidationOptions,
|
||||
) {
|
||||
return function (object: object, propertyName: string) {
|
||||
registerDecorator({
|
||||
name: 'codePointLength',
|
||||
target: object.constructor,
|
||||
propertyName,
|
||||
constraints: [min, max],
|
||||
options,
|
||||
validator: {
|
||||
validate(value: unknown): boolean {
|
||||
if (typeof value !== 'string') {
|
||||
return false;
|
||||
}
|
||||
const length = codePointLength(value);
|
||||
return length >= min && length <= max;
|
||||
},
|
||||
defaultMessage(): string {
|
||||
return `${propertyName} must be between ${min} and ${max} Unicode code points`;
|
||||
},
|
||||
},
|
||||
});
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,20 @@
|
||||
import { registerDecorator, ValidationOptions } from 'class-validator';
|
||||
|
||||
export function NotBlank(options?: ValidationOptions) {
|
||||
return function (object: object, propertyName: string) {
|
||||
registerDecorator({
|
||||
name: 'notBlank',
|
||||
target: object.constructor,
|
||||
propertyName,
|
||||
options,
|
||||
validator: {
|
||||
validate(value: unknown): boolean {
|
||||
return typeof value === 'string' && value.trim().length > 0;
|
||||
},
|
||||
defaultMessage(): string {
|
||||
return `${propertyName} must not be empty or whitespace-only`;
|
||||
},
|
||||
},
|
||||
});
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,35 @@
|
||||
import { ApiError } from '../filters/api-error';
|
||||
|
||||
const DEFAULT_LIMIT = 50;
|
||||
const MIN_LIMIT = 1;
|
||||
const MAX_LIMIT = 100;
|
||||
|
||||
export function parseLimit(raw: unknown): number {
|
||||
if (raw === undefined) {
|
||||
return DEFAULT_LIMIT;
|
||||
}
|
||||
if (typeof raw !== 'string' || !/^\d+$/.test(raw)) {
|
||||
throw new ApiError('invalid_request', 'limit must be an integer');
|
||||
}
|
||||
const limit = Number(raw);
|
||||
if (limit < MIN_LIMIT || limit > MAX_LIMIT) {
|
||||
throw new ApiError(
|
||||
'invalid_request',
|
||||
`limit must be between ${MIN_LIMIT} and ${MAX_LIMIT}`,
|
||||
);
|
||||
}
|
||||
return limit;
|
||||
}
|
||||
|
||||
export function rawCursorParam(raw: unknown): string | null {
|
||||
if (raw === undefined) {
|
||||
return null;
|
||||
}
|
||||
if (typeof raw !== 'string' || raw.length === 0) {
|
||||
throw new ApiError(
|
||||
'invalid_request',
|
||||
'cursor must be a non-empty string',
|
||||
);
|
||||
}
|
||||
return raw;
|
||||
}
|
||||
@@ -0,0 +1,34 @@
|
||||
import { HttpException, HttpStatus } from '@nestjs/common';
|
||||
|
||||
export type ErrorCode =
|
||||
| 'invalid_request'
|
||||
| 'unauthorized'
|
||||
| 'not_found'
|
||||
| 'conflict'
|
||||
| 'payload_too_large'
|
||||
| 'unsupported_media_type'
|
||||
| 'rate_limited'
|
||||
| 'unavailable'
|
||||
| 'internal_error';
|
||||
|
||||
const STATUS_BY_CODE: Record<ErrorCode, number> = {
|
||||
invalid_request: HttpStatus.BAD_REQUEST,
|
||||
unauthorized: HttpStatus.UNAUTHORIZED,
|
||||
not_found: HttpStatus.NOT_FOUND,
|
||||
conflict: HttpStatus.CONFLICT,
|
||||
payload_too_large: HttpStatus.PAYLOAD_TOO_LARGE,
|
||||
unsupported_media_type: HttpStatus.UNSUPPORTED_MEDIA_TYPE,
|
||||
rate_limited: HttpStatus.TOO_MANY_REQUESTS,
|
||||
unavailable: HttpStatus.SERVICE_UNAVAILABLE,
|
||||
internal_error: HttpStatus.INTERNAL_SERVER_ERROR,
|
||||
};
|
||||
|
||||
export class ApiError extends HttpException {
|
||||
constructor(
|
||||
public readonly code: ErrorCode,
|
||||
message: string,
|
||||
public readonly headers: Record<string, string> = {},
|
||||
) {
|
||||
super(message, STATUS_BY_CODE[code]);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,121 @@
|
||||
import {
|
||||
ArgumentsHost,
|
||||
Catch,
|
||||
ExceptionFilter,
|
||||
HttpException,
|
||||
HttpStatus,
|
||||
Logger,
|
||||
} from '@nestjs/common';
|
||||
import { FastifyReply, FastifyRequest } from 'fastify';
|
||||
import { isDatabaseUnavailable } from '../../db/prisma.service';
|
||||
import { routeTemplate } from '../../telemetry/http';
|
||||
import { ApiError, ErrorCode } from './api-error';
|
||||
|
||||
const CODE_BY_STATUS: Partial<Record<number, ErrorCode>> = {
|
||||
[HttpStatus.BAD_REQUEST]: 'invalid_request',
|
||||
[HttpStatus.UNAUTHORIZED]: 'unauthorized',
|
||||
[HttpStatus.NOT_FOUND]: 'not_found',
|
||||
[HttpStatus.CONFLICT]: 'conflict',
|
||||
[HttpStatus.PAYLOAD_TOO_LARGE]: 'payload_too_large',
|
||||
[HttpStatus.UNSUPPORTED_MEDIA_TYPE]: 'unsupported_media_type',
|
||||
[HttpStatus.TOO_MANY_REQUESTS]: 'rate_limited',
|
||||
[HttpStatus.SERVICE_UNAVAILABLE]: 'unavailable',
|
||||
};
|
||||
|
||||
interface Resolved {
|
||||
status: number;
|
||||
code: ErrorCode;
|
||||
message: string;
|
||||
}
|
||||
|
||||
@Catch()
|
||||
export class HttpExceptionFilter implements ExceptionFilter {
|
||||
private readonly logger = new Logger('HttpExceptionFilter');
|
||||
|
||||
catch(exception: unknown, host: ArgumentsHost): void {
|
||||
const ctx = host.switchToHttp();
|
||||
const reply = ctx.getResponse<FastifyReply>();
|
||||
const request = ctx.getRequest<FastifyRequest>();
|
||||
const requestId = request.requestId;
|
||||
|
||||
const { status, code, message } = this.resolve(exception);
|
||||
if (status >= 500) {
|
||||
this.logger.error({
|
||||
event: 'http.error',
|
||||
message,
|
||||
method: request.method,
|
||||
route: routeTemplate(request),
|
||||
status,
|
||||
request_id: requestId,
|
||||
});
|
||||
}
|
||||
|
||||
if (exception instanceof ApiError) {
|
||||
reply.headers(exception.headers);
|
||||
}
|
||||
reply
|
||||
.code(status)
|
||||
.header('x-request-id', requestId)
|
||||
.send({ error: { code, message, request_id: requestId } });
|
||||
}
|
||||
|
||||
private resolve(exception: unknown): Resolved {
|
||||
if (exception instanceof ApiError) {
|
||||
return {
|
||||
status: exception.getStatus(),
|
||||
code: exception.code,
|
||||
message: exception.message,
|
||||
};
|
||||
}
|
||||
if (exception instanceof HttpException) {
|
||||
const status = exception.getStatus();
|
||||
const code = CODE_BY_STATUS[status] ?? 'internal_error';
|
||||
const message =
|
||||
extractMessage(exception.getResponse()) ?? exception.message;
|
||||
return { status, code, message };
|
||||
}
|
||||
const fastifyStatus = (exception as { statusCode?: unknown })
|
||||
?.statusCode;
|
||||
if (typeof fastifyStatus === 'number' && CODE_BY_STATUS[fastifyStatus]) {
|
||||
const message =
|
||||
exception instanceof Error ? exception.message : 'Invalid request';
|
||||
return {
|
||||
status: fastifyStatus,
|
||||
code: CODE_BY_STATUS[fastifyStatus]!,
|
||||
message,
|
||||
};
|
||||
}
|
||||
if (isDatabaseUnavailable(exception)) {
|
||||
return {
|
||||
status: 503,
|
||||
code: 'unavailable',
|
||||
message: 'database unavailable',
|
||||
};
|
||||
}
|
||||
return {
|
||||
status: 500,
|
||||
code: 'internal_error',
|
||||
message: 'Internal server error',
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
function extractMessage(response: unknown): string | null {
|
||||
if (typeof response === 'string') {
|
||||
return response;
|
||||
}
|
||||
if (
|
||||
typeof response === 'object' &&
|
||||
response !== null &&
|
||||
'message' in response
|
||||
) {
|
||||
const value = (response as { message: unknown }).message;
|
||||
if (typeof value === 'string') {
|
||||
return value;
|
||||
}
|
||||
if (Array.isArray(value) && value.every((v) => typeof v === 'string')) {
|
||||
return value.join('; ');
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
import { randomUUID } from 'crypto';
|
||||
import { FastifyInstance, FastifyRequest } from 'fastify';
|
||||
import { hostname } from 'os';
|
||||
|
||||
const REQUEST_ID_HEADER = 'x-request-id';
|
||||
const INSTANCE_ID_HEADER = 'x-instance-id';
|
||||
|
||||
export const INSTANCE_ID = process.env.INSTANCE_ID ?? hostname();
|
||||
|
||||
declare module 'fastify' {
|
||||
interface FastifyRequest {
|
||||
requestId: string;
|
||||
}
|
||||
}
|
||||
|
||||
export function registerRequestId(instance: FastifyInstance): void {
|
||||
instance.addHook('onRequest', (request: FastifyRequest, reply, done) => {
|
||||
request.requestId = randomUUID();
|
||||
reply.header(REQUEST_ID_HEADER, request.requestId);
|
||||
reply.header(INSTANCE_ID_HEADER, INSTANCE_ID);
|
||||
done();
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,6 @@
|
||||
const UUID_PATTERN =
|
||||
/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
|
||||
|
||||
export function validate(value: string): boolean {
|
||||
return UUID_PATTERN.test(value);
|
||||
}
|
||||
@@ -0,0 +1,21 @@
|
||||
import { ValidationPipe } from '@nestjs/common';
|
||||
import { NestFastifyApplication } from '@nestjs/platform-fastify';
|
||||
import { registerUploadBody } from './attachments/upload-body';
|
||||
import { HttpExceptionFilter } from './common/filters/http-exception.filter';
|
||||
import { registerRequestId } from './common/interceptors/request-id';
|
||||
|
||||
export function configureApp(app: NestFastifyApplication): void {
|
||||
const instance = app.getHttpAdapter().getInstance();
|
||||
registerRequestId(instance);
|
||||
registerUploadBody(instance);
|
||||
|
||||
app.useGlobalPipes(
|
||||
new ValidationPipe({
|
||||
whitelist: true,
|
||||
forbidNonWhitelisted: true,
|
||||
transform: true,
|
||||
forbidUnknownValues: true,
|
||||
}),
|
||||
);
|
||||
app.useGlobalFilters(new HttpExceptionFilter());
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
import { Global, Module } from '@nestjs/common';
|
||||
import { PrismaService } from './prisma.service';
|
||||
|
||||
@Global()
|
||||
@Module({
|
||||
providers: [PrismaService],
|
||||
exports: [PrismaService],
|
||||
})
|
||||
export class DbModule {}
|
||||
@@ -0,0 +1,52 @@
|
||||
import { Injectable, OnModuleDestroy } from '@nestjs/common';
|
||||
import { Prisma, PrismaClient } from '@prisma/client';
|
||||
import { readFileSync } from 'fs';
|
||||
import { registerDbPool } from '../telemetry/db-pool';
|
||||
|
||||
const UNAVAILABLE_CODES = new Set(['P1001', 'P1002', 'P1008', 'P1017', 'P2024']);
|
||||
|
||||
@Injectable()
|
||||
export class PrismaService extends PrismaClient implements OnModuleDestroy {
|
||||
constructor() {
|
||||
const url = readDatabaseUrl();
|
||||
super({ datasourceUrl: url });
|
||||
registerDbPool(this, url);
|
||||
}
|
||||
|
||||
async onModuleDestroy(): Promise<void> {
|
||||
await this.$disconnect();
|
||||
}
|
||||
}
|
||||
|
||||
function readDatabaseUrl(): string {
|
||||
const file = process.env.DATABASE_URL_FILE;
|
||||
const url = file ? readFileSync(file, 'utf8').trim() : process.env.DATABASE_URL;
|
||||
if (!url) {
|
||||
throw new Error('DATABASE_URL_FILE or DATABASE_URL must be set');
|
||||
}
|
||||
return url;
|
||||
}
|
||||
|
||||
export function isUniqueViolation(error: unknown): boolean {
|
||||
return (
|
||||
error instanceof Prisma.PrismaClientKnownRequestError &&
|
||||
error.code === 'P2002'
|
||||
);
|
||||
}
|
||||
|
||||
export function isForeignKeyViolation(error: unknown): boolean {
|
||||
return (
|
||||
error instanceof Prisma.PrismaClientKnownRequestError &&
|
||||
error.code === 'P2003'
|
||||
);
|
||||
}
|
||||
|
||||
export function isDatabaseUnavailable(error: unknown): boolean {
|
||||
if (error instanceof Prisma.PrismaClientInitializationError) {
|
||||
return true;
|
||||
}
|
||||
return (
|
||||
error instanceof Prisma.PrismaClientKnownRequestError &&
|
||||
UNAVAILABLE_CODES.has(error.code)
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,351 @@
|
||||
const ALICE_ID = '11111111-1111-4111-8111-111111111111';
|
||||
const BOB_ID = '22222222-2222-4222-8222-222222222222';
|
||||
const CHAT_ID = '33333333-3333-4333-8333-333333333333';
|
||||
const MESSAGE_ID = '44444444-4444-4444-8444-444444444444';
|
||||
const CURSOR =
|
||||
'eyJjaGF0SWQiOiIzMzMzMzMzMy0zMzMzLTQzMzMtODMzMy0zMzMzMzMzMzMzMzMiLCJpZCI6IjQ0NDQ0NDQ0LTQ0NDQtNDQ0NC04NDQ0LTQ0NDQ0NDQ0NDQ0NCJ9';
|
||||
|
||||
const ALICE = {
|
||||
id: ALICE_ID,
|
||||
username: 'alice',
|
||||
display_name: 'Алиса',
|
||||
created_at: '2026-09-01T12:00:00.000Z',
|
||||
};
|
||||
|
||||
const CHAT = {
|
||||
id: CHAT_ID,
|
||||
title: 'Проект по ВНП',
|
||||
member_ids: [ALICE_ID, BOB_ID],
|
||||
created_at: '2026-09-01T12:01:00.000Z',
|
||||
};
|
||||
|
||||
const MESSAGE = {
|
||||
id: MESSAGE_ID,
|
||||
chat_id: CHAT_ID,
|
||||
sender_id: ALICE_ID,
|
||||
text: 'Привет, Боб! 👋',
|
||||
created_at: '2026-09-01T12:02:00.000Z',
|
||||
attachment_ids: [],
|
||||
};
|
||||
|
||||
export interface Example {
|
||||
summary: string;
|
||||
value: unknown;
|
||||
}
|
||||
|
||||
export interface OperationExamples {
|
||||
request?: Record<string, Example>;
|
||||
responses: Record<string, Record<string, Example>>;
|
||||
}
|
||||
|
||||
function example(summary: string, value: unknown): Example {
|
||||
return { summary, value };
|
||||
}
|
||||
|
||||
function error(code: string, message: string): unknown {
|
||||
return {
|
||||
error: {
|
||||
code,
|
||||
message,
|
||||
request_id: '9b2c6f1e-5d4a-4c3b-8a2f-1e0d9c8b7a6f',
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
const UNAUTHORIZED = {
|
||||
missing: example(
|
||||
'Нет/неверный/истёкший/отозванный Bearer-токен',
|
||||
error('unauthorized', 'missing or invalid session'),
|
||||
),
|
||||
};
|
||||
|
||||
const UNAVAILABLE = {
|
||||
database: example(
|
||||
'PostgreSQL недоступен',
|
||||
error('unavailable', 'database unavailable'),
|
||||
),
|
||||
};
|
||||
|
||||
const CHAT_NOT_FOUND = {
|
||||
notFound: example(
|
||||
'Чата нет или вызывающий не участник (не 403)',
|
||||
error('not_found', 'chat not found'),
|
||||
),
|
||||
};
|
||||
|
||||
const INVALID_CHAT_ID = example(
|
||||
'chat_id не UUID',
|
||||
error('invalid_request', 'chatId must be a valid UUID'),
|
||||
);
|
||||
|
||||
export const GUIDE = `
|
||||
Учебный мессенджер, лабораторная 6: PostgreSQL, сессии, Redis-кэш сессий, вложения в S3 и их фоновая обработка.
|
||||
Документ — обязательный контракт \`contracts/openapi.json\`, дополненный примерами запросов и ответов.
|
||||
|
||||
**Как пройти сценарий в Swagger UI**
|
||||
|
||||
1. \`POST /api/v1/users\` — зарегистрируйте пользователей (примеры alice, bob, eve в списке Examples). Повторная регистрация того же username — 409, меняйте username.
|
||||
2. **Authorize → basicAuth**: username и password из шага 1 → \`POST /api/v1/auth/sessions\` → скопируйте \`token\`.
|
||||
3. **Authorize → bearerAuth**: вставьте token (без слова Bearer). Basic отправляется только на вход, остальные запросы идут с Bearer.
|
||||
4. \`GET /api/v1/users/me\` — свой id; id второго участника — из ответа его регистрации.
|
||||
5. \`POST /api/v1/chats\` с \`member_ids\` = [свой id, id собеседника] → \`chat_id\`.
|
||||
6. \`POST /api/v1/chats/{chat_id}/messages\` с заголовком \`Idempotency-Key\` (любая строка из \`A-Za-z0-9._:-\`, до 128 символов). Повтор с тем же ключом и телом вернёт то же сообщение, с другим телом — 409. Затем \`GET …/messages?limit=1\` и следующая страница с \`cursor\` = \`next_cursor\`.
|
||||
7. \`POST /api/v1/chats/{chat_id}/attachments?filename=photo.png\` с телом PNG/JPEG → 202 \`queued\`. \`GET /api/v1/attachments/{id}\` повторять, пока \`status\` не станет \`ready\` (до этого \`/content\` отвечает 409), затем \`/content?variant=thumbnail\`, \`crop\` или \`optimized\`.
|
||||
8. Чтобы действовать от другого пользователя: Logout в Authorize, вход им и новый token.
|
||||
9. \`DELETE /api/v1/auth/sessions/current\` — logout; тот же token дальше даёт 401.
|
||||
|
||||
Сессия живёт \`SESSION_TTL_SECONDS\` (по умолчанию 3600 с). Пароль — 12–128 символов Unicode, двоеточие допустимо.
|
||||
`;
|
||||
|
||||
export const OPERATION_EXAMPLES: Record<string, OperationExamples> = {
|
||||
'GET /health/live': {
|
||||
responses: {
|
||||
'200': { ok: example('Процесс жив', { status: 'ok' }) },
|
||||
},
|
||||
},
|
||||
'GET /health/ready': {
|
||||
responses: {
|
||||
'200': {
|
||||
ok: example('PostgreSQL доступен (Redis не критичен)', {
|
||||
status: 'ok',
|
||||
}),
|
||||
},
|
||||
'503': UNAVAILABLE,
|
||||
},
|
||||
},
|
||||
'POST /api/v1/users': {
|
||||
request: {
|
||||
alice: example('Алиса', {
|
||||
username: 'alice',
|
||||
display_name: 'Алиса',
|
||||
password: 'Alice-password-123',
|
||||
}),
|
||||
bob: example('Боб', {
|
||||
username: 'bob',
|
||||
display_name: 'Боб',
|
||||
password: 'Bob-password-123',
|
||||
}),
|
||||
eve: example('Ева (посторонняя)', {
|
||||
username: 'eve',
|
||||
display_name: 'Ева',
|
||||
password: 'Eve-password-123',
|
||||
}),
|
||||
colon: example('Пароль с двоеточием и эмодзи', {
|
||||
username: 'carol',
|
||||
display_name: 'Кэрол 🌸',
|
||||
password: 'пароль:с:двоеточием🔑',
|
||||
}),
|
||||
},
|
||||
responses: {
|
||||
'201': { created: example('Пользователь создан', ALICE) },
|
||||
'400': {
|
||||
username: example(
|
||||
'username не соответствует [a-z0-9_]{3,32}',
|
||||
error('invalid_request', 'username must match [a-z0-9_]{3,32}'),
|
||||
),
|
||||
password: example(
|
||||
'password короче 12 символов',
|
||||
error(
|
||||
'invalid_request',
|
||||
'password must be between 12 and 128 Unicode code points',
|
||||
),
|
||||
),
|
||||
extra: example(
|
||||
'Лишнее поле в JSON',
|
||||
error('invalid_request', 'property extra should not exist'),
|
||||
),
|
||||
},
|
||||
'409': {
|
||||
taken: example(
|
||||
'username уже занят',
|
||||
error('conflict', 'username is already taken'),
|
||||
),
|
||||
},
|
||||
'503': UNAVAILABLE,
|
||||
},
|
||||
},
|
||||
'POST /api/v1/auth/sessions': {
|
||||
responses: {
|
||||
'201': {
|
||||
created: example('Сессия создана', {
|
||||
token: 'q8Hn3vXl0ZbJ7mWcY2sRkT5uPaE9dFgVhN1oLxI4w6A',
|
||||
token_type: 'Bearer',
|
||||
expires_at: '2026-09-01T13:00:00.000Z',
|
||||
user: ALICE,
|
||||
}),
|
||||
},
|
||||
'401': {
|
||||
invalid: example(
|
||||
'Неверный пароль, неизвестный пользователь или нет Basic — ответ одинаковый',
|
||||
error('unauthorized', 'invalid credentials'),
|
||||
),
|
||||
},
|
||||
'503': UNAVAILABLE,
|
||||
},
|
||||
},
|
||||
'DELETE /api/v1/auth/sessions/current': {
|
||||
responses: {
|
||||
'401': UNAUTHORIZED,
|
||||
'503': {
|
||||
...UNAVAILABLE,
|
||||
cache: example(
|
||||
'Redis недоступен — отзыв не выполнен, повторите позже',
|
||||
error('unavailable', 'session cache unavailable'),
|
||||
),
|
||||
},
|
||||
},
|
||||
},
|
||||
'GET /api/v1/users/me': {
|
||||
responses: {
|
||||
'200': { me: example('Текущий пользователь из сессии', ALICE) },
|
||||
'401': UNAUTHORIZED,
|
||||
'503': UNAVAILABLE,
|
||||
},
|
||||
},
|
||||
'POST /api/v1/chats': {
|
||||
request: {
|
||||
pair: example('Чат Алисы и Боба', {
|
||||
title: CHAT.title,
|
||||
member_ids: CHAT.member_ids,
|
||||
}),
|
||||
solo: example('Чат только с собой', {
|
||||
title: 'Заметки',
|
||||
member_ids: [ALICE_ID],
|
||||
}),
|
||||
},
|
||||
responses: {
|
||||
'201': { created: example('Чат создан атомарно', CHAT) },
|
||||
'400': {
|
||||
noCaller: example(
|
||||
'Вызывающий не в member_ids',
|
||||
error('invalid_request', 'member_ids must include the caller'),
|
||||
),
|
||||
unknown: example(
|
||||
'Неизвестный участник — транзакция откатывается',
|
||||
error('invalid_request', 'member_ids contain an unknown user'),
|
||||
),
|
||||
duplicate: example(
|
||||
'Дубликаты в member_ids',
|
||||
error(
|
||||
'invalid_request',
|
||||
"All member_ids's elements must be unique",
|
||||
),
|
||||
),
|
||||
},
|
||||
'401': UNAUTHORIZED,
|
||||
'503': UNAVAILABLE,
|
||||
},
|
||||
},
|
||||
'GET /api/v1/chats': {
|
||||
responses: {
|
||||
'200': {
|
||||
page: example('Страница чатов с продолжением', {
|
||||
items: [CHAT],
|
||||
next_cursor: 'eyJpZCI6IjMzMzMzMzMzLTMzMzMtNDMzMy04MzMzLTMzMzMzMzMzMzMzMyJ9',
|
||||
}),
|
||||
last: example('Последняя страница', {
|
||||
items: [CHAT],
|
||||
next_cursor: null,
|
||||
}),
|
||||
},
|
||||
'400': {
|
||||
limit: example(
|
||||
'limit вне 1–100',
|
||||
error('invalid_request', 'limit must be between 1 and 100'),
|
||||
),
|
||||
cursor: example(
|
||||
'Испорченный или чужой cursor',
|
||||
error('invalid_request', 'invalid cursor'),
|
||||
),
|
||||
},
|
||||
'401': UNAUTHORIZED,
|
||||
'503': UNAVAILABLE,
|
||||
},
|
||||
},
|
||||
'GET /api/v1/chats/{chat_id}': {
|
||||
responses: {
|
||||
'200': { chat: example('Чат, где вызывающий — участник', CHAT) },
|
||||
'400': { chatId: INVALID_CHAT_ID },
|
||||
'401': UNAUTHORIZED,
|
||||
'404': CHAT_NOT_FOUND,
|
||||
'503': UNAVAILABLE,
|
||||
},
|
||||
},
|
||||
'POST /api/v1/chats/{chat_id}/messages': {
|
||||
request: {
|
||||
text: example('Обычное сообщение', { text: MESSAGE.text }),
|
||||
long: example('Максимум — 4000 символов Unicode', {
|
||||
text: 'Длинный текст…',
|
||||
}),
|
||||
},
|
||||
responses: {
|
||||
'201': {
|
||||
created: example('Отправитель берётся из сессии', MESSAGE),
|
||||
},
|
||||
'400': {
|
||||
blank: example(
|
||||
'Пустой или пробельный текст',
|
||||
error(
|
||||
'invalid_request',
|
||||
'text must not be empty or whitespace-only',
|
||||
),
|
||||
),
|
||||
sender: example(
|
||||
'sender_id в теле не принимается',
|
||||
error('invalid_request', 'property sender_id should not exist'),
|
||||
),
|
||||
chatId: INVALID_CHAT_ID,
|
||||
key: example(
|
||||
'Нет Idempotency-Key или неверный формат',
|
||||
error(
|
||||
'invalid_request',
|
||||
'Idempotency-Key header must match [A-Za-z0-9._:-]{1,128}',
|
||||
),
|
||||
),
|
||||
},
|
||||
'401': UNAUTHORIZED,
|
||||
'404': CHAT_NOT_FOUND,
|
||||
'409': {
|
||||
keyReused: example(
|
||||
'Тот же Idempotency-Key с другим телом',
|
||||
error(
|
||||
'conflict',
|
||||
'Idempotency-Key was already used with a different request',
|
||||
),
|
||||
),
|
||||
},
|
||||
'503': UNAVAILABLE,
|
||||
},
|
||||
},
|
||||
'GET /api/v1/chats/{chat_id}/messages': {
|
||||
responses: {
|
||||
'200': {
|
||||
page: example('Страница истории (limit=1) с cursor', {
|
||||
items: [MESSAGE],
|
||||
next_cursor: CURSOR,
|
||||
}),
|
||||
last: example('Последняя страница', {
|
||||
items: [MESSAGE],
|
||||
next_cursor: null,
|
||||
}),
|
||||
},
|
||||
'400': {
|
||||
otherChat: example(
|
||||
'cursor от другого чата',
|
||||
error('invalid_request', 'cursor belongs to a different chat'),
|
||||
),
|
||||
cursor: example(
|
||||
'Испорченный cursor или cursor несуществующего сообщения',
|
||||
error('invalid_request', 'invalid cursor'),
|
||||
),
|
||||
limit: example(
|
||||
'limit вне 1–100',
|
||||
error('invalid_request', 'limit must be between 1 and 100'),
|
||||
),
|
||||
},
|
||||
'401': UNAUTHORIZED,
|
||||
'404': CHAT_NOT_FOUND,
|
||||
'503': UNAVAILABLE,
|
||||
},
|
||||
},
|
||||
};
|
||||
@@ -0,0 +1,43 @@
|
||||
import { NestFastifyApplication } from '@nestjs/platform-fastify';
|
||||
import { OpenAPIObject, SwaggerModule } from '@nestjs/swagger';
|
||||
import { readFileSync } from 'fs';
|
||||
import { join } from 'path';
|
||||
import { GUIDE, OPERATION_EXAMPLES } from './examples';
|
||||
|
||||
const CONTRACT_PATH = join(process.cwd(), 'contracts', 'openapi.json');
|
||||
|
||||
export function setupSwagger(app: NestFastifyApplication): void {
|
||||
SwaggerModule.setup('docs', app, loadDocument(), {
|
||||
customSiteTitle: 'Messenger lab6 API',
|
||||
swaggerOptions: {
|
||||
persistAuthorization: true,
|
||||
displayRequestDuration: true,
|
||||
tryItOutEnabled: true,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
export function loadDocument(): OpenAPIObject {
|
||||
const document = JSON.parse(readFileSync(CONTRACT_PATH, 'utf8'));
|
||||
document.info.description = GUIDE;
|
||||
document.servers = [{ url: '/', description: 'Этот экземпляр API' }];
|
||||
for (const [key, examples] of Object.entries(OPERATION_EXAMPLES)) {
|
||||
const [method, path] = key.split(' ');
|
||||
const operation = document.paths[path]?.[method.toLowerCase()];
|
||||
if (!operation) {
|
||||
throw new Error(`examples for unknown operation ${key}`);
|
||||
}
|
||||
if (examples.request) {
|
||||
operation.requestBody.content['application/json'].examples =
|
||||
examples.request;
|
||||
}
|
||||
for (const [status, named] of Object.entries(examples.responses)) {
|
||||
const content = operation.responses[status]?.content;
|
||||
if (!content) {
|
||||
throw new Error(`examples for unknown response ${key} ${status}`);
|
||||
}
|
||||
content['application/json'].examples = named;
|
||||
}
|
||||
}
|
||||
return document;
|
||||
}
|
||||
@@ -0,0 +1,44 @@
|
||||
import { Controller, Get } from '@nestjs/common';
|
||||
import { ApiError } from '../common/filters/api-error';
|
||||
import { PrismaService } from '../db/prisma.service';
|
||||
import { S3Service } from '../storage/s3.service';
|
||||
import { isDraining } from './shutdown';
|
||||
|
||||
@Controller('health')
|
||||
export class HealthController {
|
||||
constructor(
|
||||
private readonly prisma: PrismaService,
|
||||
private readonly s3: S3Service,
|
||||
) {}
|
||||
|
||||
@Get('live')
|
||||
live(): { status: 'ok' } {
|
||||
return { status: 'ok' };
|
||||
}
|
||||
|
||||
@Get('core')
|
||||
async core(): Promise<{ status: 'ok' }> {
|
||||
await this.assertCoreReady();
|
||||
return { status: 'ok' };
|
||||
}
|
||||
|
||||
@Get('ready')
|
||||
async ready(): Promise<{ status: 'ok' }> {
|
||||
await this.assertCoreReady();
|
||||
if (!(await this.s3.isAvailable())) {
|
||||
throw new ApiError('unavailable', 'object storage unavailable');
|
||||
}
|
||||
return { status: 'ok' };
|
||||
}
|
||||
|
||||
private async assertCoreReady(): Promise<void> {
|
||||
if (isDraining()) {
|
||||
throw new ApiError('unavailable', 'shutting down');
|
||||
}
|
||||
try {
|
||||
await this.prisma.$queryRaw`SELECT 1`;
|
||||
} catch {
|
||||
throw new ApiError('unavailable', 'database unavailable');
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,7 @@
|
||||
import { Module } from '@nestjs/common';
|
||||
import { HealthController } from './health.controller';
|
||||
|
||||
@Module({
|
||||
controllers: [HealthController],
|
||||
})
|
||||
export class HealthModule {}
|
||||
@@ -0,0 +1,33 @@
|
||||
import { INestApplication, Logger } from '@nestjs/common';
|
||||
import { shutdownTelemetry } from '../telemetry/sdk';
|
||||
|
||||
let draining = false;
|
||||
|
||||
export function isDraining(): boolean {
|
||||
return draining;
|
||||
}
|
||||
|
||||
export function enableGracefulShutdown(app: INestApplication): void {
|
||||
const delayMs = Number(process.env.SHUTDOWN_DELAY_MS ?? 3000);
|
||||
const shutdown = (signal: string): void => {
|
||||
if (draining) {
|
||||
return;
|
||||
}
|
||||
draining = true;
|
||||
Logger.log(`${signal}: readiness → 503, closing in ${delayMs} ms`, 'Shutdown');
|
||||
setTimeout(() => {
|
||||
app
|
||||
.close()
|
||||
.then(
|
||||
() => 0,
|
||||
() => 1,
|
||||
)
|
||||
.then(async (code) => {
|
||||
await shutdownTelemetry();
|
||||
process.exit(code);
|
||||
});
|
||||
}, delayMs);
|
||||
};
|
||||
process.once('SIGTERM', () => shutdown('SIGTERM'));
|
||||
process.once('SIGINT', () => shutdown('SIGINT'));
|
||||
}
|
||||
@@ -0,0 +1,98 @@
|
||||
import { context, propagation } from '@opentelemetry/api';
|
||||
import { validate as isUuid } from '../common/uuid';
|
||||
|
||||
export const PIPELINE_VERSION = 1;
|
||||
export const JOB_TYPE = 'attachment.process.v1';
|
||||
|
||||
const TRACEPARENT = /^00-[0-9a-f]{32}-[0-9a-f]{16}-[0-9a-f]{2}$/;
|
||||
const TRACESTATE_MAX = 512;
|
||||
|
||||
export interface TraceCarrier {
|
||||
traceparent?: string;
|
||||
tracestate?: string;
|
||||
}
|
||||
|
||||
export interface JobMessage extends TraceCarrier {
|
||||
schema_version: 1;
|
||||
job_id: string;
|
||||
type: typeof JOB_TYPE;
|
||||
attachment_id: string;
|
||||
pipeline_version: number;
|
||||
attempt: number;
|
||||
created_at: string;
|
||||
}
|
||||
|
||||
const FIELDS = [
|
||||
'schema_version',
|
||||
'job_id',
|
||||
'type',
|
||||
'attachment_id',
|
||||
'pipeline_version',
|
||||
'attempt',
|
||||
'created_at',
|
||||
'traceparent',
|
||||
'tracestate',
|
||||
];
|
||||
|
||||
export function jobMessage(
|
||||
job: { id: string; attachmentId: string; pipelineVersion: number; attempt: number },
|
||||
createdAt: Date,
|
||||
): JobMessage {
|
||||
return {
|
||||
schema_version: 1,
|
||||
job_id: job.id,
|
||||
type: JOB_TYPE,
|
||||
attachment_id: job.attachmentId,
|
||||
pipeline_version: job.pipelineVersion,
|
||||
attempt: job.attempt,
|
||||
created_at: createdAt.toISOString(),
|
||||
...currentTraceCarrier(),
|
||||
};
|
||||
}
|
||||
|
||||
export function currentTraceCarrier(): TraceCarrier {
|
||||
const carrier: Record<string, string> = {};
|
||||
propagation.inject(context.active(), carrier);
|
||||
const result: TraceCarrier = {};
|
||||
if (TRACEPARENT.test(carrier.traceparent ?? '')) {
|
||||
result.traceparent = carrier.traceparent;
|
||||
if (carrier.tracestate && carrier.tracestate.length <= TRACESTATE_MAX) {
|
||||
result.tracestate = carrier.tracestate;
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
export function parseJobMessage(raw: Buffer): JobMessage | null {
|
||||
let value: any;
|
||||
try {
|
||||
value = JSON.parse(raw.toString('utf8'));
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
if (
|
||||
typeof value !== 'object' ||
|
||||
value === null ||
|
||||
Array.isArray(value) ||
|
||||
Object.keys(value).some((key) => !FIELDS.includes(key)) ||
|
||||
value.schema_version !== 1 ||
|
||||
value.type !== JOB_TYPE ||
|
||||
typeof value.job_id !== 'string' ||
|
||||
!isUuid(value.job_id) ||
|
||||
typeof value.attachment_id !== 'string' ||
|
||||
!isUuid(value.attachment_id) ||
|
||||
!Number.isInteger(value.pipeline_version) ||
|
||||
value.pipeline_version < 1 ||
|
||||
!Number.isInteger(value.attempt) ||
|
||||
value.attempt < 1 ||
|
||||
typeof value.created_at !== 'string' ||
|
||||
Number.isNaN(Date.parse(value.created_at)) ||
|
||||
(value.traceparent !== undefined &&
|
||||
(typeof value.traceparent !== 'string' || !TRACEPARENT.test(value.traceparent))) ||
|
||||
(value.tracestate !== undefined &&
|
||||
(typeof value.tracestate !== 'string' || value.tracestate.length > TRACESTATE_MAX))
|
||||
) {
|
||||
return null;
|
||||
}
|
||||
return value as JobMessage;
|
||||
}
|
||||
@@ -0,0 +1,257 @@
|
||||
import { AttachmentJob, Prisma, PrismaClient } from '@prisma/client';
|
||||
import { randomUUID } from 'crypto';
|
||||
import { jobMessage, PIPELINE_VERSION } from './job-message';
|
||||
|
||||
type Tx = Prisma.TransactionClient;
|
||||
|
||||
export const RETRY_EXHAUSTED = 'retry_exhausted';
|
||||
|
||||
export function maxAttempts(): number {
|
||||
return Number(process.env.JOB_MAX_ATTEMPTS ?? 5);
|
||||
}
|
||||
|
||||
export function backoffMs(attempt: number, random = Math.random): number {
|
||||
const base = Number(process.env.JOB_RETRY_BASE_MS ?? 1000);
|
||||
const cap = Number(process.env.JOB_RETRY_MAX_MS ?? 30000);
|
||||
const delay = Math.min(cap, base * 2 ** (attempt - 1));
|
||||
return Math.round(delay / 2 + random() * (delay / 2));
|
||||
}
|
||||
|
||||
export async function enqueueJob(
|
||||
tx: Tx,
|
||||
attachmentId: string,
|
||||
now: Date,
|
||||
): Promise<AttachmentJob> {
|
||||
const job = await tx.attachmentJob.create({
|
||||
data: {
|
||||
id: randomUUID(),
|
||||
attachmentId,
|
||||
pipelineVersion: PIPELINE_VERSION,
|
||||
status: 'queued',
|
||||
attempt: 1,
|
||||
maxAttempts: maxAttempts(),
|
||||
nextAttemptAt: now,
|
||||
createdAt: now,
|
||||
updatedAt: now,
|
||||
},
|
||||
});
|
||||
await tx.jobEvent.create({
|
||||
data: { jobId: job.id, attempt: 1, event: 'enqueued', createdAt: now },
|
||||
});
|
||||
await publishLater(tx, job, now);
|
||||
return job;
|
||||
}
|
||||
|
||||
export async function scheduleRetry(
|
||||
tx: Tx,
|
||||
job: AttachmentJob,
|
||||
reason: string,
|
||||
now: Date,
|
||||
event = 'retry_scheduled',
|
||||
): Promise<AttachmentJob> {
|
||||
const delay = backoffMs(job.attempt);
|
||||
const next = await tx.attachmentJob.update({
|
||||
where: { id: job.id },
|
||||
data: {
|
||||
status: 'queued',
|
||||
attempt: job.attempt + 1,
|
||||
nextAttemptAt: new Date(now.getTime() + delay),
|
||||
leaseOwner: null,
|
||||
leaseUntil: null,
|
||||
lastError: reason,
|
||||
},
|
||||
});
|
||||
await tx.attachment.update({
|
||||
where: { id: job.attachmentId },
|
||||
data: { status: 'queued' },
|
||||
});
|
||||
await tx.jobEvent.create({
|
||||
data: {
|
||||
jobId: job.id,
|
||||
attempt: job.attempt,
|
||||
event,
|
||||
detail: `${reason}; next attempt ${next.attempt} in ${delay} ms`,
|
||||
createdAt: now,
|
||||
},
|
||||
});
|
||||
await publishLater(tx, next, next.nextAttemptAt);
|
||||
return next;
|
||||
}
|
||||
|
||||
export async function markFailed(
|
||||
tx: Tx,
|
||||
job: AttachmentJob,
|
||||
errorCode: string,
|
||||
reason: string,
|
||||
now: Date,
|
||||
): Promise<void> {
|
||||
await tx.attachmentJob.update({
|
||||
where: { id: job.id },
|
||||
data: {
|
||||
status: 'failed',
|
||||
errorCode,
|
||||
lastError: reason,
|
||||
leaseOwner: null,
|
||||
leaseUntil: null,
|
||||
finishedAt: now,
|
||||
},
|
||||
});
|
||||
await tx.attachment.update({
|
||||
where: { id: job.attachmentId },
|
||||
data: { status: 'failed', errorCode },
|
||||
});
|
||||
await tx.jobEvent.create({
|
||||
data: {
|
||||
jobId: job.id,
|
||||
attempt: job.attempt,
|
||||
event: 'failed',
|
||||
detail: `${errorCode}: ${reason}`,
|
||||
createdAt: now,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
export async function retryOrFail(
|
||||
tx: Tx,
|
||||
job: AttachmentJob,
|
||||
reason: string,
|
||||
now: Date,
|
||||
event?: string,
|
||||
): Promise<'retry' | 'failed'> {
|
||||
if (job.attempt >= job.maxAttempts) {
|
||||
await markFailed(tx, job, RETRY_EXHAUSTED, reason, now);
|
||||
return 'failed';
|
||||
}
|
||||
await scheduleRetry(tx, job, reason, now, event);
|
||||
return 'retry';
|
||||
}
|
||||
|
||||
export async function recoverExpiredLeases(
|
||||
prisma: PrismaClient,
|
||||
now = new Date(),
|
||||
limit = 100,
|
||||
): Promise<number> {
|
||||
const expired = await prisma.attachmentJob.findMany({
|
||||
where: { status: 'processing', leaseUntil: { lt: now } },
|
||||
orderBy: { leaseUntil: 'asc' },
|
||||
take: limit,
|
||||
});
|
||||
let recovered = 0;
|
||||
for (const candidate of expired) {
|
||||
await prisma.$transaction(async (tx) => {
|
||||
const taken = await tx.attachmentJob.updateMany({
|
||||
where: {
|
||||
id: candidate.id,
|
||||
status: 'processing',
|
||||
leaseOwner: candidate.leaseOwner,
|
||||
leaseUntil: { lt: now },
|
||||
},
|
||||
data: { leaseOwner: null, leaseUntil: null, status: 'queued' },
|
||||
});
|
||||
if (taken.count === 0) {
|
||||
return;
|
||||
}
|
||||
await retryOrFail(tx, candidate, 'lease expired', now, 'lease_expired');
|
||||
recovered++;
|
||||
});
|
||||
}
|
||||
return recovered;
|
||||
}
|
||||
|
||||
export async function republishStale(
|
||||
prisma: PrismaClient,
|
||||
staleMs: number,
|
||||
now = new Date(),
|
||||
limit = 100,
|
||||
): Promise<number> {
|
||||
const cutoff = new Date(now.getTime() - staleMs);
|
||||
const stale = await prisma.$queryRaw<AttachmentJob[]>`
|
||||
SELECT j.id, j.attachment_id AS "attachmentId", j.pipeline_version AS "pipelineVersion", j.attempt
|
||||
FROM attachment_jobs j
|
||||
WHERE j.status = 'queued'
|
||||
AND j.next_attempt_at < ${cutoff}
|
||||
AND NOT EXISTS (
|
||||
SELECT 1 FROM outbox o
|
||||
WHERE o.job_id = j.id AND (o.published_at IS NULL OR o.published_at > ${cutoff})
|
||||
)
|
||||
ORDER BY j.next_attempt_at
|
||||
LIMIT ${limit}`;
|
||||
for (const job of stale) {
|
||||
await prisma.$transaction(async (tx) => {
|
||||
await tx.jobEvent.create({
|
||||
data: { jobId: job.id, attempt: job.attempt, event: 'republished', createdAt: now },
|
||||
});
|
||||
await publishLater(tx, job, now);
|
||||
});
|
||||
}
|
||||
return stale.length;
|
||||
}
|
||||
|
||||
export type ReplayResult =
|
||||
| 'replayed'
|
||||
| 'already_ready'
|
||||
| 'not_failed'
|
||||
| 'source_missing'
|
||||
| 'not_found';
|
||||
|
||||
export async function replayJob(
|
||||
prisma: PrismaClient,
|
||||
jobId: string,
|
||||
actor: string,
|
||||
now = new Date(),
|
||||
): Promise<ReplayResult> {
|
||||
return prisma.$transaction(async (tx) => {
|
||||
const job = await tx.attachmentJob.findUnique({ where: { id: jobId } });
|
||||
if (!job) {
|
||||
return 'not_found';
|
||||
}
|
||||
if (job.status === 'ready') {
|
||||
return 'already_ready';
|
||||
}
|
||||
if (job.status !== 'failed') {
|
||||
return 'not_failed';
|
||||
}
|
||||
const source = await tx.attachment.updateMany({
|
||||
where: { id: job.attachmentId, status: 'failed', stagingKey: { not: null } },
|
||||
data: { status: 'queued', errorCode: null },
|
||||
});
|
||||
if (source.count === 0) {
|
||||
return 'source_missing';
|
||||
}
|
||||
const taken = await tx.attachmentJob.updateMany({
|
||||
where: { id: jobId, status: 'failed' },
|
||||
data: {
|
||||
status: 'queued',
|
||||
attempt: 1,
|
||||
errorCode: null,
|
||||
nextAttemptAt: now,
|
||||
finishedAt: null,
|
||||
},
|
||||
});
|
||||
if (taken.count === 0) {
|
||||
throw new Error('job changed during replay');
|
||||
}
|
||||
await tx.jobEvent.create({
|
||||
data: {
|
||||
jobId,
|
||||
attempt: 1,
|
||||
event: 'replayed',
|
||||
detail: `by ${actor}; previous ${job.errorCode} after attempt ${job.attempt}`,
|
||||
createdAt: now,
|
||||
},
|
||||
});
|
||||
await publishLater(tx, { ...job, attempt: 1 }, now);
|
||||
return 'replayed';
|
||||
});
|
||||
}
|
||||
|
||||
async function publishLater(tx: Tx, job: AttachmentJob, availableAt: Date): Promise<void> {
|
||||
await tx.outboxEntry.create({
|
||||
data: {
|
||||
jobId: job.id,
|
||||
payload: jobMessage(job, availableAt) as unknown as Prisma.InputJsonObject,
|
||||
availableAt,
|
||||
createdAt: new Date(),
|
||||
},
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,298 @@
|
||||
import { Logger } from '@nestjs/common';
|
||||
import { metrics, SpanStatusCode, trace } from '@opentelemetry/api';
|
||||
import { Attachment, AttachmentJob, PrismaClient } from '@prisma/client';
|
||||
import { createHash, randomUUID } from 'crypto';
|
||||
import {
|
||||
ObjectMissingError,
|
||||
ObjectTooLargeError,
|
||||
S3Service,
|
||||
} from '../storage/s3.service';
|
||||
import { JOB_TYPE, JobMessage, PIPELINE_VERSION } from './job-message';
|
||||
import { markFailed, retryOrFail } from './job-store';
|
||||
import {
|
||||
InvalidImageError,
|
||||
RenderedVariant,
|
||||
RenderTimeoutError,
|
||||
renderVariants,
|
||||
} from './variants';
|
||||
|
||||
export type Outcome = 'ready' | 'retry' | 'failed' | 'skipped';
|
||||
|
||||
export interface ProcessorOptions {
|
||||
timeoutMs: number;
|
||||
leaseMs: number;
|
||||
maxBytes: number;
|
||||
maxPixels: number;
|
||||
}
|
||||
|
||||
export function processorOptions(): ProcessorOptions {
|
||||
const timeoutMs = Number(process.env.JOB_TIMEOUT_SECONDS ?? 60) * 1000;
|
||||
return {
|
||||
timeoutMs,
|
||||
leaseMs: Number(process.env.JOB_LEASE_SECONDS ?? timeoutMs / 1000 + 30) * 1000,
|
||||
maxBytes: Number(process.env.MAX_UPLOAD_BYTES ?? 10485760),
|
||||
maxPixels: Number(process.env.MAX_IMAGE_PIXELS ?? 40000000),
|
||||
};
|
||||
}
|
||||
|
||||
export interface ProcessorHooks {
|
||||
afterStore?: () => Promise<void>;
|
||||
afterCommit?: () => Promise<void>;
|
||||
}
|
||||
|
||||
class PermanentError extends Error {
|
||||
constructor(
|
||||
readonly code: string,
|
||||
reason: string,
|
||||
) {
|
||||
super(reason);
|
||||
}
|
||||
}
|
||||
|
||||
class TimeoutError extends Error {}
|
||||
|
||||
const tracer = trace.getTracer('messenger-worker');
|
||||
const timeToReady = metrics.getMeter('messenger-worker').createHistogram('attachments.time_to_ready', {
|
||||
description: 'From durable upload acceptance to the committed ready state',
|
||||
unit: 's',
|
||||
advice: {
|
||||
explicitBucketBoundaries: [
|
||||
0.1, 0.25, 0.5, 1, 2, 3, 5, 10, 20, 30, 45, 60, 90, 120, 150, 180, 240, 300, 450, 600,
|
||||
],
|
||||
},
|
||||
});
|
||||
|
||||
export class JobProcessor {
|
||||
private readonly logger = new Logger('JobProcessor');
|
||||
|
||||
constructor(
|
||||
private readonly prisma: PrismaClient,
|
||||
private readonly s3: S3Service,
|
||||
private readonly options: ProcessorOptions = processorOptions(),
|
||||
private readonly hooks: ProcessorHooks = {},
|
||||
) {}
|
||||
|
||||
async handle(message: JobMessage): Promise<Outcome> {
|
||||
const job = await this.prisma.attachmentJob.findUnique({
|
||||
where: { id: message.job_id },
|
||||
include: { attachment: true },
|
||||
});
|
||||
if (
|
||||
!job ||
|
||||
job.attachmentId !== message.attachment_id ||
|
||||
job.pipelineVersion !== message.pipeline_version ||
|
||||
job.pipelineVersion !== PIPELINE_VERSION
|
||||
) {
|
||||
this.logger.warn(`job ${message.job_id}: unknown or mismatched, dropped`);
|
||||
return 'skipped';
|
||||
}
|
||||
const lease = await this.claim(job);
|
||||
if (!lease) {
|
||||
if (job.status === 'ready') {
|
||||
await this.removeStaging(job.attachment);
|
||||
}
|
||||
this.logger.log(`job ${job.id}: ${job.status}, not due or leased elsewhere, skipped`);
|
||||
return 'skipped';
|
||||
}
|
||||
let variants: RenderedVariant[];
|
||||
try {
|
||||
variants = await this.withTimeout(this.produce(job.attachment));
|
||||
} catch (error) {
|
||||
return this.recordFailure(lease, error);
|
||||
}
|
||||
await this.hooks.afterStore?.();
|
||||
return this.commit(lease, job.attachment, variants);
|
||||
}
|
||||
|
||||
private async claim(job: AttachmentJob): Promise<AttachmentJob | null> {
|
||||
const now = new Date();
|
||||
const owner = randomUUID();
|
||||
return this.prisma.$transaction(async (tx) => {
|
||||
const taken = await tx.attachmentJob.updateMany({
|
||||
where: { id: job.id, status: 'queued', nextAttemptAt: { lte: now } },
|
||||
data: {
|
||||
status: 'processing',
|
||||
leaseOwner: owner,
|
||||
leaseUntil: new Date(now.getTime() + this.options.leaseMs),
|
||||
},
|
||||
});
|
||||
if (taken.count === 0) {
|
||||
return null;
|
||||
}
|
||||
await tx.attachment.update({
|
||||
where: { id: job.attachmentId },
|
||||
data: { status: 'processing' },
|
||||
});
|
||||
const leased = await tx.attachmentJob.findUniqueOrThrow({ where: { id: job.id } });
|
||||
await tx.jobEvent.create({
|
||||
data: { jobId: job.id, attempt: leased.attempt, event: 'processing', createdAt: now },
|
||||
});
|
||||
return leased;
|
||||
});
|
||||
}
|
||||
|
||||
private async produce(attachment: Attachment): Promise<RenderedVariant[]> {
|
||||
if (!attachment.stagingKey) {
|
||||
throw new PermanentError('staging_missing', 'staging object was already removed');
|
||||
}
|
||||
let original: Buffer;
|
||||
try {
|
||||
original = await this.s3.getBuffer(attachment.stagingKey, this.options.maxBytes);
|
||||
} catch (error) {
|
||||
if (error instanceof ObjectMissingError) {
|
||||
throw new PermanentError('staging_missing', 'staging object not found');
|
||||
}
|
||||
if (error instanceof ObjectTooLargeError) {
|
||||
throw new PermanentError('invalid_image', 'staging object exceeds size limit');
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
const sha256 = createHash('sha256').update(original).digest('hex');
|
||||
if (original.length !== attachment.sizeBytes || sha256 !== attachment.sha256) {
|
||||
throw new PermanentError('checksum_mismatch', 'staging object differs from accepted upload');
|
||||
}
|
||||
let variants: RenderedVariant[];
|
||||
try {
|
||||
variants = await tracer.startActiveSpan('attachment.render_variants', async (span) => {
|
||||
span.setAttribute('messenger.image.bytes', original.length);
|
||||
try {
|
||||
return await renderVariants(
|
||||
original,
|
||||
this.options.maxPixels,
|
||||
Math.ceil(this.options.timeoutMs / 1000),
|
||||
);
|
||||
} catch (error) {
|
||||
span.setStatus({ code: SpanStatusCode.ERROR, message: error instanceof Error ? error.name : 'error' });
|
||||
throw error;
|
||||
} finally {
|
||||
span.end();
|
||||
}
|
||||
});
|
||||
} catch (error) {
|
||||
if (error instanceof InvalidImageError) {
|
||||
throw new PermanentError('invalid_image', 'image cannot be decoded');
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
await this.s3.putBuffer(attachment.objectKey, original, attachment.contentType, sha256);
|
||||
for (const variant of variants) {
|
||||
await this.s3.putBuffer(
|
||||
variantKey(attachment.id, variant),
|
||||
variant.body,
|
||||
variant.contentType,
|
||||
variant.sha256,
|
||||
);
|
||||
}
|
||||
return variants;
|
||||
}
|
||||
|
||||
private async commit(
|
||||
lease: AttachmentJob,
|
||||
attachment: Attachment,
|
||||
variants: RenderedVariant[],
|
||||
): Promise<Outcome> {
|
||||
const now = new Date();
|
||||
const committed = await this.prisma.$transaction(async (tx) => {
|
||||
const owned = await tx.attachmentJob.updateMany({
|
||||
where: { id: lease.id, status: 'processing', leaseOwner: lease.leaseOwner },
|
||||
data: {
|
||||
status: 'ready',
|
||||
leaseOwner: null,
|
||||
leaseUntil: null,
|
||||
lastError: null,
|
||||
finishedAt: now,
|
||||
},
|
||||
});
|
||||
if (owned.count === 0) {
|
||||
return false;
|
||||
}
|
||||
await tx.attachmentVariant.createMany({
|
||||
data: variants.map((variant) => ({
|
||||
attachmentId: attachment.id,
|
||||
pipelineVersion: PIPELINE_VERSION,
|
||||
name: variant.name,
|
||||
contentType: variant.contentType,
|
||||
objectKey: variantKey(attachment.id, variant),
|
||||
sizeBytes: variant.body.length,
|
||||
width: variant.width,
|
||||
height: variant.height,
|
||||
sha256: variant.sha256,
|
||||
createdAt: now,
|
||||
})),
|
||||
skipDuplicates: true,
|
||||
});
|
||||
await tx.attachment.update({
|
||||
where: { id: attachment.id },
|
||||
data: { status: 'ready', errorCode: null },
|
||||
});
|
||||
await tx.jobEvent.create({
|
||||
data: { jobId: lease.id, attempt: lease.attempt, event: 'ready', createdAt: now },
|
||||
});
|
||||
return true;
|
||||
});
|
||||
if (!committed) {
|
||||
this.logger.warn(`job ${lease.id}: lease lost before commit, result left to owner`);
|
||||
return 'skipped';
|
||||
}
|
||||
timeToReady.record((now.getTime() - attachment.createdAt.getTime()) / 1000, {
|
||||
'job.type': JOB_TYPE,
|
||||
});
|
||||
await this.hooks.afterCommit?.();
|
||||
await this.removeStaging(attachment);
|
||||
return 'ready';
|
||||
}
|
||||
|
||||
private async recordFailure(lease: AttachmentJob, error: unknown): Promise<Outcome> {
|
||||
const now = new Date();
|
||||
const reason =
|
||||
error instanceof PermanentError
|
||||
? error.message
|
||||
: error instanceof TimeoutError || error instanceof RenderTimeoutError
|
||||
? 'processing timed out'
|
||||
: `temporary failure: ${error instanceof Error ? error.name : 'unknown'}`;
|
||||
this.logger.warn(`job ${lease.id} attempt ${lease.attempt}: ${reason}`);
|
||||
return this.prisma.$transaction(async (tx) => {
|
||||
const owned = await tx.attachmentJob.count({
|
||||
where: { id: lease.id, status: 'processing', leaseOwner: lease.leaseOwner },
|
||||
});
|
||||
if (owned === 0) {
|
||||
return 'skipped' as const;
|
||||
}
|
||||
if (error instanceof PermanentError) {
|
||||
await markFailed(tx, lease, error.code, reason, now);
|
||||
return 'failed' as const;
|
||||
}
|
||||
return retryOrFail(tx, lease, reason, now);
|
||||
});
|
||||
}
|
||||
|
||||
private async removeStaging(attachment: Attachment): Promise<void> {
|
||||
if (!attachment.stagingKey) {
|
||||
return;
|
||||
}
|
||||
try {
|
||||
await this.s3.delete(attachment.stagingKey);
|
||||
await this.prisma.attachment.update({
|
||||
where: { id: attachment.id },
|
||||
data: { stagingKey: null },
|
||||
});
|
||||
} catch {
|
||||
this.logger.warn(`staging cleanup postponed for ${attachment.id}`);
|
||||
}
|
||||
}
|
||||
|
||||
private withTimeout<T>(work: Promise<T>): Promise<T> {
|
||||
let timer: NodeJS.Timeout;
|
||||
const deadline = new Promise<never>((_, reject) => {
|
||||
timer = setTimeout(() => reject(new TimeoutError()), this.options.timeoutMs);
|
||||
});
|
||||
return Promise.race([work, deadline]).finally(() => clearTimeout(timer));
|
||||
}
|
||||
}
|
||||
|
||||
export function variantKey(
|
||||
attachmentId: string,
|
||||
variant: Pick<RenderedVariant, 'name' | 'extension'>,
|
||||
): string {
|
||||
return `attachments/${attachmentId}/v${PIPELINE_VERSION}/${variant.name}.${variant.extension}`;
|
||||
}
|
||||
@@ -0,0 +1,111 @@
|
||||
import { createHash } from 'crypto';
|
||||
import sharp, { Sharp } from 'sharp';
|
||||
|
||||
sharp.cache(false);
|
||||
sharp.concurrency(1);
|
||||
|
||||
const WHITE = { r: 255, g: 255, b: 255 };
|
||||
const JPEG_QUALITY = 82;
|
||||
const WEBP_QUALITY = 80;
|
||||
|
||||
export interface RenderedVariant {
|
||||
name: 'thumbnail' | 'crop' | 'optimized';
|
||||
contentType: string;
|
||||
extension: string;
|
||||
body: Buffer;
|
||||
width: number;
|
||||
height: number;
|
||||
sha256: string;
|
||||
}
|
||||
|
||||
export class InvalidImageError extends Error {
|
||||
constructor(reason: string) {
|
||||
super(reason);
|
||||
this.name = 'InvalidImageError';
|
||||
}
|
||||
}
|
||||
|
||||
export class RenderTimeoutError extends Error {
|
||||
constructor() {
|
||||
super('image processing exceeded its time limit');
|
||||
this.name = 'RenderTimeoutError';
|
||||
}
|
||||
}
|
||||
|
||||
export function fitInside(
|
||||
width: number,
|
||||
height: number,
|
||||
box: number,
|
||||
): { width: number; height: number } {
|
||||
if (width <= box && height <= box) {
|
||||
return { width, height };
|
||||
}
|
||||
if (width >= height) {
|
||||
return { width: box, height: Math.max(1, Math.floor((height * box) / width)) };
|
||||
}
|
||||
return { width: Math.max(1, Math.floor((width * box) / height)), height: box };
|
||||
}
|
||||
|
||||
export async function renderVariants(
|
||||
original: Buffer,
|
||||
maxPixels: number,
|
||||
timeoutSeconds = 0,
|
||||
): Promise<RenderedVariant[]> {
|
||||
const open = (): Sharp =>
|
||||
sharp(original, { failOn: 'error', limitInputPixels: maxPixels })
|
||||
.timeout({ seconds: timeoutSeconds })
|
||||
.rotate();
|
||||
try {
|
||||
const meta = await sharp(original, { limitInputPixels: maxPixels }).metadata();
|
||||
if (!meta.width || !meta.height) {
|
||||
throw new InvalidImageError('image has no dimensions');
|
||||
}
|
||||
const swapped = (meta.orientation ?? 1) >= 5;
|
||||
const width = swapped ? meta.height : meta.width;
|
||||
const height = swapped ? meta.width : meta.height;
|
||||
const thumb = fitInside(width, height, 256);
|
||||
const large = fitInside(width, height, 1280);
|
||||
const thumbnail = open()
|
||||
.resize(thumb.width, thumb.height, { fit: 'fill' })
|
||||
.flatten({ background: WHITE })
|
||||
.jpeg({ quality: JPEG_QUALITY });
|
||||
const crop = open()
|
||||
.resize(128, 128, { fit: 'cover', position: 'centre' })
|
||||
.flatten({ background: WHITE })
|
||||
.jpeg({ quality: JPEG_QUALITY });
|
||||
const optimized = open()
|
||||
.resize(large.width, large.height, { fit: 'fill' })
|
||||
.webp({ quality: WEBP_QUALITY, effort: 4 });
|
||||
return [
|
||||
await finish('thumbnail', 'image/jpeg', 'jpg', thumbnail),
|
||||
await finish('crop', 'image/jpeg', 'jpg', crop),
|
||||
await finish('optimized', 'image/webp', 'webp', optimized),
|
||||
];
|
||||
} catch (error) {
|
||||
if (error instanceof InvalidImageError) {
|
||||
throw error;
|
||||
}
|
||||
if (error instanceof Error && /timeout/i.test(error.message)) {
|
||||
throw new RenderTimeoutError();
|
||||
}
|
||||
throw new InvalidImageError(error instanceof Error ? error.message : 'decode failed');
|
||||
}
|
||||
}
|
||||
|
||||
async function finish(
|
||||
name: RenderedVariant['name'],
|
||||
contentType: string,
|
||||
extension: string,
|
||||
pipeline: Sharp,
|
||||
): Promise<RenderedVariant> {
|
||||
const { data, info } = await pipeline.toBuffer({ resolveWithObject: true });
|
||||
return {
|
||||
name,
|
||||
contentType,
|
||||
extension,
|
||||
body: data,
|
||||
width: info.width,
|
||||
height: info.height,
|
||||
sha256: createHash('sha256').update(data).digest('hex'),
|
||||
};
|
||||
}
|
||||
+39
@@ -0,0 +1,39 @@
|
||||
import './telemetry/register';
|
||||
import { Logger } from '@nestjs/common';
|
||||
import { NestFactory } from '@nestjs/core';
|
||||
import {
|
||||
FastifyAdapter,
|
||||
NestFastifyApplication,
|
||||
} from '@nestjs/platform-fastify';
|
||||
import { AppModule } from './app.module';
|
||||
import { configureApp } from './configure-app';
|
||||
import { setupSwagger } from './docs/swagger';
|
||||
import { enableGracefulShutdown } from './health/shutdown';
|
||||
import { registerHttpTelemetry } from './telemetry/http';
|
||||
import { JsonLogger } from './telemetry/json-logger';
|
||||
|
||||
async function bootstrap(): Promise<void> {
|
||||
const app = await NestFactory.create<NestFastifyApplication>(
|
||||
AppModule,
|
||||
new FastifyAdapter(),
|
||||
{
|
||||
bufferLogs: true,
|
||||
logger: new JsonLogger(),
|
||||
},
|
||||
);
|
||||
|
||||
configureApp(app);
|
||||
registerHttpTelemetry(app.getHttpAdapter().getInstance());
|
||||
setupSwagger(app);
|
||||
enableGracefulShutdown(app);
|
||||
|
||||
const host = process.env.APP_HOST ?? '0.0.0.0';
|
||||
const port = Number(process.env.APP_PORT ?? 8080);
|
||||
await app.listen(port, host);
|
||||
Logger.log(`Listening on http://${host}:${port}`, 'Bootstrap');
|
||||
}
|
||||
|
||||
bootstrap().catch((error) => {
|
||||
Logger.error('Fatal error during bootstrap', error);
|
||||
process.exit(1);
|
||||
});
|
||||
@@ -0,0 +1,41 @@
|
||||
import {
|
||||
ArrayMaxSize,
|
||||
IsArray,
|
||||
IsString,
|
||||
Validate,
|
||||
ValidateIf,
|
||||
ValidatorConstraint,
|
||||
ValidatorConstraintInterface,
|
||||
} from 'class-validator';
|
||||
import { CodePointLength } from '../../common/dto/code-point-length.validator';
|
||||
import { validate as isUuid } from '../../common/uuid';
|
||||
|
||||
@ValidatorConstraint({ name: 'uniqueUuidList' })
|
||||
class UniqueUuidList implements ValidatorConstraintInterface {
|
||||
validate(value: unknown): boolean {
|
||||
if (
|
||||
!Array.isArray(value) ||
|
||||
!value.every((id) => typeof id === 'string' && isUuid(id))
|
||||
) {
|
||||
return false;
|
||||
}
|
||||
const normalized = (value as string[]).map((id) => id.toLowerCase());
|
||||
return new Set(normalized).size === normalized.length;
|
||||
}
|
||||
|
||||
defaultMessage(): string {
|
||||
return 'attachment_ids must contain unique UUIDs';
|
||||
}
|
||||
}
|
||||
|
||||
export class CreateMessageDto {
|
||||
@IsString()
|
||||
@CodePointLength(0, 4000)
|
||||
text!: string;
|
||||
|
||||
@ValidateIf((dto: CreateMessageDto) => dto.attachment_ids !== undefined)
|
||||
@IsArray()
|
||||
@ArrayMaxSize(10)
|
||||
@Validate(UniqueUuidList)
|
||||
attachment_ids?: string[];
|
||||
}
|
||||
@@ -0,0 +1,26 @@
|
||||
import { Message } from '../../common/domain';
|
||||
|
||||
export interface MessageResponse {
|
||||
id: string;
|
||||
chat_id: string;
|
||||
sender_id: string;
|
||||
text: string;
|
||||
created_at: string;
|
||||
attachment_ids: string[];
|
||||
}
|
||||
|
||||
export interface MessagePageResponse {
|
||||
items: MessageResponse[];
|
||||
next_cursor: string | null;
|
||||
}
|
||||
|
||||
export function toMessageResponse(message: Message): MessageResponse {
|
||||
return {
|
||||
id: message.id,
|
||||
chat_id: message.chatId,
|
||||
sender_id: message.senderId,
|
||||
text: message.text,
|
||||
created_at: message.createdAt,
|
||||
attachment_ids: message.attachmentIds,
|
||||
};
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user