ЛР6 - БПИ2301 - Антонов - на 4 #58

Open
aegor wants to merge 8 commits from aegor/mtusi2026lab6:solution/БПИ2301-Antonov into main
137 changed files with 26711 additions and 15 deletions
+4
View File
@@ -16,5 +16,9 @@ backups
.venv
__pycache__
node_modules
dist
coverage
evidence
.DS_Store
.idea
.vscode
+4
View File
@@ -23,6 +23,10 @@ WORKER_CONCURRENCY=2
JOB_MAX_ATTEMPTS=5
JOB_TIMEOUT_SECONDS=60
STAGING_RETENTION_SECONDS=86400
JOB_LEASE_SECONDS=90
JOB_STALE_SECONDS=300
RECONCILE_GRACE_SECONDS=3600
RECONCILE_INTERVAL_SECONDS=300
OTEL_SERVICE_NAME=messenger-api
# Worker must set a distinct service.name.
OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4318
+2
View File
@@ -0,0 +1,2 @@
* text=auto eol=lf
*.png binary
+6
View File
@@ -0,0 +1,6 @@
{
"useTabs": true,
"tabWidth": 4,
"singleQuote": true,
"trailingComma": "all"
}
+45
View File
@@ -0,0 +1,45 @@
# syntax=docker/dockerfile:1
FROM node:20.17.0-slim AS base
RUN apt-get update \
&& apt-get install -y --no-install-recommends openssl \
&& rm -rf /var/lib/apt/lists/*
WORKDIR /app
FROM base AS deps
COPY package.json package-lock.json ./
RUN npm ci
FROM deps AS build
COPY prisma ./prisma
RUN npx prisma generate
COPY tsconfig.json tsconfig.build.json nest-cli.json ./
COPY src ./src
RUN npm run build
FROM build AS test
COPY contracts/openapi.json ./contracts/openapi.json
COPY tests/fixtures/sample.png ./tests/fixtures/sample.png
COPY test ./test
CMD ["npm", "test"]
FROM build AS migrate
CMD ["sh", "-c", "DATABASE_URL=\"$(cat \"$DATABASE_URL_FILE\")\" exec npx prisma migrate deploy"]
FROM build AS prod-deps
RUN npm prune --omit=dev
FROM base AS runtime
ENV NODE_ENV=production
COPY package.json ./
COPY --from=prod-deps /app/node_modules ./node_modules
COPY --from=build /app/dist ./dist
COPY contracts/openapi.json ./contracts/openapi.json
USER node
EXPOSE 8080
HEALTHCHECK --interval=5s --timeout=2s --start-period=5s --retries=3 \
CMD node -e "require('http').get('http://127.0.0.1:8080/health/core',r=>process.exit(r.statusCode===200?0:1)).on('error',()=>process.exit(1))"
CMD ["node", "dist/main.js"]
+190 -15
View File
@@ -1,37 +1,212 @@
# Отчёт — лабораторная 6
- Автор, группа: TODO
- Целевая оценка; выбранный трек 5, если нужен: TODO
- Commit/PR: TODO после публикации
- Использованные источники, библиотеки и помощь ИИ: TODO
- Автор, группа: Антонов Егор, БПИ2301
- Целевая оценка; выбранный трек 5, если нужен: **4**, трек 5 не заявляется
- Commit/PR: ветка `solution/БПИ2301-Antonov`; ссылка на PR — после публикации
- Использованные источники, библиотеки и помощь ИИ: документация OpenTelemetry (JS SDK 1.30/0.57, semantic conventions HTTP/messaging/DB, W3C Trace Context, Collector configuration: `memory_limiter`, `batch`, `sending_queue`), Prometheus (OTLP receiver, histograms and summaries, recording/alerting rules), Loki 3 (OTLP ingestion, structured metadata), Tempo 2.6, Grafana provisioning, Prisma (tracing, `metrics` preview). Новые библиотеки: `@opentelemetry/*` (api 1.9.0, sdk-node 0.57.2, sdk-metrics/trace 1.30.1, OTLP proto exporters 0.57.2, instrumentation-http 0.57.2, instrumentation-aws-sdk 0.49.1), `@prisma/instrumentation` 6.1.0. Перед началом работы выгружены и учтены замечания проверяющих к чужим PR этой лабораторной (см. «Сверка с замечаниями проверяющих»); код из чужих решений не использовался. ИИ-ассистент использовался как парный программист: код, конфигурации, сценарии инцидентов и текст отчёта по моим решениям с последующим ревью; проверки воспроизведены на своём стенде.
## Запуск из чистого clone
TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных.
На хосте нужен **только Docker** (Engine + Compose v2); python/node/make/sh на хосте не требуются, команды одинаковы в PowerShell и bash. Проверено: Docker 29.3.1, Docker Desktop (WSL2), 12 vCPU, 8 GiB RAM.
| Компонент | Версия |
| --- | --- |
| API, worker, relay (один образ) | `node:20.17.0-slim`, NestJS 10 / Fastify, Prisma 6.1.0, OpenTelemetry JS SDK 1.30.1 / 0.57.2 |
| OTel Collector | `otel/opentelemetry-collector-contrib:0.115.1`, закреплён digest |
| Метрики | `prom/prometheus:v3.1.0`, закреплён digest |
| Логи | `grafana/loki:3.3.2`, закреплён digest |
| Трассы | `grafana/tempo:2.6.1`, закреплён digest |
| Интерфейс | `grafana/grafana:11.4.0`, закреплён digest |
| Остальное (lab5) | RabbitMQ 4.0.5, PostgreSQL 17.2, Redis 7.4.1, HAProxy 3.0.6, MinIO (`pgsty/minio`), tools `docker:27.3.1-cli` |
```sh
# 1. Запуск всего стенда, включая Collector, Prometheus, Loki, Tempo и Grafana
docker compose up -d --build --wait
# 2. Публичные проверки (телеметрия включена)
docker compose run --rm --build tools make check
docker compose run --rm tools make test
# 3. Наблюдаемость
docker compose run --rm tools python3 scripts/trace-check.py # контрольный upload: trace в Tempo + логи в Loki
docker compose run --rm tools python3 scripts/telemetry-privacy.py # секреты/контент не в telemetry, нет ID в labels
docker compose run --rm tools python3 scripts/latency-compare.py # p50/p95/p99 dashboard против клиента
docker compose run --rm tools python3 scripts/dashboard-check.py # каждый запрос dashboard возвращает данные
# 4. Инциденты (каждый ждёт firing → resolved своего alert)
docker compose run --rm tools python3 scripts/incidents.py retry-trace # одна трасса через все попытки job
docker compose run --rm tools python3 scripts/incidents.py worker-stop # инцидент 1
docker compose run --rm tools python3 scripts/incidents.py db # инцидент 2: медленная и недоступная БД
docker compose run --rm tools python3 scripts/incidents.py telemetry # инцидент 3: недоступен telemetry backend
# 5. Все доказательства в evidence/ одной командой (~50 минут)
docker compose run --rm tools sh scripts/collect-evidence.sh
# Безопасная остановка (данные, метрики, логи, трассы, секреты сохраняются в volumes)
docker compose stop
# Намеренное удаление учебных данных, телеметрии и сгенерированных секретов
docker compose down -v
```
- **Grafana:** http://localhost:3000, анонимный вход с ролью Viewer (просмотр и Explore). Dashboard «Messenger overview» в папке Messenger создаётся provisioning'ом. Пароль `admin` генерирует `secrets-init`, посмотреть: `docker compose exec grafana cat /run/secrets/admin_password`.
- **Наружу опубликованы только** LB `127.0.0.1:8080` и Grafana `127.0.0.1:3000`. OTLP (4318), Prometheus, Loki, Tempo наружу не открыты; хранилища находятся во внутренней сети `telemetry` (`internal: true`), куда из сети приложения подключены только Collector и Grafana.
- `.env` необязателен; секреты — как в lab5 (`secrets-init`, раздельно по сервисам), добавлен `grafana/admin_password`.
## Архитектура и решения
TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab5, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации.
```
api ×2 ─┐ traces, metrics, logs (OTLP/HTTP protobuf, :4318)
relay ──┼──────────────▶ otel-collector ── memory_limiter → batch → exporters (ограниченная sending_queue, retry 60 s)
worker ─┘ default net │ telemetry net (internal)
├─ metrics ─▶ prometheus (/api/v1/otlp, TSDB 7d/1GB, rules + alerts) ◀─ scrape collector :8888
├─ logs ────▶ loki (/otlp, filesystem, 7d, trace_id/span_id = structured metadata)
└─ traces ──▶ tempo (OTLP, local blocks, 7d)
grafana (127.0.0.1:3000) ── datasources uid prometheus / loki / tempo, dashboard и связи trace ↔ logs из provisioning
```
**Зачем каждое хранилище:**
| Сигнал | Хранилище | На какой вопрос отвечает |
| --- | --- | --- |
| Метрики | Prometheus | «Сколько и как быстро?» — агрегаты во времени: RPS, доля ошибок, перцентили из histogram, глубина очереди, насыщение. Дёшево хранить долго, на них строятся SLO и alerts. Не отвечает, *какой* запрос был медленным. |
| Логи | Loki | «Что именно произошло?» — отдельные события с контекстом (`event`, `route`, `status`, `request_id`, `job_id`, причина ошибки). Индексируются только метки потока (`service_name`), остальное — structured metadata и текст. |
| Трассы | Tempo | «Где было потрачено время и как связаны шаги?» — дерево spans одного запроса через API, relay, брокер, worker, S3 и БД. Хранит отдельные трассы по ID, без индекса по содержимому. |
Collector — получатель и маршрутизатор, а не БД: он буферизует ограниченную очередь и при длительной недоступности хранилища отбрасывает данные. Grafana — только интерфейс и запросы к трём источникам.
**Инструментация** (`src/telemetry/`):
- `sdk.ts` — `NodeSDK` запускается первым импортом в `main.ts`, `worker.ts`, `relay.ts` (до загрузки `http`, Prisma, AWS SDK). Resource: `service.name` (`messenger-api`, `messenger-worker`, `messenger-relay` — relay вынесен в отдельный сервис, поэтому третье имя), `service.version` (из `package.json`), `service.instance.id` (hostname контейнера), `deployment.environment.name=lab` (из `OTEL_RESOURCE_ATTRIBUTES`). Автоинструментация: HTTP server (входящий `traceparent` продолжается, span kind SERVER), Prisma (`prisma:client:operation`, `prisma:engine:connection` — ожидание пула, `prisma:engine:db_query` — SQL без значений параметров), AWS SDK (`S3.PutObject` и т. д.). Исходящие HTTP-spans отключены (дублируют S3), health-запросы не трассируются, из `http.target`/`http.url` удаляется query (там `filename`). Sampler: корневые spans Prisma (опрос outbox relay, `SELECT 1` health) и служебные листовые spans сериализации не записываются.
- `json-logger.ts` — замена Nest `Logger` во всех трёх процессах: каждая запись — JSON с `timestamp`, `severity`, `service.name`, `event`, `logger`, полями события, `trace_id`, `span_id` (только у записываемого span), в stdout и как OTLP log record. Ключевые события: `http.request` (method, route, status, duration_ms, request_id), `http.error` (5xx, причина), `job.handled` (job_id, attachment_id, attempt, outcome, duration_ms).
- `http.ts` — метрики HTTP в hooks Fastify с шаблоном маршрута (`/api/v1/chats/{chat_id}/messages`); имя SERVER span = `METHOD route`.
- `db-pool.ts` — пул Prisma через `$metrics` (preview `metrics`), `relay.ts` — очередь, `worker.ts` — задания и насыщение, `processor.ts` — span рендеринга и время до ready.
**Метрики** (имена в Prometheus):
| Сигнал контракта | Метрика | Labels | Тип |
| --- | --- | --- | --- |
| HTTP requests | `http_server_requests_total` | `http_request_method`, `http_route` (шаблон), `http_response_status_class` | counter |
| HTTP duration | `http_server_request_duration_seconds` | те же | histogram, s; buckets 1 ms … 10 s |
| Jobs processed/failed | `jobs_processed_total` | `job_type`, `outcome` (`success`/`retry`/`failed`/`skipped`/`malformed`/`interrupted`) | counter |
| Job duration | `jobs_duration_seconds` | `job_type`, `outcome` | histogram, s; 10 ms … 120 s |
| Queue depth / oldest age | `jobs_queue_depth`, `jobs_queue_oldest_age_seconds` | `messaging_destination_name`, `state` (`queued`/`due`/`processing`) | gauge |
| DB pool | `db_client_connection_count`, `db_client_connection_pending_requests`, `db_client_connection_max` | `db_client_connection_state` (`used`/`idle`) | gauge |
| Worker busy/capacity | `worker_jobs_busy`, `worker_jobs_capacity` | — | gauge |
| CPU/RAM процесса | `process_cpu_time_seconds_total`, `process_memory_usage_bytes`, `process_memory_heap_used_bytes` | `cpu_mode` | counter, gauge |
| Time to ready | `attachments_time_to_ready_seconds` | `job_type` | histogram, s; 0,1 … 600 s |
Ресурсные атрибуты становятся `job` (= `service.name`) и `instance` (= `service.instance.id`), остальные — в `target_info`. В labels нет `user_id`, `chat_id`, `message_id`, `attachment_id`, `request_id`, raw URL, filename, текста: маршрут — шаблон, метод вне списка → `_OTHER`, несопоставленный путь → `unmatched`. Это проверено по всем сериям (`telemetry-privacy.py`).
**Почему очередь считается по БД, а не по брокеру.** Источник истины — `attachment_jobs` (lab5). Gauge отдаёт relay (а не worker), поэтому при остановленном worker метрика продолжает расти. `oldest_age` — возраст старейшего `queued`-задания, срок которого наступил (`now − next_attempt_at`): для нового задания это время с момента принятия upload, для повтора — с момента, когда повтор стал разрешён (ожидание backoff не считается отставанием).
**Buckets.** Длительности в секундах, границы рассчитаны на миллисекундные запросы (1, 2,5, 5, 10, 25, 50, 100, 250, 500 ms, 1, 2, 2,5, 5, 10 s) и на пороги SLO (0,5 s; 10 s для готовности). Перцентили считаются `histogram_quantile` по **сумме buckets всех реплик** (`sum by (le)`), а не усреднением перцентилей реплик. Сверка с клиентом — в разделе экспериментов.
**Сквозная трасса через очередь (W3C Trace Context).**
1. API продолжает входящий `traceparent` клиента (SERVER span).
2. В транзакции upload `jobMessage()` записывает в payload outbox поля `traceparent`/`tracestate` активного контекста — они уже есть в `jobs.schema.json`. Публикация отложенная, поэтому контекст должен пережить outbox: обычная автоинструментация amqplib связала бы сообщение с циклом опроса relay, а не с upload.
3. Relay для каждого сообщения создаёт span `attachment.jobs publish` (PRODUCER), **родитель — контекст из payload**, и кладёт новый `traceparent` в AMQP headers.
4. Worker создаёт `attachment.jobs process` (CONSUMER) с родителем из headers (запасной вариант — из payload); внутри — `S3.GetObject`, `attachment.render_variants`, `S3.PutObject` ×4, транзакции Prisma, `S3.DeleteObject`.
5. Повтор: `scheduleRetry` вызывается внутри CONSUMER span попытки N, новая запись outbox несёт его контекст, поэтому попытка N+1 — потомок попытки N в той же трассе. Выбрана семантика parent-child (одна логическая операция), а не links.
Отображение полей: payload `traceparent`/`tracestate` ↔ заголовки AMQP `traceparent`/`tracestate`; `messaging.message.id` = `job_id`, `messaging.job.attempt` = `attempt`. Parser сообщений проверяет формат `traceparent` (регулярное выражение схемы) и длину `tracestate` ≤ 512.
**Collector и поведение при backpressure** (`infra/otel/collector.yaml`): `memory_limiter` 160 MiB (+40 spike, контейнер 256 MiB) — при превышении receiver отказывает (`otelcol_receiver_refused_*`); `batch` 512/1024, 2 s; у каждого exporter `sending_queue` 200 batch в памяти, `retry_on_failure` до 60 s, затем данные отбрасываются (`otelcol_exporter_send_failed_*`). В SDK: batch processors с очередью 2048 записей и таймаутом экспорта 5 s, метрики раз в 10 s. Экспорт асинхронный: запросы пользователей не ждут телеметрию.
**SLI/SLO и alerts** — `infra/prometheus/rules/messenger.yml`, runbooks — [docs/runbooks.md](docs/runbooks.md).
| SLI | SLO | Окно |
| --- | --- | --- |
| Доступность: 1 − 5xx / бизнес-запросы (без `/health/*`) | ≥ 99% | 1 ч |
| Задержка: доля бизнес-запросов ≤ 0,5 s | ≥ 95% | 1 ч |
| Готовность: доля вложений `ready` за ≤ 10 s от принятия | ≥ 95% | 1 ч |
| Alert | Условие | `for` |
| --- | --- | --- |
| `MessengerHighErrorRate` | доля 5xx бизнес-запросов за 5m > 5% и трафик > 0,05 rps | 1m |
| `MessengerApiNotReady` | > 10 ответов 5xx `/health/core` за 1m (реплики выведены из LB) | 30s |
| `MessengerJobQueueLag` | старейшее готовое задание ждёт > 60 s | 1m |
| `MessengerSlowApi` | p95 бизнес-запросов за 5m > 1 s и трафик > 0,05 rps | 2m |
| `TelemetryExportFailing` | есть отказы отправки Collector или очередь exporter > 50% | 1m |
**Dashboard** (`infra/grafana/dashboards/messenger.json`, uid `messenger-overview`): SLO и история alerts; API (RPS по маршрутам, классы статуса, доля 5xx, p50/p95/p99, p95 по маршрутам); очередь и worker (глубина, возраст, busy/capacity/saturation, задания по исходу, длительность, время до ready); БД и ресурсы (пул used/waiting/max по сервисам, CPU, RSS); конвейер телеметрии (принято/отправлено/ошибки, очередь exporter); логи и таблица последних трасс. Datasources с явными UID `prometheus`, `loki`, `tempo` — те же UID в панелях; Tempo → Loki по `trace_id`, Loki → Tempo по полю `trace_id`.
**Перенесено из lab5** без изменения поведения: весь API, outbox, relay, worker, lease, retry, DLQ, replay, reconciler, тесты. **Изменено:** логирование стало JSON (лог HTTP-ошибки содержит шаблон маршрута вместо URL с query), сообщение задания может нести `traceparent`/`tracestate`, `PrismaService` регистрирует пул в метриках, preview `metrics` в `schema.prisma` (миграций нет).
**Соответствие `.env.example`:**
| Переменная | Где используется |
| --- | --- |
| переменные lab5 | как в lab5 |
| `OTEL_SERVICE_NAME` | задаётся для каждого сервиса в `compose.yaml`: `messenger-api`, `messenger-worker`, `messenger-relay` |
| `OTEL_EXPORTER_OTLP_ENDPOINT` | `http://otel-collector:4318`, общий якорь `x-otel` в `compose.yaml`; без него SDK не запускается (собственные тесты) |
| `OTEL_EXPORTER_OTLP_PROTOCOL` | `http/protobuf` (используются `*-otlp-proto` exporters) |
| `OTEL_RESOURCE_ATTRIBUTES` | `deployment.environment.name=lab` |
## Доказательства критериев
Файлы собраны `docker compose run --rm tools sh scripts/collect-evidence.sh` на commit `d83876f` с чистых volumes (`docker compose down -v`, затем `up`); в заголовке каждого файла — дата и commit. `incident-telemetry.txt` пересобран на commit `2718c3b`: фаза «Collector остановлен» удлинена с 60 до 300 s, код приложения не менялся.
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
| --- | --- | --- | --- |
| Публичный smoke | make test | TODO | TODO |
| Остальные пункты уровня 3 | TODO | TODO | TODO |
| Все пункты уровня 4, если заявлен | TODO | TODO | TODO |
| Выбранный трек 5, если заявлен | TODO | TODO | TODO |
TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены.
| Публичный smoke с телеметрией | `make check`, `make test` через LB | `make check` PASS; `PASS: lab6 public API scenario` при включённой телеметрии; запросы одной сессии обслуживают обе реплики | [evidence/smoke.txt](evidence/smoke.txt) |
| **3**: минимум lab5 сохранён | собственные тесты, `make test-persistence`, `make test-failover`, `check-s3.sh`, сценарии `job-scenario.py` | 201/201 тест; persistence PASS; failover PASS (128 записей, 0 ошибок, восстановление 3,57 s); S3 PASS; все 7 сценариев lab5 (crash-after-store/commit, outbox-gap, relay-crash, retry-replay, broker) воспроизводятся с телеметрией | [evidence/tests.txt](evidence/tests.txt), [evidence/persistence.txt](evidence/persistence.txt), [evidence/failover.txt](evidence/failover.txt), [evidence/s3.txt](evidence/s3.txt), [evidence/jobs.txt](evidence/jobs.txt) |
| **3**: Collector, хранилища трёх сигналов, Grafana в стенде; конфиги и provisioning в Git; volumes | `docker compose ps`, images, сети, datasources | сервисы `otel-collector`, `prometheus`, `loki`, `tempo`, `grafana` работают, образы закреплены по digest; наружу только `127.0.0.1:8080` и `127.0.0.1:3000`; хранилища только в сети `telemetry` (`internal=true`); volumes `prometheusdata`, `lokidata`, `tempodata`, `grafanadata`; datasources с UID `prometheus`/`loki`/`tempo` | [evidence/container.txt](evidence/container.txt), [evidence/telemetry.txt](evidence/telemetry.txt) |
| **3**: API и worker отдают OTLP; RPS, ошибки, длительность API, jobs и длительность worker, логи, трассы видны в Grafana | `dashboard-check.py` | dashboard загружен из provisioning (папка Messenger); все запросы панелей возвращают данные, кроме истории alerts (в спокойном состоянии alerts нет); «время до ready» за последнюю минуту — NaN, т. к. новых готовых вложений в эту минуту не было | [evidence/telemetry.txt](evidence/telemetry.txt) |
| **3**: логи с trace_id/span_id; для контрольного запроса находятся trace и лог, для job — trace и лог worker | `trace-check.py` | upload с клиентским `traceparent` → trace из 79 spans: SERVER API (родитель — span клиента) → PRODUCER relay → CONSUMER worker → S3 и SQL; в Loki по `trace_id` — `http.request` API и `job.handled` worker, у каждого свой `span_id` | [evidence/telemetry.txt](evidence/telemetry.txt) |
| **3**: пароли, токены, тексты сообщений не экспортируются; нет ID в labels | `telemetry-privacy.py` | пароль, session token, Basic-учётка, заголовок Authorization, текст сообщения, filename — 0 вхождений в 6 трассах этих запросов и в Loki (контроль: 7 строк логов этих запросов найдены); 893 серии, 0 с UUID в значениях labels; `http_route` — только шаблоны | [evidence/telemetry.txt](evidence/telemetry.txt) |
| **4**: W3C context API → broker → worker, producer/consumer spans, цепочка до S3/DB включая retry | `incidents.py retry-trace`, `trace-check.py` | одна загрузка — одна трасса: 3 пары PRODUCER → CONSUMER (`attempt` 1, 2, 3); в попытках 1–2 `S3.GetObject` с ошибкой (S3 остановлен), попытка 3 — S3 Put ×4, COMMIT, ready; события `enqueued → processing → retry_scheduled → processing → retry_scheduled → processing → ready` | [evidence/incident-retry-trace.txt](evidence/incident-retry-trace.txt) |
| **4**: p50/p95/p99 из histogram, доля ошибок, очередь и возраст, пул БД, насыщение worker, CPU/RAM, time to ready | `dashboard-check.py`, `latency-compare.py` | панели p50/p95/p99 (`sum by (le)`), доля 5xx, глубина/возраст очереди, пул used/waiting/max, busy/capacity, CPU, RSS, time to ready возвращают данные; клиент / histogram: p50 18,3 / 19,7 ms, p95 89,2 / 92,2 ms, p99 163,3 / 176,8 ms (3411 POST, 113 rps) | [evidence/telemetry.txt](evidence/telemetry.txt) |
| **4**: SLI/SLO с окном; alert rules на error rate и queue lag, firing → resolved | `promtool`, `incidents.py worker-stop`, `db` | `promtool`: 13 правил; все `health ok`; `MessengerJobQueueLag` и `MessengerApiNotReady` прошли pending → firing → inactive | [evidence/telemetry.txt](evidence/telemetry.txt), [evidence/incident-worker-stop.txt](evidence/incident-worker-stop.txt), [evidence/incident-db.txt](evidence/incident-db.txt) |
| **4**: Collector с memory_limiter, batch, ограниченной очередью; остановка backend не ломает API и не растит RAM | `incidents.py telemetry` | Loki+Tempo остановлены: 0 ошибок API, очередь exporter упирается в 200, `memory_limiter` отказывает в приёме, `TelemetryExportFailing` firing → resolved; Collector остановлен 300 s: 0 ошибок API, память API ограничена и освобождается после восстановления; перезапусков/OOM нет | [evidence/incident-telemetry.txt](evidence/incident-telemetry.txt) |
| **4**: инцидент 1 — worker остановлен | `incidents.py worker-stop` | см. «Эксперименты и отказы» | [evidence/incident-worker-stop.txt](evidence/incident-worker-stop.txt) |
| **4**: инцидент 2 — медленная/недоступная БД отличима от медленного handler | `incidents.py db` | см. «Эксперименты и отказы» | [evidence/incident-db.txt](evidence/incident-db.txt) |
| Собственные тесты | `docker compose -f compose.test.yaml run --rm --build tests` | `Tests: 201 passed, 201 total` | [evidence/tests.txt](evidence/tests.txt) |
## Эксперименты и отказы
TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения.
Стенд: Docker Desktop WSL2, 12 vCPU, 8 GiB; лимиты контейнеров — в `compose.yaml` и `evidence/container.txt` (API 0,5 CPU / 256M ×2, worker 1 CPU / 768M, Collector 0,5 CPU / 256M, хранилища по 0,5 CPU / 512M). Каждый сценарий выполнен один раз в составе `collect-evidence.sh`. Фоновая нагрузка в инцидентах — `Traffic` из `scripts/incidents.py`: 4–6 потоков closed loop через LB, POST сообщения и GET истории по очереди, пауза 0,05–0,2 s; файл для job — `sample.png` 320×200. Клиентские p50/p95/p99 считаются по успешным ответам фазы.
**Сверка перцентилей** (`latency-compare.py`, 30 s, 4 потока, POST сообщений): клиент 3411 успешных запросов, 113 rps, 0 ошибок; p50 / p95 / p99 — 18,3 / 89,2 / 163,3 ms у клиента и 19,7 / 92,2 / 176,8 ms по `histogram_quantile` (`increase` за окно). Серверная гистограмма не включает сеть и клиент, значение интерполируется внутри bucket (50–100 ms и 100–250 ms), поэтому расхождение в пределах ширины bucket. Счётчик Prometheus за окно (3661) больше клиентского: `increase()` экстраполирует до границ окна.
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
| --- | --- | --- | --- |
| TODO | TODO | TODO | TODO |
| **Инцидент 1:** `docker compose stop worker`, 12 загрузок по одной в 10 s | 202 продолжаются; растёт возраст старейшего задания; alert по lag; после старта backlog уменьшается | все загрузки 202; `jobs_queue_depth{state="due"}` до 12, `oldest_age` до 132,5 s (обновляется с шагом экспорта метрик), `worker_busy` 0, серии worker перестают обновляться; `MessengerJobQueueLag` pending → **firing через 151 s** простоя | после `start worker` очередь пуста через ≤ 20 s (12 → 2 → 0), alert **resolved через 30 s**; 12 из 12 `ready`; фактическое время до ready по БД 41,9–152,1 s, histogram p99 176 s (bucket 150–180 s); потерь нет |
| **Инцидент 2а — медленный handler (контроль):** upload PNG 1700×1700 шума, 8,3 MiB | время в обработчике и S3, БД быстрая | trace 525 ms: вызовы БД 40 ms (SQL 22 ms, ожидание пула 1 ms), S3 246 ms, сам обработчик (полное декодирование) 240 ms | — |
| **Инцидент 2б — медленная БД:** `docker update --cpus 0.05 postgres` на 150 s под нагрузкой | рост задержки, причина в SQL | 0 ошибок; клиент p95 23 → 196 ms, p99 49 → 637 ms; p95 в Prometheus (1m) до ~250 ms; trace 790 ms: БД 788 ms (SQL 781 ms, пул 0 ms), обработчик 1 ms; trace 505 ms: SQL 498 ms. `MessengerSlowApi` (порог p95 > 1 s) не сработал — деградация ниже порога | после `--cpus 1.0` p95 вернулся к ~24 ms за 1–2 мин |
| **Инцидент 2в — БД недоступна:** `docker pause postgres` на ~105 s | 503 вместо зависаний; alert | API: ожидание пула 5 s → `503 database unavailable` (trace 5005 ms: `prisma:engine:connection` 5000 ms); `/health/core` 503 → HAProxy выводит обе реплики, клиенты получают `503 no API replica available` (1910 ответов) и 3 × 504; доля 5xx в метриках API не растёт — запросы не доходят до API; `MessengerApiNotReady` **firing**; в Loki `http.error` / `database unavailable` | после `unpause` реплики вернулись в LB за секунды, 25 ответов 503 в хвосте; alert **resolved через ~60 s**; p95 23 ms; данных не потеряно (неподтверждённые записи получили 503) |
| **Инцидент 3а — Loki и Tempo остановлены** до срабатывания alert (~150 s) под нагрузкой 6 потоков | API работает; Collector ограничивает очередь и память; потери считаются | клиент: 13 901 успешный, **0 ошибок**, p95 43 ms (до — 53 ms); очередь exporter 0 → **200 из 200**; `send_failed` ~1350 spans и ~1100 log records в минуту; затем `memory_limiter` отказывает в приёме (`refused` до ~89 000 в минуту; в логе Collector: «above soft limit, refusing data», «above hard limit, forcing GC» при 120–170 MiB кучи); `TelemetryExportFailing` **firing**. Перезапусков и OOM Collector нет | после `start loki tempo` очередь опустела за ~15 s, alert **resolved** через ~120 s; трассы и логи за время простоя потеряны (посчитаны в `send_failed`/`refused`), логи остаются в `docker compose logs` |
| **Инцидент 3б — Collector остановлен на 300 s** под той же нагрузкой | API работает; экспорт падает быстро (таймаут 5 s), очереди SDK ограничены | клиент: 30 904 успешных, **0 ошибок**, p95 38 ms; память контейнеров API (`docker stats`) 183 → 211 MiB и 174 → 197 MiB из 256M: +9 MiB за первые 45 s, затем ~3 MiB/мин; worker стабильно ~57 MiB; перезапусков и OOM нет | после `start otel-collector` экспорт возобновился без перезапуска API; через ~1 мин память реплик 159 и 149 MiB — буферы телеметрии освобождены. Отдельный предварительный замер (5 мин, `load.py` 6 потоков): RSS до ~228 MiB и сброс сборщиком мусора до ~185 MiB на 280-й секунде, 38 141 запрос без ошибок |
| S3 остановлен на время первых попыток job (`retry-trace`) | повторы в одной трассе | см. критерий W3C выше: 3 попытки в одной трассе | ready через ~9 s после upload |
| `restart.sh`, `stop-one.sh`, сценарии lab5 | как в lab5 | PASS; failover 128 записей, 0 ошибок, восстановление 3,57 s | потерь нет |
## Сверка с замечаниями проверяющих
Замечания к PR других студентов этой лабораторной выгружены **до начала работы** (2026-11-18, 42 PR, два содержательных отзыва) и учтены при проектировании. Код из чужих решений не использовался.
| Замечание к чужому PR | Наш результат |
| --- | --- |
| Стандартные buckets (0, 5, 10 … s) дают p95 4,75 s при реальных 5 ms; сверить график с контрольным измерением | buckets от 1 ms; `latency-compare.py` сравнивает с клиентом — p95 89 ms у клиента и 92 ms по histogram. Та же проблема найдена у нас для time to ready (p99 296 s при реальных ≤ 160 s между границами 120 и 300 s) и исправлена добавлением границ 150/180/240 s |
| Datasources без UID → rules и панели с UID `prometheus`/`loki` в состоянии `data source not found` | UID заданы в provisioning и используются во всех панелях; `telemetry.txt` показывает UID, все запросы dashboard возвращают данные. Alerts вычисляет Prometheus, а не Grafana, health правил `ok` |
| Alert «остановка worker» по числу retry не срабатывает, т. к. без worker retry не появляются; нужны queue depth и oldest age | alert по возрасту старейшего задания из БД, gauge отдаёт relay, поэтому он растёт при мёртвом worker; firing → resolved показан |
| API создаёт новый trace вместо продолжения клиентского `traceparent`, нет SERVER spans | SERVER span — потомок клиентского span (`trace-check.py`: `parent=client`) |
| В worker виден только общий span, нет S3/DB | `S3.GetObject`/`PutObject`/`DeleteObject`, `prisma:*`, `attachment.render_variants` — потомки CONSUMER span |
| Нет DB pool, worker busy/capacity, histogram времени до ready | все три есть и на dashboard |
## Ограничения и следующие шаги
TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза.
- **Окна SLO по 1 ч** — учебный масштаб, чтобы эффект был виден за время защиты; в эксплуатации — 28–30 дней и multi-window burn-rate alerts.
- **Перцентили — оценка внутри bucket**, точность ограничена шириной bucket; события медленнее 10 s (HTTP) и 600 s (ready) попадают в `+Inf`.
- **`MessengerHighErrorRate` не видит ответы LB**, когда все реплики выведены (HAProxy отвечает 503 сам) — это закрыто `MessengerApiNotReady`; метрик HAProxy в стенде нет.
- **Запас памяти Collector небольшой.** `memory_limiter` ограничивает кучу Go (порог 160 MiB, отказ в приёме с 120 MiB), но RSS процесса больше кучи: метрика `otelcol_process_memory_rss` при остановленных Loki/Tempo показывала до 287 MiB (счёт процесса, с разделяемыми страницами), лимит контейнера 256M при этом не превышен, OOM не было. При большей нагрузке стоит снизить `limit_mib` или поднять лимит контейнера.
- **Память API без Collector** за 5 минут выросла на ~25 MiB с замедлением и освободилась после восстановления; более длинный простой не измерялся. Ограничители — очереди SDK по 2048 записей, таймаут экспорта 5 s, лимит контейнера 256M.
- **Потери телеметрии при недоступности backend допустимы:** после 60 s повторов и заполнения очереди Collector отбрасывает данные; логи при этом остаются в stdout контейнеров. Персистентной очереди (file storage) у Collector нет.
- **Пропуски метрик при остановке процесса** (worker stop): серии worker перестают обновляться — это видно, но отдельного alert на отсутствие серий нет; признак остановки — рост `oldest_age` при `busy = 0`.
- **Метрики очереди зависят от relay:** если упал relay, исчезают и они (но не задания — они в БД).
- **Один экземпляр** Collector и каждого хранилища, без репликации; TLS между компонентами — lab7.
- Уведомления (Alertmanager, почта) не настроены — по заданию не требуются; состояние alerts видно в Prometheus API и на dashboard.
## Ответы на вопросы защиты
- **Лог, метрика и trace — на какие вопросы отвечает каждый?** Метрика: «сколько, как часто, насколько медленно в целом» — дешёвые агрегаты для dashboard, SLO и alerts. Лог: «что именно случилось в этом событии» — причина ошибки, ID запроса/задания. Trace: «где ушло время и через какие компоненты прошёл этот конкретный запрос», включая асинхронное продолжение в worker.
- **Почему `chat_id` в labels может исчерпать хранилище?** Каждое уникальное сочетание labels — отдельная временная серия со своим индексом и чанками; для histogram это ещё ×(число buckets + 2). Число чатов растёт неограниченно, значит число серий тоже (cardinality explosion): растут память и диск Prometheus, замедляются запросы. ID место в логах и трассах.
- **Как очередь переносит причинную связь?** Контекст W3C (`traceparent` = версия, trace-id, parent span-id, флаги) кладётся в сообщение: у нас в payload outbox (переживает отложенную публикацию) и в AMQP headers. Producer span — потомок контекста upload, consumer span — потомок producer; повтор несёт контекст предыдущей попытки. Это parent-child; при batch-обработке нескольких сообщений правильнее links.
- **Почему среднее не заменяет p99 и почему нельзя усреднять p99 реплик?** Среднее скрывает хвост: 99 быстрых и 1 очень медленный запрос дают «хорошее» среднее, а пользователь из хвоста ждёт долго. Перцентиль — не аддитивная величина: среднее p99 двух реплик не равно p99 объединённого трафика (реплика с малым трафиком и плохим хвостом искажает результат). Правильно — суммировать buckets histogram всех реплик (`sum by (le)`) и считать `histogram_quantile` по сумме.
+55
View File
@@ -0,0 +1,55 @@
name: messenger-lab6-test
services:
postgres:
image: postgres:17.2-alpine
environment:
POSTGRES_USER: test
POSTGRES_PASSWORD: test
POSTGRES_DB: messenger_test
tmpfs:
- /var/lib/postgresql/data
healthcheck:
test: ["CMD", "pg_isready", "-U", "test", "-d", "messenger_test"]
interval: 2s
timeout: 3s
retries: 15
redis:
image: redis:7.4.1-alpine
command: ["redis-server", "--save", "", "--appendonly", "no"]
s3:
image: pgsty/minio:RELEASE.2026-08-04T00-00-00Z@sha256:b6bfe7239bfc83fb90d31612d9704d86039dd714f7904b3f1ad68f211e602372
command: ["server", "/data", "--address", ":9000"]
environment:
MINIO_ROOT_USER: test-root
MINIO_ROOT_PASSWORD: test-root-password
MINIO_BROWSER: "off"
tmpfs:
- /data
healthcheck:
test: ["CMD", "curl", "-fsS", "-o", "/dev/null", "http://127.0.0.1:9000/minio/health/live"]
interval: 2s
timeout: 3s
retries: 15
tests:
build:
context: .
target: test
image: mtusi-messenger-lab6-test:dev
environment:
DATABASE_URL: postgresql://test:test@postgres:5432/messenger_test?connection_limit=20
CACHE_URL: redis://redis:6379/0
S3_ENDPOINT: http://s3:9000
S3_BUCKET: messenger-test
S3_ACCESS_KEY_ID: test-root
S3_SECRET_ACCESS_KEY: test-root-password
depends_on:
postgres:
condition: service_healthy
redis:
condition: service_started
s3:
condition: service_healthy
+448
View File
@@ -0,0 +1,448 @@
name: messenger-lab6
x-otel: &otel
OTEL_EXPORTER_OTLP_ENDPOINT: ${OTEL_EXPORTER_OTLP_ENDPOINT:-http://otel-collector:4318}
OTEL_EXPORTER_OTLP_PROTOCOL: http/protobuf
OTEL_RESOURCE_ATTRIBUTES: ${OTEL_RESOURCE_ATTRIBUTES:-deployment.environment.name=lab}
services:
secrets-init:
image: alpine:3.20.3
volumes:
- secrets:/secrets
command:
- sh
- -ec
- |
cd /secrets
mkdir -p postgres redis api migrate s3 s3-init reconciler broker rabbitmq relay worker grafana
for entry in pg_password:postgres:24 redis_password:redis:24 s3_root_user:s3:10 s3_root_password:s3:20 s3_access_key_id:api:10 s3_secret_access_key:api:20 broker_password:broker:24 admin_password:grafana:16; do
file=$${entry%%:*}; rest=$${entry#*:}; dir=$${rest%%:*}; size=$${rest##*:}
[ -s "$$dir/$$file" ] || head -c "$$size" /dev/urandom | od -An -tx1 | tr -d ' \n' > "$$dir/$$file"
done
rm -f s3-init/* rabbitmq/* relay/* worker/*
cp s3/s3_root_user s3/s3_root_password api/s3_access_key_id api/s3_secret_access_key s3-init/
db_url() { printf 'postgresql://messenger:%s@postgres:5432/messenger?connection_limit=%s&pool_timeout=5&connect_timeout=5&socket_timeout=10&application_name=%s&options=-c%%20statement_timeout%%3D5000' "$$(cat postgres/pg_password)" "$$1" "$$2"; }
db_url 10 api > api/database_url
db_url 5 worker > worker/database_url
db_url 3 relay > relay/database_url
printf 'redis://:%s@redis:6379/0' "$$(cat redis/redis_password)" > api/cache_url
printf 'postgresql://messenger:%s@postgres:5432/messenger' "$$(cat postgres/pg_password)" > migrate/database_url
printf 'amqp://messenger:%s@rabbitmq:5672/' "$$(cat broker/broker_password)" > worker/broker_url
cp worker/broker_url relay/
printf 'default_user = messenger\ndefault_pass = %s\nloopback_users = none\nlog.console = true\nlisteners.tcp.default = 5672\n' "$$(cat broker/broker_password)" > rabbitmq/10-messenger.conf
cp api/database_url api/s3_access_key_id api/s3_secret_access_key reconciler/
cp api/s3_access_key_id api/s3_secret_access_key worker/
chmod 0444 postgres/* redis/* api/* migrate/* s3/* s3-init/* reconciler/* broker/* rabbitmq/* relay/* worker/* grafana/*
deploy:
resources:
limits: { cpus: "0.1", memory: 32M }
postgres:
image: postgres:17.2-alpine
environment:
POSTGRES_USER: messenger
POSTGRES_DB: messenger
POSTGRES_PASSWORD_FILE: /run/secrets/pg_password
volumes:
- pgdata:/var/lib/postgresql/data
- { type: volume, source: secrets, target: /run/secrets, read_only: true, volume: { subpath: postgres } }
deploy:
resources:
limits: { cpus: "1.0", memory: 512M }
healthcheck:
test: ["CMD", "pg_isready", "-U", "messenger", "-d", "messenger"]
interval: 5s
timeout: 3s
retries: 10
depends_on:
secrets-init:
condition: service_completed_successfully
restart: unless-stopped
redis:
image: redis:7.4.1-alpine
user: redis
command:
- sh
- -c
- exec redis-server --save '' --appendonly no --maxmemory 64mb --maxmemory-policy allkeys-lru --requirepass "$$(cat /run/secrets/redis_password)"
volumes:
- { type: volume, source: secrets, target: /run/secrets, read_only: true, volume: { subpath: redis } }
deploy:
resources:
limits: { cpus: "0.25", memory: 128M }
healthcheck:
test: ["CMD-SHELL", "redis-cli --no-auth-warning -a \"$$(cat /run/secrets/redis_password)\" ping | grep -q PONG"]
interval: 5s
timeout: 3s
retries: 10
depends_on:
secrets-init:
condition: service_completed_successfully
restart: unless-stopped
s3:
image: pgsty/minio:RELEASE.2026-08-04T00-00-00Z@sha256:b6bfe7239bfc83fb90d31612d9704d86039dd714f7904b3f1ad68f211e602372
command: ["server", "/data", "--address", ":9000"]
environment:
MINIO_ROOT_USER_FILE: /run/secrets/s3_root_user
MINIO_ROOT_PASSWORD_FILE: /run/secrets/s3_root_password
MINIO_BROWSER: "off"
volumes:
- s3data:/data
- { type: volume, source: secrets, target: /run/secrets, read_only: true, volume: { subpath: s3 } }
deploy:
resources:
limits: { cpus: "0.5", memory: 512M }
healthcheck:
test: ["CMD", "curl", "-fsS", "-o", "/dev/null", "http://127.0.0.1:9000/minio/health/live"]
interval: 5s
timeout: 3s
retries: 10
depends_on:
secrets-init:
condition: service_completed_successfully
restart: unless-stopped
s3-init:
image: pgsty/mc:RELEASE.2026-10-26T00-00-00Z@sha256:ce62c9053c69ce12a527a322345b5ad9f44b5f8a7b8189f82e516f2e3f30f1d8
entrypoint: ["sh", "-ec"]
command:
- |
cd /run/secrets
export MC_HOST_root="http://$$(cat s3_root_user):$$(cat s3_root_password)@s3:9000"
mc mb --ignore-existing "root/$$S3_BUCKET"
mc anonymous set none "root/$$S3_BUCKET"
cat > /tmp/policy.json <<POLICY
{"Version":"2012-10-17","Statement":[
{"Effect":"Allow","Action":["s3:ListBucket","s3:GetBucketLocation","s3:ListBucketMultipartUploads"],"Resource":["arn:aws:s3:::$$S3_BUCKET"]},
{"Effect":"Allow","Action":["s3:GetObject","s3:PutObject","s3:DeleteObject","s3:AbortMultipartUpload","s3:ListMultipartUploadParts"],"Resource":["arn:aws:s3:::$$S3_BUCKET/*"]}
]}
POLICY
mc admin policy create root messenger-api /tmp/policy.json
mc admin user add root "$$(cat s3_access_key_id)" "$$(cat s3_secret_access_key)" > /dev/null
mc admin policy attach root messenger-api --user "$$(cat s3_access_key_id)" > /dev/null
echo "bucket $$S3_BUCKET ready"
environment:
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
volumes:
- { type: volume, source: secrets, target: /run/secrets, read_only: true, volume: { subpath: s3-init } }
deploy:
resources:
limits: { cpus: "0.25", memory: 256M }
depends_on:
s3:
condition: service_healthy
migrate:
build:
context: .
target: migrate
image: mtusi-messenger-lab6-migrate:dev
environment:
DATABASE_URL_FILE: /run/secrets/database_url
volumes:
- { type: volume, source: secrets, target: /run/secrets, read_only: true, volume: { subpath: migrate } }
deploy:
resources:
limits: { cpus: "0.5", memory: 512M }
depends_on:
postgres:
condition: service_healthy
api:
build:
context: .
target: runtime
image: mtusi-messenger-lab6-api:dev
environment:
APP_HOST: 0.0.0.0
APP_PORT: 8080
APP_ENV: ${APP_ENV:-lab}
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600}
IDEMPOTENCY_TTL_SECONDS: ${IDEMPOTENCY_TTL_SECONDS:-86400}
SHUTDOWN_DELAY_MS: 3000
DATABASE_URL_FILE: /run/secrets/database_url
CACHE_URL_FILE: /run/secrets/cache_url
S3_ENDPOINT: http://s3:9000
S3_REGION: ${S3_REGION:-us-east-1}
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id
S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key
MAX_UPLOAD_BYTES: ${MAX_UPLOAD_BYTES:-10485760}
MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS:-40000000}
JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5}
<<: *otel
OTEL_SERVICE_NAME: messenger-api
volumes:
- { type: volume, source: secrets, target: /run/secrets, read_only: true, volume: { subpath: api } }
deploy:
replicas: ${API_REPLICAS:-2}
resources:
limits: { cpus: "0.5", memory: 256M }
stop_grace_period: 10s
depends_on:
migrate:
condition: service_completed_successfully
redis:
condition: service_started
s3-init:
condition: service_completed_successfully
restart: unless-stopped
reconciler:
build:
context: .
target: runtime
image: mtusi-messenger-lab6-api:dev
command: ["node", "dist/reconcile.js", "--apply", "--loop", "--grace-seconds=${RECONCILE_GRACE_SECONDS:-3600}", "--interval-seconds=${RECONCILE_INTERVAL_SECONDS:-300}"]
environment:
DATABASE_URL_FILE: /run/secrets/database_url
STAGING_RETENTION_SECONDS: ${STAGING_RETENTION_SECONDS:-86400}
S3_ENDPOINT: http://s3:9000
S3_REGION: ${S3_REGION:-us-east-1}
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id
S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key
volumes:
- { type: volume, source: secrets, target: /run/secrets, read_only: true, volume: { subpath: reconciler } }
deploy:
resources:
limits: { cpus: "0.25", memory: 128M }
healthcheck:
disable: true
depends_on:
migrate:
condition: service_completed_successfully
s3-init:
condition: service_completed_successfully
restart: unless-stopped
rabbitmq:
image: rabbitmq:4.0.5-alpine@sha256:e045b5166b9188a38eee5c99411709aa1f0ffa2ac75664053e64cb69411f7c32
hostname: rabbitmq
volumes:
- rabbitmqdata:/var/lib/rabbitmq
- { type: volume, source: secrets, target: /etc/rabbitmq/conf.d, read_only: true, volume: { subpath: rabbitmq } }
deploy:
resources:
limits: { cpus: "0.5", memory: 512M }
healthcheck:
test: ["CMD", "rabbitmq-diagnostics", "-q", "check_port_connectivity"]
interval: 10s
timeout: 10s
retries: 12
start_period: 20s
stop_grace_period: 30s
depends_on:
secrets-init:
condition: service_completed_successfully
restart: unless-stopped
relay:
build:
context: .
target: runtime
image: mtusi-messenger-lab6-api:dev
command: ["node", "dist/relay.js"]
environment:
DATABASE_URL_FILE: /run/secrets/database_url
BROKER_URL_FILE: /run/secrets/broker_url
JOB_STALE_SECONDS: ${JOB_STALE_SECONDS:-300}
RELAY_FAULT: ${RELAY_FAULT:-}
<<: *otel
OTEL_SERVICE_NAME: messenger-relay
volumes:
- { type: volume, source: secrets, target: /run/secrets, read_only: true, volume: { subpath: relay } }
deploy:
resources:
limits: { cpus: "0.25", memory: 128M }
healthcheck:
disable: true
stop_grace_period: 10s
depends_on:
migrate:
condition: service_completed_successfully
rabbitmq:
condition: service_healthy
restart: unless-stopped
worker:
build:
context: .
target: runtime
image: mtusi-messenger-lab6-api:dev
command: ["node", "dist/worker.js"]
environment:
DATABASE_URL_FILE: /run/secrets/database_url
BROKER_URL_FILE: /run/secrets/broker_url
S3_ENDPOINT: http://s3:9000
S3_REGION: ${S3_REGION:-us-east-1}
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id
S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key
MAX_UPLOAD_BYTES: ${MAX_UPLOAD_BYTES:-10485760}
MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS:-40000000}
WORKER_CONCURRENCY: ${WORKER_CONCURRENCY:-2}
JOB_TIMEOUT_SECONDS: ${JOB_TIMEOUT_SECONDS:-60}
JOB_LEASE_SECONDS: ${JOB_LEASE_SECONDS:-90}
WORKER_FAULT: ${WORKER_FAULT:-}
<<: *otel
OTEL_SERVICE_NAME: messenger-worker
volumes:
- { type: volume, source: secrets, target: /run/secrets, read_only: true, volume: { subpath: worker } }
deploy:
resources:
limits: { cpus: "1.0", memory: 768M }
healthcheck:
disable: true
stop_grace_period: 30s
depends_on:
migrate:
condition: service_completed_successfully
rabbitmq:
condition: service_healthy
s3-init:
condition: service_completed_successfully
restart: unless-stopped
lb:
image: haproxy:3.0.6-alpine
ports:
- "127.0.0.1:${APP_PORT:-8080}:8080"
volumes:
- ./infra/haproxy/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro
deploy:
resources:
limits: { cpus: "0.25", memory: 64M }
depends_on:
api:
condition: service_healthy
restart: unless-stopped
otel-collector:
image: otel/opentelemetry-collector-contrib:0.115.1@sha256:d2da12c4336a79758826700be9e21ecf4a9f7d945b7f8a58ba55ee3fa45427c8
command: ["--config=/etc/otelcol/collector.yaml"]
volumes:
- ./infra/otel/collector.yaml:/etc/otelcol/collector.yaml:ro
networks: [default, telemetry]
deploy:
resources:
limits: { cpus: "0.5", memory: 256M }
restart: unless-stopped
prometheus:
image: prom/prometheus:v3.1.0@sha256:6559acbd5d770b15bb3c954629ce190ac3cbbdb2b7f1c30f0385c4e05104e218
command:
- --config.file=/etc/prometheus/prometheus.yml
- --storage.tsdb.path=/prometheus
- --storage.tsdb.retention.time=7d
- --storage.tsdb.retention.size=1GB
- --web.enable-otlp-receiver
volumes:
- ./infra/prometheus:/etc/prometheus:ro
- prometheusdata:/prometheus
networks: [telemetry]
deploy:
resources:
limits: { cpus: "0.5", memory: 512M }
healthcheck:
test: ["CMD", "wget", "-q", "-O", "/dev/null", "http://127.0.0.1:9090/-/ready"]
interval: 5s
timeout: 3s
retries: 20
restart: unless-stopped
loki:
image: grafana/loki:3.3.2@sha256:8af2de1abbdd7aa92b27c9bcc96f0f4140c9096b507c77921ffddf1c6ad6c48f
command: ["-config.file=/etc/loki/loki.yaml"]
volumes:
- ./infra/loki/loki.yaml:/etc/loki/loki.yaml:ro
- lokidata:/loki
networks: [telemetry]
deploy:
resources:
limits: { cpus: "0.5", memory: 512M }
restart: unless-stopped
tempo:
image: grafana/tempo:2.6.1@sha256:ef4384fce6e8ad22b95b243d8fc165628cda655376fd50e7850536ad89d71d50
command: ["-config.file=/etc/tempo/tempo.yaml"]
volumes:
- ./infra/tempo/tempo.yaml:/etc/tempo/tempo.yaml:ro
- tempodata:/var/tempo
networks: [telemetry]
deploy:
resources:
limits: { cpus: "0.5", memory: 512M }
restart: unless-stopped
grafana:
image: grafana/grafana:11.4.0@sha256:d8ea37798ccc41061a62ab080f2676dda6bf7815558499f901bdb0f533a456fb
ports:
- "127.0.0.1:${GRAFANA_PORT:-3000}:3000"
environment:
GF_SECURITY_ADMIN_USER: admin
GF_SECURITY_ADMIN_PASSWORD__FILE: /run/secrets/admin_password
GF_AUTH_ANONYMOUS_ENABLED: "true"
GF_AUTH_ANONYMOUS_ORG_ROLE: Viewer
GF_USERS_VIEWERS_CAN_EDIT: "true"
GF_ANALYTICS_REPORTING_ENABLED: "false"
GF_ANALYTICS_CHECK_FOR_UPDATES: "false"
GF_NEWS_NEWS_FEED_ENABLED: "false"
volumes:
- ./infra/grafana/provisioning:/etc/grafana/provisioning:ro
- ./infra/grafana/dashboards:/var/lib/grafana/dashboards:ro
- grafanadata:/var/lib/grafana
- { type: volume, source: secrets, target: /run/secrets, read_only: true, volume: { subpath: grafana } }
networks: [default, telemetry]
deploy:
resources:
limits: { cpus: "0.5", memory: 256M }
healthcheck:
test: ["CMD", "wget", "-q", "-O", "/dev/null", "http://127.0.0.1:3000/api/health"]
interval: 5s
timeout: 3s
retries: 20
depends_on:
secrets-init:
condition: service_completed_successfully
prometheus:
condition: service_healthy
loki:
condition: service_started
tempo:
condition: service_started
restart: unless-stopped
tools:
profiles: [tools]
build:
dockerfile_inline: |
FROM docker:27.3.1-cli
RUN apk add --no-cache python3 make git && git config --global --add safe.directory /work
WORKDIR /work
image: mtusi-messenger-lab6-tools:dev
environment:
BASE_URL: http://lb:8080
volumes:
- .:/work
- /var/run/docker.sock:/var/run/docker.sock
deploy:
resources:
limits: { cpus: "1.0", memory: 256M }
volumes:
pgdata:
s3data:
rabbitmqdata:
prometheusdata:
lokidata:
tempodata:
grafanadata:
secrets:
networks:
telemetry:
internal: true
+36
View File
@@ -0,0 +1,36 @@
{
"openapi": "3.1.0",
"info": {
"title": "MTUSI Messenger — расширения лабораторной 5",
"version": "1.5.0",
"description": "Обратно совместимое дополнение к contracts/openapi.json (см. COURSE.md, раздел «Контракт и совместимость»). Обязательный контракт не меняется. Обоснование — REPORT.md, раздел «Критичные зависимости и проверка готовности»."
},
"paths": {
"/health/core": {
"get": {
"operationId": "getCoreReadiness",
"summary": "Служебная проверка для балансировщика и Docker healthcheck: может ли реплика обслуживать API без вложений",
"description": "В отличие от /health/ready не учитывает S3. Проверяет PostgreSQL и то, что процесс не останавливается. Падение общего S3 не выводит реплики из балансировки: сообщения продолжают работать, вложения отвечают 503.",
"responses": {
"200": {
"description": "Реплика готова",
"content": {
"application/json": {
"schema": {
"type": "object",
"properties": { "status": { "const": "ok" } },
"required": ["status"],
"additionalProperties": false
}
}
}
},
"503": {
"description": "PostgreSQL недоступен или процесс останавливается; тело в общем формате Error",
"content": { "application/json": { "schema": { "type": "object" } } }
}
}
}
}
}
}
+75
View File
@@ -0,0 +1,75 @@
# Runbooks
Alert rules: [infra/prometheus/rules/messenger.yml](../infra/prometheus/rules/messenger.yml). Состояние: Grafana → dashboard «Messenger overview» → панель «Alerts»; или Prometheus API `GET /api/v1/alerts` (из контейнера `tools`: `wget -qO- http://grafana:3000/api/datasources/proxy/uid/prometheus/api/v1/alerts`). Уведомления наружу не отправляются.
## SLO
| SLI | Формула | SLO | Окно |
| --- | --- | --- | --- |
| Доступность API | 1 − 5xx / все бизнес-запросы (без `/health/*`) | ≥ 99% | 1 ч (учебный стенд; в эксплуатации — 30 дней) |
| Задержка API | доля бизнес-запросов с длительностью ≤ 0,5 с (bucket `le="0.5"`) | ≥ 95% | 1 ч |
| Готовность вложений | доля вложений, ставших `ready` за ≤ 10 с от принятия upload (bucket `le="10"`) | ≥ 95% | 1 ч |
## MessengerHighErrorRate
**Условие:** доля 5xx среди бизнес-запросов за 5 минут > 5% при трафике > 0,05 rps, держится 1 минуту. При SLO 99% это сжигание бюджета ошибок в 5 раз быстрее допустимого.
**Проверить:**
1. Панель «Ответы по классам статуса» и «p95 по маршрутам»: какой маршрут даёт 5xx.
2. Loki: `{service_name="messenger-api"} | event="http.error"` — поле `message` (`database unavailable`, `object storage unavailable`, `Internal server error`), `route`, `request_id`. По `trace_id` записи — trace в Tempo.
3. В trace: на каком span ошибка — `prisma:engine:db_query`, `S3.PutObject` или код обработчика.
4. `docker compose ps` — живы ли postgres, s3, redis; `/health/ready` реплик.
**Действия:** восстановить зависимость (`docker compose start postgres` и т. п.); при ошибке кода — откат образа. Readiness исключает реплику без БД из HAProxy автоматически.
**Resolved:** доля 5xx за 5 минут снова ниже 5%.
## MessengerApiNotReady
**Условие:** больше 10 ответов 5xx на `/health/core` (проверка HAProxy раз в секунду, `SELECT 1` в PostgreSQL) за минуту, держится 30 с. Штатная остановка реплики даёт 3–4 таких ответа (3 с drain) и alert не вызывает; недоступная БД — около 140 в минуту на две реплики.
**Почему отдельный alert:** когда все реплики не готовы, HAProxy сам отвечает клиентам `503 no API replica available`; эти ответы не проходят через API и не входят в `http_server_requests_total`, поэтому доля 5xx бизнес-запросов может оставаться нулевой.
**Проверить:** Loki `{service_name="messenger-api"} | event="http.error"` — `route=/health/core`, `message` (`database unavailable` или `shutting down`); `docker compose ps postgres`; панель «Пул БД» (рост `waiting`).
**Действия:** восстановить PostgreSQL (`docker compose start postgres` / `docker compose unpause postgres`). После первых успешных проверок (`rise 2`) HAProxy возвращает реплики.
**Resolved:** проверки готовности снова 200 в течение минуты.
## MessengerJobQueueLag
**Условие:** старейшее задание в состоянии `queued` с наступившим `next_attempt_at` ждёт дольше 60 с, держится 1 минуту. Источник — таблица `attachment_jobs` (метрика `jobs_queue_oldest_age_seconds` от relay), а не брокер: она показывает и задания, сообщения которых ещё не опубликованы.
**Проверить:**
1. Панель «Worker: занято / ёмкость». `busy = capacity` — worker насыщен (медленная обработка); метрик worker нет совсем — процесс остановлен; `busy = 0` при растущей очереди — worker не получает сообщения (брокер, relay).
2. «Задания по исходу»: много `retry` — зависимость worker (S3/БД) отвечает ошибками.
3. `docker compose ps worker relay rabbitmq`; логи worker в Loki: `{service_name="messenger-worker"}`.
4. Возраст отдельных заданий: `SELECT id, status, attempt, next_attempt_at FROM attachment_jobs WHERE status='queued' ORDER BY next_attempt_at LIMIT 10;`
**Действия:** запустить worker (`docker compose start worker`) или добавить реплик/увеличить `WORKER_CONCURRENCY` при насыщении. Задания не теряются: при старте worker обрабатывает backlog.
**Resolved:** backlog разобран, возраст старейшего задания < 60 с.
## MessengerSlowApi
**Условие:** p95 бизнес-запросов за 5 минут > 1 с при трафике > 0,05 rps, держится 2 минуты.
**Как отличить медленную БД от медленного обработчика:**
- медленная/недоступная БД: растёт `db_client_connection_pending_requests` (ожидание пула) и/или в trace медленного запроса основное время — spans `prisma:engine:db_query`; ошибки `database unavailable` (503);
- медленный обработчик: пул свободен, spans БД короткие, время приходится на сам SERVER span или S3/рендеринг; растёт CPU API.
**Действия:** для БД — проверить postgres (`pg_stat_activity`, блокировки, ресурсы контейнера); для обработчика — профилировать маршрут, найденный в «p95 по маршрутам».
**Resolved:** p95 за 5 минут < 1 с.
## TelemetryExportFailing
**Условие:** Collector получает ошибки при отправке в Loki/Tempo/Prometheus или очередь exporter заполнена больше чем наполовину, держится 1 минуту.
**Что происходит:** API и worker продолжают работать: SDK отправляет телеметрию асинхронно с таймаутом 5 с и ограниченными очередями (2048 записей), Collector держит ограниченную очередь (`queue_size: 200` batch на exporter), повторяет отправку до 60 с и затем отбрасывает данные. `memory_limiter` (160 MiB) отказывает в приёме при нехватке памяти. Потери видны в `otelcol_exporter_send_failed_*` и `otelcol_receiver_refused_*`.
**Проверить:** панель «Конвейер телеметрии»; `docker compose ps loki tempo prometheus`; логи Collector `docker compose logs otel-collector`.
**Действия:** запустить хранилище (`docker compose start loki tempo`). Данные, отброшенные за время недоступности, не восстанавливаются; логи остаются в stdout контейнеров (`docker compose logs`).
**Resolved:** отказов отправки нет, очередь exporter опустела.
+318
View File
@@ -0,0 +1,318 @@
# Стенд: сервисы, порты, лимиты, секреты, миграции, топология брокера
# date: 2026-11-20T14:30:52Z
# commit: d83876f
$ docker compose ps -a --format table {{.Service}}\t{{.Name}}\t{{.Image}}\t{{.Status}}\t{{.Ports}}
SERVICE NAME IMAGE STATUS PORTS
api messenger-lab6-api-1 mtusi-messenger-lab6-api:dev Up 13 seconds (healthy) 8080/tcp
api messenger-lab6-api-2 mtusi-messenger-lab6-api:dev Up 13 seconds (healthy) 8080/tcp
grafana messenger-lab6-grafana-1 grafana/grafana:11.4.0@sha256:d8ea37798ccc41061a62ab080f2676dda6bf7815558499f901bdb0f533a456fb Up 19 seconds (healthy) 127.0.0.1:3000->3000/tcp
lb messenger-lab6-lb-1 haproxy:3.0.6-alpine Up 7 seconds 127.0.0.1:8080->8080/tcp
loki messenger-lab6-loki-1 grafana/loki:3.3.2@sha256:8af2de1abbdd7aa92b27c9bcc96f0f4140c9096b507c77921ffddf1c6ad6c48f Up 25 seconds 3100/tcp
migrate messenger-lab6-migrate-1 mtusi-messenger-lab6-migrate:dev Exited (0) 14 seconds ago
otel-collector messenger-lab6-otel-collector-1 otel/opentelemetry-collector-contrib:0.115.1@sha256:d2da12c4336a79758826700be9e21ecf4a9f7d945b7f8a58ba55ee3fa45427c8 Up 25 seconds 4317-4318/tcp, 55678-55679/tcp
postgres messenger-lab6-postgres-1 postgres:17.2-alpine Up 24 seconds (healthy) 5432/tcp
prometheus messenger-lab6-prometheus-1 prom/prometheus:v3.1.0@sha256:6559acbd5d770b15bb3c954629ce190ac3cbbdb2b7f1c30f0385c4e05104e218 Up 25 seconds (healthy) 9090/tcp
rabbitmq messenger-lab6-rabbitmq-1 rabbitmq:4.0.5-alpine@sha256:e045b5166b9188a38eee5c99411709aa1f0ffa2ac75664053e64cb69411f7c32 Up 24 seconds (healthy) 4369/tcp, 5671-5672/tcp, 15691-15692/tcp, 25672/tcp
reconciler messenger-lab6-reconciler-1 mtusi-messenger-lab6-api:dev Up 13 seconds 8080/tcp
redis messenger-lab6-redis-1 redis:7.4.1-alpine Up 24 seconds (healthy) 6379/tcp
relay messenger-lab6-relay-1 mtusi-messenger-lab6-api:dev Up 7 seconds 8080/tcp
s3 messenger-lab6-s3-1 pgsty/minio:RELEASE.2026-08-04T00-00-00Z@sha256:b6bfe7239bfc83fb90d31612d9704d86039dd714f7904b3f1ad68f211e602372 Up 24 seconds (healthy) 9000/tcp
s3-init messenger-lab6-s3-init-1 pgsty/mc:RELEASE.2026-10-26T00-00-00Z@sha256:ce62c9053c69ce12a527a322345b5ad9f44b5f8a7b8189f82e516f2e3f30f1d8 Exited (0) 14 seconds ago
secrets-init messenger-lab6-secrets-init-1 alpine:3.20.3 Exited (0) 24 seconds ago
tempo messenger-lab6-tempo-1 grafana/tempo:2.6.1@sha256:ef4384fce6e8ad22b95b243d8fc165628cda655376fd50e7850536ad89d71d50 Up 25 seconds
tools messenger-lab6-tools-run-68bd7ed38f92 mtusi-messenger-lab6-tools:dev Up 5 seconds
worker messenger-lab6-worker-1 mtusi-messenger-lab6-api:dev Up 7 seconds 8080/tcp
## CPU/RAM-лимиты (NanoCpus: 1e9 = 1 CPU; Memory в байтах)
$ sh -c docker inspect -f '{{.Name}} cpus={{.HostConfig.NanoCpus}} memory={{.HostConfig.Memory}}' $(docker compose ps -aq)
/messenger-lab6-api-1 cpus=500000000 memory=268435456
/messenger-lab6-api-2 cpus=500000000 memory=268435456
/messenger-lab6-grafana-1 cpus=500000000 memory=268435456
/messenger-lab6-lb-1 cpus=250000000 memory=67108864
/messenger-lab6-loki-1 cpus=500000000 memory=536870912
/messenger-lab6-migrate-1 cpus=500000000 memory=536870912
/messenger-lab6-otel-collector-1 cpus=500000000 memory=268435456
/messenger-lab6-postgres-1 cpus=1000000000 memory=536870912
/messenger-lab6-prometheus-1 cpus=500000000 memory=536870912
/messenger-lab6-rabbitmq-1 cpus=500000000 memory=536870912
/messenger-lab6-reconciler-1 cpus=250000000 memory=134217728
/messenger-lab6-redis-1 cpus=250000000 memory=134217728
/messenger-lab6-relay-1 cpus=250000000 memory=134217728
/messenger-lab6-s3-1 cpus=500000000 memory=536870912
/messenger-lab6-s3-init-1 cpus=250000000 memory=268435456
/messenger-lab6-secrets-init-1 cpus=100000000 memory=33554432
/messenger-lab6-tempo-1 cpus=500000000 memory=536870912
/messenger-lab6-tools-run-68bd7ed38f92 cpus=1000000000 memory=268435456
/messenger-lab6-worker-1 cpus=1000000000 memory=805306368
## Опубликованные порты: только LB и Grafana на 127.0.0.1; OTLP, Prometheus, Loki, Tempo наружу не открыты
$ sh -c docker compose ps --format '{{.Service}} {{.Ports}}' | grep -v '^\S* *$' || true
api 8080/tcp
api 8080/tcp
grafana 127.0.0.1:3000->3000/tcp
lb 127.0.0.1:8080->8080/tcp
loki 3100/tcp
otel-collector 4317-4318/tcp, 55678-55679/tcp
postgres 5432/tcp
prometheus 9090/tcp
rabbitmq 4369/tcp, 5671-5672/tcp, 15691-15692/tcp, 25672/tcp
reconciler 8080/tcp
redis 6379/tcp
relay 8080/tcp
s3 9000/tcp
worker 8080/tcp
## Сети телеметрии: хранилища только во внутренней сети telemetry
$ sh -c for s in otel-collector prometheus loki tempo grafana api worker; do printf '%s: ' $s; docker inspect -f '{{range $k, $v := .NetworkSettings.Networks}}{{$k}} {{end}}' $(docker compose ps -q $s | head -1); done
otel-collector: messenger-lab6_default messenger-lab6_telemetry
prometheus: messenger-lab6_telemetry
loki: messenger-lab6_telemetry
tempo: messenger-lab6_telemetry
grafana: messenger-lab6_default messenger-lab6_telemetry
api: messenger-lab6_default
worker: messenger-lab6_default
$ docker network inspect -f {{.Name}} internal={{.Internal}} messenger-lab6_telemetry
messenger-lab6_telemetry internal=true
## Секреты: каждый сервис видит только свои файлы (содержимое не выводится)
$ docker compose exec -T postgres ls /run/secrets
pg_password
$ docker compose exec -T redis ls /run/secrets
redis_password
$ docker compose exec -T s3 ls /run/secrets
s3_root_password
s3_root_user
$ docker compose exec -T api ls /run/secrets
cache_url
database_url
s3_access_key_id
s3_secret_access_key
$ docker compose exec -T reconciler ls /run/secrets
database_url
s3_access_key_id
s3_secret_access_key
$ docker compose exec -T relay ls /run/secrets
broker_url
database_url
$ docker compose exec -T worker ls /run/secrets
broker_url
database_url
s3_access_key_id
s3_secret_access_key
$ docker compose exec -T grafana ls /run/secrets
admin_password
$ docker compose exec -T rabbitmq ls /etc/rabbitmq/conf.d
10-messenger.conf
$ docker compose exec -T lb sh -c ls /run/secrets 2>/dev/null || echo "no /run/secrets"
no /run/secrets
## Версии
$ docker compose exec -T lb haproxy -v
HAProxy version 3.0.6-c2c0090 2024/11/07 - https://haproxy.org/
Status: long-term supported branch - will stop receiving fixes around Q2 2029.
Known bugs: http://www.haproxy.org/bugs/bugs-3.0.6.html
Running on: Linux 5.15.153.1-microsoft-standard-WSL2 #1 SMP Fri Mar 29 23:14:13 UTC 2024 x86_64
$ docker compose exec -T postgres postgres --version
postgres (PostgreSQL) 17.2
$ docker compose exec -T redis redis-server --version
Redis server v=7.4.1 sha=00000000:0 malloc=jemalloc-5.3.0 bits=64 build=a93ce81042b3a6d7
$ docker compose exec -T s3 minio --version
minio version RELEASE.2026-08-04T00-00-00Z (commit-id=d88f46ccee345a9c2fabe2d221d9a9e56bc11aec)
Runtime: go1.26.5 linux/amd64
License: GNU AGPLv3 - https://www.gnu.org/licenses/agpl-3.0.html
Copyright: 2015-2026 MinIO, Inc.
$ docker compose exec -T rabbitmq rabbitmqctl -q version
4.0.5
$ sh -c docker compose images otel-collector prometheus loki tempo grafana
CONTAINER REPOSITORY TAG IMAGE ID SIZE
messenger-lab6-grafana-1 grafana/grafana <none> d8ea37798ccc 133MB
messenger-lab6-loki-1 grafana/loki <none> 8af2de1abbdd 33.3MB
messenger-lab6-otel-collector-1 otel/opentelemetry-collector-contrib <none> d2da12c4336a 74.3MB
messenger-lab6-prometheus-1 prom/prometheus <none> 6559acbd5d77 117MB
messenger-lab6-tempo-1 grafana/tempo <none> ef4384fce6e8 52.4MB
## Топология брокера: durable quorum-очереди, delivery-limit и DLQ
$ docker compose exec -T rabbitmq rabbitmqctl -q list_queues name type durable arguments messages
name type durable arguments messages
attachment.jobs quorum true [{"x-queue-type","quorum"},{"x-delivery-limit",20},{"x-dead-letter-exchange",[]},{"x-dead-letter-routing-key","attachment.jobs.dlq"}] 0
attachment.jobs.dlq quorum true [{"x-queue-type","quorum"}] 0
$ docker compose exec -T rabbitmq rabbitmqctl -q list_consumers queue_name prefetch_count ack_required
queue_name ack_required prefetch_count
attachment.jobs true 2
## Таблицы
$ docker compose exec -T postgres psql -U messenger -d messenger -c \d attachments -c \d attachment_variants -c \d attachment_jobs -c \d outbox -c \d job_events
Table "public.attachments"
Column | Type | Collation | Nullable | Default
--------------+-----------------------------+-----------+----------+-------------------
id | uuid | | not null |
chat_id | uuid | | not null |
owner_id | uuid | | not null |
filename | character varying(255) | | not null |
content_type | character varying(32) | | not null |
size_bytes | integer | | not null |
sha256 | character(64) | | not null |
object_key | character varying(128) | | not null |
created_at | timestamp(3) with time zone | | not null | CURRENT_TIMESTAMP
updated_at | timestamp(3) with time zone | | not null | CURRENT_TIMESTAMP
staging_key | character varying(128) | | |
status | character varying(16) | | not null |
error_code | character varying(64) | | |
Indexes:
"attachments_pkey" PRIMARY KEY, btree (id)
"attachments_chat_id_idx" btree (chat_id)
"attachments_object_key_key" UNIQUE, btree (object_key)
"attachments_owner_id_idx" btree (owner_id)
"attachments_staging_key_key" UNIQUE, btree (staging_key)
Check constraints:
"attachments_content_type_allowed" CHECK (content_type::text = ANY (ARRAY['image/png'::character varying, 'image/jpeg'::character varying]::text[]))
"attachments_error_code_when_failed" CHECK ((status::text = 'failed'::text) = (error_code IS NOT NULL))
"attachments_filename_not_blank" CHECK (btrim(filename::text) <> ''::text)
"attachments_sha256_format" CHECK (sha256 ~ '^[a-f0-9]{64}$'::text)
"attachments_size_range" CHECK (size_bytes >= 1 AND size_bytes <= 10485760)
"attachments_status_allowed" CHECK (status::text = ANY (ARRAY['queued'::character varying, 'processing'::character varying, 'ready'::character varying, 'failed'::character varying]::text[]))
Foreign-key constraints:
"attachments_chat_id_fkey" FOREIGN KEY (chat_id) REFERENCES chats(id) ON UPDATE CASCADE ON DELETE CASCADE
"attachments_owner_id_fkey" FOREIGN KEY (owner_id) REFERENCES users(id) ON UPDATE CASCADE ON DELETE RESTRICT
Referenced by:
TABLE "attachment_jobs" CONSTRAINT "attachment_jobs_attachment_id_fkey" FOREIGN KEY (attachment_id) REFERENCES attachments(id) ON UPDATE CASCADE ON DELETE CASCADE
TABLE "attachment_variants" CONSTRAINT "attachment_variants_attachment_id_fkey" FOREIGN KEY (attachment_id) REFERENCES attachments(id) ON UPDATE CASCADE ON DELETE CASCADE
TABLE "message_attachments" CONSTRAINT "message_attachments_attachment_id_fkey" FOREIGN KEY (attachment_id) REFERENCES attachments(id) ON UPDATE CASCADE
Table "public.attachment_variants"
Column | Type | Collation | Nullable | Default
------------------+-----------------------------+-----------+----------+-------------------
attachment_id | uuid | | not null |
pipeline_version | integer | | not null |
name | character varying(16) | | not null |
content_type | character varying(32) | | not null |
object_key | character varying(128) | | not null |
size_bytes | integer | | not null |
width | integer | | not null |
height | integer | | not null |
sha256 | character(64) | | not null |
created_at | timestamp(3) with time zone | | not null | CURRENT_TIMESTAMP
Indexes:
"attachment_variants_pkey" PRIMARY KEY, btree (attachment_id, pipeline_version, name)
"attachment_variants_object_key_key" UNIQUE, btree (object_key)
Check constraints:
"attachment_variants_content_type_allowed" CHECK (content_type::text = ANY (ARRAY['image/jpeg'::character varying, 'image/png'::character varying, 'image/webp'::character varying]::text[]))
"attachment_variants_name_allowed" CHECK (name::text = ANY (ARRAY['thumbnail'::character varying, 'crop'::character varying, 'optimized'::character varying]::text[]))
"attachment_variants_positive" CHECK (pipeline_version >= 1 AND size_bytes >= 1 AND width >= 1 AND height >= 1)
"attachment_variants_sha256_format" CHECK (sha256 ~ '^[a-f0-9]{64}$'::text)
Foreign-key constraints:
"attachment_variants_attachment_id_fkey" FOREIGN KEY (attachment_id) REFERENCES attachments(id) ON UPDATE CASCADE ON DELETE CASCADE
Table "public.attachment_jobs"
Column | Type | Collation | Nullable | Default
------------------+-----------------------------+-----------+----------+-------------------
id | uuid | | not null |
attachment_id | uuid | | not null |
pipeline_version | integer | | not null |
status | character varying(16) | | not null |
attempt | integer | | not null |
max_attempts | integer | | not null |
next_attempt_at | timestamp(3) with time zone | | not null |
lease_owner | uuid | | |
lease_until | timestamp(3) with time zone | | |
error_code | character varying(64) | | |
last_error | character varying(200) | | |
created_at | timestamp(3) with time zone | | not null | CURRENT_TIMESTAMP
updated_at | timestamp(3) with time zone | | not null | CURRENT_TIMESTAMP
finished_at | timestamp(3) with time zone | | |
Indexes:
"attachment_jobs_pkey" PRIMARY KEY, btree (id)
"attachment_jobs_attachment_id_pipeline_version_key" UNIQUE, btree (attachment_id, pipeline_version)
"attachment_jobs_status_lease_until_idx" btree (status, lease_until)
Check constraints:
"attachment_jobs_attempt_range" CHECK (pipeline_version >= 1 AND attempt >= 1 AND attempt <= max_attempts)
"attachment_jobs_error_code_when_failed" CHECK ((status::text = 'failed'::text) = (error_code IS NOT NULL))
"attachment_jobs_lease_when_processing" CHECK (status::text <> 'processing'::text OR lease_owner IS NOT NULL AND lease_until IS NOT NULL)
"attachment_jobs_status_allowed" CHECK (status::text = ANY (ARRAY['queued'::character varying, 'processing'::character varying, 'ready'::character varying, 'failed'::character varying]::text[]))
Foreign-key constraints:
"attachment_jobs_attachment_id_fkey" FOREIGN KEY (attachment_id) REFERENCES attachments(id) ON UPDATE CASCADE ON DELETE CASCADE
Referenced by:
TABLE "job_events" CONSTRAINT "job_events_job_id_fkey" FOREIGN KEY (job_id) REFERENCES attachment_jobs(id) ON UPDATE CASCADE ON DELETE CASCADE
TABLE "outbox" CONSTRAINT "outbox_job_id_fkey" FOREIGN KEY (job_id) REFERENCES attachment_jobs(id) ON UPDATE CASCADE ON DELETE CASCADE
Table "public.outbox"
Column | Type | Collation | Nullable | Default
------------------+-----------------------------+-----------+----------+------------------------------------
id | bigint | | not null | nextval('outbox_id_seq'::regclass)
job_id | uuid | | not null |
payload | jsonb | | not null |
available_at | timestamp(3) with time zone | | not null | CURRENT_TIMESTAMP
published_at | timestamp(3) with time zone | | |
publish_attempts | integer | | not null | 0
created_at | timestamp(3) with time zone | | not null | CURRENT_TIMESTAMP
Indexes:
"outbox_pkey" PRIMARY KEY, btree (id)
"outbox_job_id_idx" btree (job_id)
"outbox_published_at_available_at_idx" btree (published_at, available_at)
Foreign-key constraints:
"outbox_job_id_fkey" FOREIGN KEY (job_id) REFERENCES attachment_jobs(id) ON UPDATE CASCADE ON DELETE CASCADE
Table "public.job_events"
Column | Type | Collation | Nullable | Default
------------+-----------------------------+-----------+----------+----------------------------------------
id | bigint | | not null | nextval('job_events_id_seq'::regclass)
job_id | uuid | | not null |
attempt | integer | | not null |
event | character varying(32) | | not null |
detail | character varying(200) | | |
created_at | timestamp(3) with time zone | | not null | CURRENT_TIMESTAMP
Indexes:
"job_events_pkey" PRIMARY KEY, btree (id)
"job_events_job_id_idx" btree (job_id)
Foreign-key constraints:
"job_events_job_id_fkey" FOREIGN KEY (job_id) REFERENCES attachment_jobs(id) ON UPDATE CASCADE ON DELETE CASCADE
## Миграции: отдельный one-shot сервис, повторный запуск ничего не применяет, схема совпадает с БД
$ docker compose run --rm migrate
Container messenger-lab6-secrets-init-1 Created
Container messenger-lab6-postgres-1 Running
Container messenger-lab6-secrets-init-1 Starting
Container messenger-lab6-secrets-init-1 Started
Container messenger-lab6-secrets-init-1 Waiting
Container messenger-lab6-secrets-init-1 Exited
Prisma schema loaded from prisma/schema.prisma
Datasource "db": PostgreSQL database "messenger", schema "public" at "postgres:5432"
5 migrations found in prisma/migrations
No pending migrations to apply.
$ docker compose run --rm --no-deps migrate sh -c export DATABASE_URL="$(cat "$DATABASE_URL_FILE")"; npx prisma migrate diff --from-url "$DATABASE_URL" --to-schema-datamodel prisma/schema.prisma --exit-code
No difference detected.
$ docker compose exec -T postgres psql -U messenger -d messenger -c SELECT migration_name, finished_at FROM _prisma_migrations ORDER BY started_at
migration_name | finished_at
---------------------------------+-------------------------------
20261021000000_init | 2026-11-20 14:30:37.860315+00
20261021010000_auth_sessions | 2026-11-20 14:30:37.880522+00
20261101000000_idempotency_keys | 2026-11-20 14:30:37.895909+00
20261105000000_attachments | 2026-11-20 14:30:37.918762+00
20261115000000_attachment_jobs | 2026-11-20 14:30:37.951618+00
(5 rows)
+17
View File
@@ -0,0 +1,17 @@
# make test-failover: stop-one.sh останавливает реплику, обслужившую тест
# date: 2026-11-20T14:33:10Z
# commit: d83876f
## до
/messenger-lab6-api-1 id=64377e499417 running exit=0 finished=2026-11-20T14:33:08.109900069Z
/messenger-lab6-api-2 id=25ff01cef182 running exit=0 finished=2026-11-20T14:33:05.646476717Z
$ make test-failover ACTION_SCRIPT=scripts/stop-one.sh
python3 tests/resilience.py --mode failover --action-script "scripts/stop-one.sh" --base-url "http://lb:8080"
Executing explicit failover hook; hook output suppressed to avoid leaking secrets.
{"observed_instances_before": 2, "successful_writes": 128, "failed_attempts": 0, "first_recovery_seconds": 3.571}
PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.
## после: остановленная реплика Exited (0) — штатное завершение по SIGTERM
/messenger-lab6-api-1 id=64377e499417 running exit=0 finished=2026-11-20T14:33:08.109900069Z
/messenger-lab6-api-2 id=25ff01cef182 exited exit=0 finished=2026-11-20T14:33:19.073873959Z
+64
View File
@@ -0,0 +1,64 @@
# Инцидент: scripts/incidents.py db
# date: 2026-11-20T14:41:32Z
# commit: d83876f
$ python3 scripts/incidents.py db
[ 0.1s] user df6eb947-ae87-4aa5-9d75-0c0c8091ab4d chat e46df417-75d8-4eaa-bca3-e19e3a86da1a
[ 0.1s] == baseline: 60 s of traffic
[ 0.2s] p95_1m=16ms 5xx_1m=0.0% pool_wait=0 pool_used=0 core_5xx_1m=- api_cpu=0 MessengerSlowApi=inactive MessengerHighErrorRate=inactive MessengerApiNotReady=inactive
[ 20.2s] p95_1m=25ms 5xx_1m=0.0% pool_wait=0 pool_used=0 core_5xx_1m=- api_cpu=0 MessengerSlowApi=inactive MessengerHighErrorRate=inactive MessengerApiNotReady=inactive
[ 40.2s] p95_1m=24ms 5xx_1m=0.0% pool_wait=0 pool_used=0 core_5xx_1m=- api_cpu=0.1 MessengerSlowApi=inactive MessengerHighErrorRate=inactive MessengerApiNotReady=inactive
[ 60.3s] p95_1m=24ms 5xx_1m=0.0% pool_wait=0 pool_used=0 core_5xx_1m=- api_cpu=0.1 MessengerSlowApi=inactive MessengerHighErrorRate=inactive MessengerApiNotReady=inactive
[ 60.3s] client [baseline]: 200:567 201:568; p50=11ms p95=23ms p99=49ms
[ 60.3s] == slow handler (not DB): 1700x1700 noise PNG upload — decoding and S3 dominate
[ 61.1s] POST 8.3 MiB attachment -> 202
[ 76.1s] slow handler: trace 7c5c169b5e229a27d34765e20c7dc102 POST /api/v1/chats/{chat_id}/attachments -> 202, 525 ms
[ 76.1s] DB calls 40 ms (pool wait 1 ms, SQL 22 ms); S3 246 ms; handler itself 240 ms
[ 76.1s] == slow DB: postgres limited to 0.05 CPU for 150 s
[ 76.3s] $ docker update --cpus 0.05 27a9b1f465438247ed81c48a9b572b24ab12ea5ffc7112d9817be18636e47310
[ 76.3s] p95_1m=24ms 5xx_1m=0.0% pool_wait=0 pool_used=0 core_5xx_1m=- api_cpu=0.2 MessengerSlowApi=inactive MessengerHighErrorRate=inactive MessengerApiNotReady=inactive
[ 91.4s] p95_1m=24ms 5xx_1m=0.0% pool_wait=0 pool_used=0 core_5xx_1m=- api_cpu=0.1 MessengerSlowApi=inactive MessengerHighErrorRate=inactive MessengerApiNotReady=inactive
[ 106.4s] p95_1m=75ms 5xx_1m=0.0% pool_wait=0 pool_used=0 core_5xx_1m=- api_cpu=0.1 MessengerSlowApi=inactive MessengerHighErrorRate=inactive MessengerApiNotReady=inactive
[ 121.4s] p95_1m=102ms 5xx_1m=0.0% pool_wait=0 pool_used=1 core_5xx_1m=- api_cpu=0.1 MessengerSlowApi=inactive MessengerHighErrorRate=inactive MessengerApiNotReady=inactive
[ 136.5s] p95_1m=319ms 5xx_1m=0.0% pool_wait=0 pool_used=1 core_5xx_1m=- api_cpu=0.1 MessengerSlowApi=inactive MessengerHighErrorRate=inactive MessengerApiNotReady=inactive
[ 151.5s] p95_1m=309ms 5xx_1m=0.0% pool_wait=0 pool_used=0 core_5xx_1m=- api_cpu=0.1 MessengerSlowApi=inactive MessengerHighErrorRate=inactive MessengerApiNotReady=inactive
[ 166.5s] p95_1m=268ms 5xx_1m=0.0% pool_wait=0 pool_used=0 core_5xx_1m=- api_cpu=0.1 MessengerSlowApi=inactive MessengerHighErrorRate=inactive MessengerApiNotReady=inactive
[ 181.6s] p95_1m=282ms 5xx_1m=0.0% pool_wait=0 pool_used=0 core_5xx_1m=- api_cpu=0.1 MessengerSlowApi=inactive MessengerHighErrorRate=inactive MessengerApiNotReady=inactive
[ 196.6s] p95_1m=180ms 5xx_1m=0.0% pool_wait=0 pool_used=0 core_5xx_1m=- api_cpu=0.1 MessengerSlowApi=inactive MessengerHighErrorRate=inactive MessengerApiNotReady=inactive
[ 211.6s] p95_1m=187ms 5xx_1m=0.0% pool_wait=0 pool_used=1 core_5xx_1m=- api_cpu=0.1 MessengerSlowApi=inactive MessengerHighErrorRate=inactive MessengerApiNotReady=inactive
[ 226.7s] p95_1m=183ms 5xx_1m=0.0% pool_wait=0 pool_used=0 core_5xx_1m=- api_cpu=0.1 MessengerSlowApi=inactive MessengerHighErrorRate=inactive MessengerApiNotReady=inactive
[ 226.7s] client [slow DB]: 200:1391 201:1390; p50=12ms p95=196ms p99=637ms
[ 226.7s] slow DB: trace 4fa46b85e073eddb48c09bce00a7b150 POST /api/v1/chats/{chat_id}/messages -> 201, 790 ms
[ 226.7s] DB calls 788 ms (pool wait 0 ms, SQL 781 ms); S3 0 ms; handler itself 1 ms
[ 226.7s] slow DB: trace 20868735bb9111efc35f20b042032875 POST /api/v1/chats/{chat_id}/messages -> 201, 505 ms
[ 226.7s] DB calls 502 ms (pool wait 0 ms, SQL 498 ms); S3 0 ms; handler itself 3 ms
[ 226.7s] $ docker update --cpus 1.0 27a9b1f465438247ed81c48a9b572b24ab12ea5ffc7112d9817be18636e47310
[ 226.8s] p95_1m=183ms 5xx_1m=0.0% pool_wait=0 pool_used=0 core_5xx_1m=- api_cpu=0.1 MessengerSlowApi=inactive MessengerHighErrorRate=inactive MessengerApiNotReady=inactive
[ 226.8s] client [after slow DB]: ; p50=0ms p95=0ms p99=0ms
[ 226.8s] == DB unavailable: postgres paused for 120 s
[ 226.8s] $ docker pause 27a9b1f465438247ed81c48a9b572b24ab12ea5ffc7112d9817be18636e47310
[ 226.9s] p95_1m=183ms 5xx_1m=0.0% pool_wait=0 pool_used=0 core_5xx_1m=- api_cpu=0.1 MessengerSlowApi=inactive MessengerHighErrorRate=inactive MessengerApiNotReady=inactive
[ 242.0s] p95_1m=176ms 5xx_1m=0.0% pool_wait=5 pool_used=5 core_5xx_1m=- api_cpu=0.1 MessengerSlowApi=inactive MessengerHighErrorRate=inactive MessengerApiNotReady=inactive
[ 257.0s] p95_1m=212ms 5xx_1m=0.0% pool_wait=6 pool_used=0 core_5xx_1m=8 api_cpu=0.1 MessengerSlowApi=inactive MessengerHighErrorRate=inactive MessengerApiNotReady=inactive
[ 272.1s] p95_1m=197ms 5xx_1m=0.0% pool_wait=7 pool_used=0 core_5xx_1m=10.9 api_cpu=0 MessengerSlowApi=inactive MessengerHighErrorRate=inactive MessengerApiNotReady=inactive
[ 287.1s] p95_1m=nanms 5xx_1m=nan% pool_wait=7 pool_used=0 core_5xx_1m=15.7 api_cpu=0 MessengerSlowApi=inactive MessengerHighErrorRate=inactive MessengerApiNotReady=inactive
[ 302.1s] p95_1m=nanms 5xx_1m=nan% pool_wait=6 pool_used=0 core_5xx_1m=18 api_cpu=0 MessengerSlowApi=inactive MessengerHighErrorRate=inactive MessengerApiNotReady=pending
[ 317.2s] p95_1m=nanms 5xx_1m=nan% pool_wait=7 pool_used=0 core_5xx_1m=16.8 api_cpu=0 MessengerSlowApi=inactive MessengerHighErrorRate=inactive MessengerApiNotReady=pending
[ 332.2s] p95_1m=nanms 5xx_1m=nan% pool_wait=7 pool_used=0 core_5xx_1m=16.8 api_cpu=0 MessengerSlowApi=inactive MessengerHighErrorRate=inactive MessengerApiNotReady=firing
[ 332.2s] client [DB paused]: 201:1 503:1910 504:3; p50=41ms p95=41ms p99=41ms
[ 332.3s] DB unavailable: trace 96521ffdb7e21480980d75738932c94d GET /api/v1/chats/{chat_id}/messages -> 503, 5005 ms
[ 332.3s] DB calls 5001 ms (pool wait 5000 ms, SQL 5000 ms); S3 0 ms; handler itself 3 ms
[ 332.3s] spans with error status: GET /api/v1/chats/{chat_id}/messages
[ 332.3s] loki: {"timestamp":"2026-11-20T14:47:01.051Z","severity":"ERROR","service.name":"messenger-api","event":"http.error","logger":"HttpExceptionFilter","message":"database unavailable","method":"GET","route":"/
[ 332.3s] loki: {"timestamp":"2026-11-20T14:47:01.115Z","severity":"ERROR","service.name":"messenger-api","event":"http.error","logger":"HttpExceptionFilter","message":"database unavailable","method":"GET","route":"/
[ 332.3s] loki: {"timestamp":"2026-11-20T14:47:03.054Z","severity":"ERROR","service.name":"messenger-api","event":"http.error","logger":"HttpExceptionFilter","message":"database unavailable","method":"GET","route":"/
[ 332.3s] $ docker unpause 27a9b1f465438247ed81c48a9b572b24ab12ea5ffc7112d9817be18636e47310
[ 332.4s] p95_1m=nanms 5xx_1m=nan% pool_wait=7 pool_used=0 core_5xx_1m=16.8 api_cpu=0 MessengerSlowApi=inactive MessengerHighErrorRate=inactive MessengerApiNotReady=firing
[ 347.5s] p95_1m=24ms 5xx_1m=0.0% pool_wait=0 pool_used=0 core_5xx_1m=13.2 api_cpu=0 MessengerSlowApi=inactive MessengerHighErrorRate=inactive MessengerApiNotReady=firing
[ 362.5s] p95_1m=24ms 5xx_1m=0.0% pool_wait=0 pool_used=0 core_5xx_1m=9.6 api_cpu=0.1 MessengerSlowApi=inactive MessengerHighErrorRate=inactive MessengerApiNotReady=firing
[ 377.5s] p95_1m=24ms 5xx_1m=0.0% pool_wait=0 pool_used=0 core_5xx_1m=3.6 api_cpu=0.1 MessengerSlowApi=inactive MessengerHighErrorRate=inactive MessengerApiNotReady=firing
[ 392.6s] p95_1m=24ms 5xx_1m=0.0% pool_wait=0 pool_used=0 core_5xx_1m=0 api_cpu=0.1 MessengerSlowApi=inactive MessengerHighErrorRate=inactive MessengerApiNotReady=inactive
[ 392.6s] client [after DB unpaused]: 200:556 201:556 503:25; p50=11ms p95=23ms p99=42ms
[ 393.1s] MessengerApiNotReady fired: True, resolved after recovery: True
[ 393.1s] PASS
+36
View File
@@ -0,0 +1,36 @@
# Инцидент: scripts/incidents.py retry-trace
# date: 2026-11-20T14:37:50Z
# commit: d83876f
$ python3 scripts/incidents.py retry-trace
[ 0.1s] user cd196c28-14b5-40a2-9c7d-c9a822ff9787 chat ef415dfb-4b85-421e-9e6f-45ecba61630e
[ 0.1s] $ docker compose stop worker
[ 0.8s] upload -> 202 b1a9f7a4-1eeb-4549-8fea-702494f5e4b7 trace d703294a338bfef3bf9f088ecc76f645
[ 0.8s] $ docker compose stop s3
[ 1.3s] $ docker start worker (without dependencies)
[ 6.8s] $ docker start s3 (without dependencies)
[ 7.2s] GET attachment -> status=queued error_code=None
[ 8.2s] GET attachment -> status=processing error_code=None
[ 9.2s] GET attachment -> status=ready error_code=None
[ 9.4s] job: d7ba0597-706c-4e41-bf88-353c21ab4e76 | ready | 3 | 5 |
[ 9.5s] variants rows: 3
[ 9.7s] events: enqueued -> processing -> retry_scheduled -> processing -> retry_scheduled -> processing -> ready
[ 24.7s] messenger-api SERVER POST /api/v1/chats/{chat_id}/attachments attempt= 57.8 ms
[ 24.7s] messenger-api CLIENT S3.PutObject attempt= 14.8 ms
[ 24.7s] messenger-relay PRODUCER attachment.jobs publish attempt=1 4.5 ms
[ 24.7s] messenger-worker CONSUMER attachment.jobs process attempt=1 1065.4 ms
[ 24.7s] messenger-worker CLIENT S3.GetObject attempt= 1013.7 ms ERROR
[ 24.7s] messenger-relay PRODUCER attachment.jobs publish attempt=2 10.5 ms
[ 24.7s] messenger-worker CONSUMER attachment.jobs process attempt=2 1028.0 ms
[ 24.7s] messenger-worker CLIENT S3.GetObject attempt= 1003.1 ms ERROR
[ 24.7s] messenger-relay PRODUCER attachment.jobs publish attempt=3 7.4 ms
[ 24.7s] messenger-worker CONSUMER attachment.jobs process attempt=3 1052.5 ms
[ 24.7s] messenger-worker CLIENT S3.GetObject attempt= 969.5 ms
[ 24.7s] messenger-worker CLIENT S3.PutObject attempt= 13.8 ms
[ 24.7s] messenger-worker CLIENT S3.PutObject attempt= 6.5 ms
[ 24.7s] messenger-worker CLIENT S3.PutObject attempt= 5.9 ms
[ 24.7s] messenger-worker CLIENT S3.PutObject attempt= 6.0 ms
[ 24.7s] messenger-worker CLIENT S3.DeleteObject attempt= 3.5 ms
[ 24.7s] consumer spans in one trace: 3 (attempts ['1', '2', '3'])
[ 24.7s] PASS
+59
View File
@@ -0,0 +1,59 @@
# Инцидент: scripts/incidents.py telemetry
# date: 2026-11-20T15:12:09Z
# commit: 2718c3b
$ python3 scripts/incidents.py telemetry
[ 0.1s] user 82d06d15-aef5-49d9-b445-a93d0dd4b886 chat a583d7a0-7c8c-4261-971b-8affca86fb22
[ 0.2s] spans_failed_1m=0 logs_failed_1m=0 queue=0 queue_cap=200 refused_1m=0 collector_rss=169MiB api_rss=202MiB TelemetryExportFailing=inactive MessengerHighErrorRate=inactive
[ 15.3s] spans_failed_1m=0 logs_failed_1m=0 queue=0 queue_cap=200 refused_1m=0 collector_rss=169MiB api_rss=198MiB TelemetryExportFailing=inactive MessengerHighErrorRate=inactive
[ 30.3s] spans_failed_1m=0 logs_failed_1m=0 queue=0 queue_cap=200 refused_1m=0 collector_rss=188MiB api_rss=193MiB TelemetryExportFailing=inactive MessengerHighErrorRate=inactive
[ 30.3s] client [baseline]: 200:1350 201:1353; p50=11ms p95=53ms p99=110ms
[ 30.3s] == Loki and Tempo stopped until TelemetryExportFailing fires (max 240 s)
[ 30.3s] $ docker compose stop loki tempo
[ 32.4s] spans_failed_1m=0 logs_failed_1m=0 queue=0 queue_cap=200 refused_1m=0 collector_rss=188MiB api_rss=193MiB TelemetryExportFailing=inactive MessengerHighErrorRate=inactive
[ 47.5s] spans_failed_1m=0 logs_failed_1m=0 queue=6 queue_cap=200 refused_1m=0 collector_rss=188MiB api_rss=196MiB TelemetryExportFailing=inactive MessengerHighErrorRate=inactive
[ 62.6s] spans_failed_1m=0 logs_failed_1m=0 queue=46 queue_cap=200 refused_1m=0 collector_rss=194MiB api_rss=195MiB TelemetryExportFailing=inactive MessengerHighErrorRate=inactive
[ 77.6s] spans_failed_1m=0 logs_failed_1m=0 queue=86 queue_cap=200 refused_1m=0 collector_rss=209MiB api_rss=194MiB TelemetryExportFailing=inactive MessengerHighErrorRate=inactive
[ 92.7s] spans_failed_1m=0 logs_failed_1m=0 queue=128 queue_cap=200 refused_1m=0 collector_rss=226MiB api_rss=194MiB TelemetryExportFailing=inactive MessengerHighErrorRate=inactive
[ 107.7s] spans_failed_1m=674.2 logs_failed_1m=529.4 queue=168 queue_cap=200 refused_1m=0 collector_rss=234MiB api_rss=195MiB TelemetryExportFailing=inactive MessengerHighErrorRate=inactive
[ 122.7s] spans_failed_1m=1349.4 logs_failed_1m=1096.5 queue=199 queue_cap=200 refused_1m=5892.3 collector_rss=247MiB api_rss=199MiB TelemetryExportFailing=pending MessengerHighErrorRate=inactive
[ 137.8s] spans_failed_1m=1350.5 logs_failed_1m=1097.4 queue=200 queue_cap=200 refused_1m=37471.3 collector_rss=286MiB api_rss=199MiB TelemetryExportFailing=pending MessengerHighErrorRate=inactive
[ 152.8s] spans_failed_1m=682.7 logs_failed_1m=573.4 queue=200 queue_cap=200 refused_1m=65169.9 collector_rss=286MiB api_rss=201MiB TelemetryExportFailing=pending MessengerHighErrorRate=inactive
[ 167.9s] spans_failed_1m=1365.5 logs_failed_1m=589.4 queue=200 queue_cap=200 refused_1m=84554.2 collector_rss=287MiB api_rss=196MiB TelemetryExportFailing=pending MessengerHighErrorRate=inactive
[ 182.9s] spans_failed_1m=1365.4 logs_failed_1m=1224.1 queue=200 queue_cap=200 refused_1m=84860.3 collector_rss=287MiB api_rss=196MiB TelemetryExportFailing=firing MessengerHighErrorRate=inactive
[ 182.9s] client [Loki+Tempo down]: 200:6951 201:6950; p50=11ms p95=43ms p99=106ms
[ 182.9s] $ docker compose start loki tempo
[ 183.6s] spans_failed_1m=1365.4 logs_failed_1m=1224.1 queue=200 queue_cap=200 refused_1m=88988.6 collector_rss=287MiB api_rss=196MiB TelemetryExportFailing=firing MessengerHighErrorRate=inactive
[ 198.6s] spans_failed_1m=0 logs_failed_1m=634.7 queue=0 queue_cap=200 refused_1m=76515.4 collector_rss=266MiB api_rss=176MiB TelemetryExportFailing=firing MessengerHighErrorRate=inactive
[ 213.6s] spans_failed_1m=0 logs_failed_1m=0 queue=0 queue_cap=200 refused_1m=44296.3 collector_rss=200MiB api_rss=175MiB TelemetryExportFailing=firing MessengerHighErrorRate=inactive
[ 228.7s] spans_failed_1m=0 logs_failed_1m=0 queue=0 queue_cap=200 refused_1m=12213.9 collector_rss=196MiB api_rss=174MiB TelemetryExportFailing=firing MessengerHighErrorRate=inactive
[ 243.7s] spans_failed_1m=0 logs_failed_1m=0 queue=0 queue_cap=200 refused_1m=0 collector_rss=194MiB api_rss=174MiB TelemetryExportFailing=firing MessengerHighErrorRate=inactive
[ 258.7s] spans_failed_1m=0 logs_failed_1m=0 queue=0 queue_cap=200 refused_1m=0 collector_rss=194MiB api_rss=173MiB TelemetryExportFailing=firing MessengerHighErrorRate=inactive
[ 273.8s] spans_failed_1m=0 logs_failed_1m=0 queue=0 queue_cap=200 refused_1m=0 collector_rss=194MiB api_rss=171MiB TelemetryExportFailing=firing MessengerHighErrorRate=inactive
[ 288.8s] spans_failed_1m=0 logs_failed_1m=0 queue=0 queue_cap=200 refused_1m=0 collector_rss=194MiB api_rss=174MiB TelemetryExportFailing=firing MessengerHighErrorRate=inactive
[ 303.9s] spans_failed_1m=0 logs_failed_1m=0 queue=0 queue_cap=200 refused_1m=0 collector_rss=194MiB api_rss=172MiB TelemetryExportFailing=inactive MessengerHighErrorRate=inactive
[ 303.9s] client [after start]: 200:5660 201:5663; p50=10ms p95=36ms p99=93ms
[ 303.9s] == Collector stopped for 300 s: SDK export fails fast (5 s timeout, bounded queues); RSS saw-tooth until GC
[ 303.9s] $ docker compose stop otel-collector
[ 327.6s] docker stats: messenger-lab6-api-1 183.4MiB / 256MiB; messenger-lab6-api-2 173.5MiB / 256MiB; messenger-lab6-worker-1 58.01MiB / 768MiB;
[ 350.4s] docker stats: messenger-lab6-api-1 192.4MiB / 256MiB; messenger-lab6-api-2 183.2MiB / 256MiB; messenger-lab6-worker-1 58.72MiB / 768MiB;
[ 373.2s] docker stats: messenger-lab6-api-1 192.5MiB / 256MiB; messenger-lab6-api-2 183.4MiB / 256MiB; messenger-lab6-worker-1 58.29MiB / 768MiB;
[ 396.1s] docker stats: messenger-lab6-api-1 193.4MiB / 256MiB; messenger-lab6-api-2 186.8MiB / 256MiB; messenger-lab6-worker-1 58.77MiB / 768MiB;
[ 419.1s] docker stats: messenger-lab6-api-1 194.8MiB / 256MiB; messenger-lab6-api-2 189.2MiB / 256MiB; messenger-lab6-worker-1 58.55MiB / 768MiB;
[ 441.7s] docker stats: messenger-lab6-api-1 198MiB / 256MiB; messenger-lab6-api-2 189.1MiB / 256MiB; messenger-lab6-worker-1 58.1MiB / 768MiB;
[ 464.6s] docker stats: messenger-lab6-api-1 198.4MiB / 256MiB; messenger-lab6-api-2 191.2MiB / 256MiB; messenger-lab6-worker-1 58.59MiB / 768MiB;
[ 487.5s] docker stats: messenger-lab6-api-1 201.1MiB / 256MiB; messenger-lab6-api-2 193.2MiB / 256MiB; messenger-lab6-worker-1 58.2MiB / 768MiB;
[ 509.6s] docker stats: messenger-lab6-api-1 202.7MiB / 256MiB; messenger-lab6-api-2 190.5MiB / 256MiB; messenger-lab6-worker-1 56.81MiB / 768MiB;
[ 531.6s] docker stats: messenger-lab6-api-1 203.5MiB / 256MiB; messenger-lab6-api-2 193.2MiB / 256MiB; messenger-lab6-worker-1 57.3MiB / 768MiB;
[ 554.5s] docker stats: messenger-lab6-api-1 206.1MiB / 256MiB; messenger-lab6-api-2 195.3MiB / 256MiB; messenger-lab6-worker-1 56.75MiB / 768MiB;
[ 577.4s] docker stats: messenger-lab6-api-1 206.5MiB / 256MiB; messenger-lab6-api-2 191.7MiB / 256MiB; messenger-lab6-worker-1 56.92MiB / 768MiB;
[ 599.7s] docker stats: messenger-lab6-api-1 208.1MiB / 256MiB; messenger-lab6-api-2 193.7MiB / 256MiB; messenger-lab6-worker-1 56.79MiB / 768MiB;
[ 621.7s] docker stats: messenger-lab6-api-1 208.2MiB / 256MiB; messenger-lab6-api-2 194.9MiB / 256MiB; messenger-lab6-worker-1 56.46MiB / 768MiB;
[ 644.6s] docker stats: messenger-lab6-api-1 210.6MiB / 256MiB; messenger-lab6-api-2 197.3MiB / 256MiB; messenger-lab6-worker-1 56.95MiB / 768MiB;
[ 644.6s] client [collector down]: 200:15451 201:15453; p50=11ms p95=38ms p99=131ms
[ 644.6s] $ docker compose start otel-collector
[ 675.2s] client [collector back]: 200:1376 201:1372; p50=11ms p95=49ms p99=133ms
[ 675.7s] containers: /messenger-lab6-api-1 restarts=0 oom=false; /messenger-lab6-api-2 restarts=0 oom=false; /messenger-lab6-worker-1 restarts=0 oom=false
[ 675.7s] TelemetryExportFailing fired: True, resolved: True
[ 675.7s] PASS
+47
View File
@@ -0,0 +1,47 @@
# Инцидент: scripts/incidents.py worker-stop
# date: 2026-11-20T14:38:15Z
# commit: d83876f
$ python3 scripts/incidents.py worker-stop
[ 0.1s] user d494cd19-eea4-48ae-a471-64b35e821933 chat 0369989a-4572-4abd-8dd4-7aafb56e9d84
[ 0.1s] $ docker compose stop worker
[ 0.7s] due=0 processing=0 oldest_age_s=0 worker_busy=0 worker_up=1 ready_per_s=0 MessengerJobQueueLag=inactive
[ 0.7s] POST attachment -> 202 status=queued id=2a8a08ff-2af0-47be-ac68-9404ea8c0f70 instance=25ff01cef182
[ 10.7s] due=1 processing=0 oldest_age_s=0 worker_busy=0 worker_up=1 ready_per_s=0 MessengerJobQueueLag=inactive
[ 10.8s] POST attachment -> 202 status=queued id=90500e1c-2b31-45d7-b584-a3df08532497 instance=64377e499417
[ 20.8s] due=2 processing=0 oldest_age_s=2.5 worker_busy=0 worker_up=1 ready_per_s=0 MessengerJobQueueLag=inactive
[ 20.8s] POST attachment -> 202 status=queued id=d388cce7-ef51-4bc6-a990-c9f7c25934cc instance=25ff01cef182
[ 30.9s] due=3 processing=0 oldest_age_s=2.5 worker_busy=0 worker_up=- ready_per_s=0 MessengerJobQueueLag=inactive
[ 30.9s] POST attachment -> 202 status=queued id=93a3b0d8-3a39-47ac-8bc0-79fa17c5aed0 instance=64377e499417
[ 40.9s] due=4 processing=0 oldest_age_s=12.5 worker_busy=0 worker_up=- ready_per_s=0 MessengerJobQueueLag=inactive
[ 41.0s] POST attachment -> 202 status=queued id=26b1886f-f54c-4fff-aea7-ebddbeba4662 instance=25ff01cef182
[ 51.0s] due=5 processing=0 oldest_age_s=32.5 worker_busy=0 worker_up=- ready_per_s=0 MessengerJobQueueLag=inactive
[ 51.1s] POST attachment -> 202 status=queued id=43da824d-5d9f-4ecb-9621-58a1d0ff0f73 instance=64377e499417
[ 61.1s] due=6 processing=0 oldest_age_s=32.5 worker_busy=0 worker_up=- ready_per_s=- MessengerJobQueueLag=inactive
[ 61.1s] POST attachment -> 202 status=queued id=1810e49b-e341-4820-8d01-40151a8ed795 instance=25ff01cef182
[ 71.2s] due=7 processing=0 oldest_age_s=42.5 worker_busy=0 worker_up=- ready_per_s=- MessengerJobQueueLag=inactive
[ 71.2s] POST attachment -> 202 status=queued id=9884d9e1-6e5b-4e2b-953e-c08af021adfd instance=64377e499417
[ 81.2s] due=8 processing=0 oldest_age_s=62.5 worker_busy=0 worker_up=- ready_per_s=- MessengerJobQueueLag=inactive
[ 81.3s] POST attachment -> 202 status=queued id=cb6af66e-bc34-41d8-ba90-1f5664611708 instance=25ff01cef182
[ 91.3s] due=9 processing=0 oldest_age_s=62.5 worker_busy=0 worker_up=- ready_per_s=- MessengerJobQueueLag=pending
[ 91.3s] POST attachment -> 202 status=queued id=614d8080-709e-4574-96c5-8d5b2ebe66f8 instance=64377e499417
[ 101.4s] due=10 processing=0 oldest_age_s=72.5 worker_busy=0 worker_up=- ready_per_s=- MessengerJobQueueLag=pending
[ 101.4s] POST attachment -> 202 status=queued id=5b201a4e-3f9e-4927-b282-10530a4c2be2 instance=25ff01cef182
[ 111.4s] due=11 processing=0 oldest_age_s=92.5 worker_busy=0 worker_up=- ready_per_s=- MessengerJobQueueLag=pending
[ 111.5s] POST attachment -> 202 status=queued id=607a53d8-9583-4d79-93c4-c87f320ad7c6 instance=64377e499417
[ 121.5s] due=12 processing=0 oldest_age_s=92.5 worker_busy=0 worker_up=- ready_per_s=- MessengerJobQueueLag=pending
[ 131.5s] due=12 processing=0 oldest_age_s=102.5 worker_busy=0 worker_up=- ready_per_s=- MessengerJobQueueLag=pending
[ 141.6s] due=12 processing=0 oldest_age_s=122.5 worker_busy=0 worker_up=- ready_per_s=- MessengerJobQueueLag=pending
[ 151.6s] due=12 processing=0 oldest_age_s=122.5 worker_busy=0 worker_up=- ready_per_s=- MessengerJobQueueLag=firing
[ 151.6s] alert MessengerJobQueueLag firing after 151 s of worker downtime: True
[ 151.6s] $ docker start worker (without dependencies)
[ 152.0s] due=12 processing=0 oldest_age_s=122.5 worker_busy=0 worker_up=- ready_per_s=- MessengerJobQueueLag=firing
[ 162.0s] due=2 processing=0 oldest_age_s=132.5 worker_busy=0 worker_up=- ready_per_s=- MessengerJobQueueLag=firing
[ 172.0s] due=0 processing=0 oldest_age_s=51.8 worker_busy=0 worker_up=1 ready_per_s=- MessengerJobQueueLag=firing
[ 182.0s] due=0 processing=0 oldest_age_s=51.8 worker_busy=0 worker_up=1 ready_per_s=0 MessengerJobQueueLag=inactive
[ 182.0s] backlog drained and alert resolved 30 s after worker start: True
[ 182.1s] uploads during downtime: 12, ready now: 12
[ 197.1s] time to ready, histogram over 5m: p50=100 s, p99=176.4 s
[ 197.4s] time to ready from the database (finished_at - created_at), s: 41.9 .. 152.1
[ 197.4s] PASS
+214
View File
@@ -0,0 +1,214 @@
# Фоновая обработка: сценарии защиты уровней 3 и 4 (scripts/job-scenario.py)
# date: 2026-11-20T14:33:45Z
# commit: d83876f
## restart
$ python3 scripts/job-scenario.py restart
[ 0.2s] user 83a421c6-238a-430e-b23d-7065f92692ea chat 37e2da02-e39f-4a5b-bc3e-99e3072079ad
[ 0.2s] $ docker compose stop worker
[ 1.2s] POST attachment -> 202 status=queued id=b5032f63-71ca-4684-8071-8e546a2b1d38 instance=64377e499417
[ 1.2s] content while queued -> 409
[ 1.2s] POST message with 1 attachment(s) -> 201 id=3133055e-c217-4956-8b29-6c195ad7d832
[ 1.2s] POST message with 0 attachment(s) -> 201 id=485a4b3d-e797-4cd8-8c3a-b6fbd22e51fa
[ 10.3s] queues: name type durable messages_ready messages_unacknowledged
attachment.jobs quorum true 1 0
attachment.jobs.dlq quorum true 0 0
[ 10.3s] $ docker compose restart api rabbitmq
[ 27.8s] queues after broker restart: name type durable messages_ready messages_unacknowledged
attachment.jobs quorum true 1 0
attachment.jobs.dlq quorum true 0 0
[ 27.8s] status after restarts: queued
[ 27.8s] $ docker compose start worker
[ 32.3s] GET attachment -> status=queued error_code=None
[ 33.3s] GET attachment -> status=ready error_code=None
[ 33.3s] original -> 200 sha256 matches upload: True
[ 33.4s] thumbnail -> 200 image/jpeg 256x160 sha256 matches metadata: True
[ 33.4s] crop -> 200 image/jpeg 128x128 sha256 matches metadata: True
[ 33.4s] optimized -> 200 image/webp 320x200 sha256 matches metadata: True
[ 33.6s] job: 4e569601-78cf-43c7-8e00-d11cb4929ef8 | ready | 1 | 5 |
[ 33.7s] variants rows: 3
[ 33.9s] events: enqueued -> processing -> ready
## crash-after-store
$ python3 scripts/job-scenario.py crash-after-store
[ 0.1s] user 00aee8d4-3de1-4190-ba59-145bbd9ab2e3 chat eb05fa1f-3a6f-412b-a924-23472af26d2b
[ 0.1s] $ docker compose stop worker
[ 0.7s] $ docker compose run --no-deps -e WORKER_FAULT=exit-after-store -e JOB_LEASE_SECONDS=15 worker
[ 2.7s] POST attachment -> 202 status=queued id=6c09adc1-6c17-43ad-a8ed-2fa6582fe5c5 instance=25ff01cef182
[ 3.4s] one-off: {"timestamp":"2026-11-20T14:34:22.942Z","severity":"WARN","service.name":"messenger-worker","event":"worker","logger":"Worker","message":"fault injection: worker killed after storing objects, before commit and ack","trace_id":"0f573fa6f35061a106d47bf5eb093c3d","span_id":"39a42f14e6f1823b"}
[ 3.6s] one-off process exit code 137 (137 = killed without ack)
[ 3.6s] status after crash: processing
[ 3.9s] lease: processing | 1 | t
[ 11.7s] queues (unacked message returned): name type durable messages_ready messages_unacknowledged
attachment.jobs quorum true 1 0
attachment.jobs.dlq quorum true 0 0
[ 11.7s] $ docker compose start worker
[ 16.8s] GET attachment -> status=processing error_code=None
[ 19.8s] GET attachment -> status=queued error_code=None
[ 20.8s] GET attachment -> status=processing error_code=None
[ 21.9s] GET attachment -> status=ready error_code=None
[ 21.9s] original -> 200 sha256 matches upload: True
[ 21.9s] thumbnail -> 200 image/jpeg 256x160 sha256 matches metadata: True
[ 21.9s] crop -> 200 image/jpeg 128x128 sha256 matches metadata: True
[ 21.9s] optimized -> 200 image/webp 320x200 sha256 matches metadata: True
[ 22.0s] job: d619929c-d660-4311-bb5e-0d3fbc1813ef | ready | 2 | 5 |
[ 22.2s] variants rows: 3
[ 22.3s] events: enqueued -> processing -> lease_expired -> processing -> ready
[ 22.6s] worker: {"timestamp":"2026-11-20T14:34:37.365Z","severity":"INFO","service.name":"messenger-worker","event":"job_processor","logger":"JobProcessor","message":"job d619929c-d660-4311-bb5e-0d3fbc1813ef: processing, not due or leased elsewhere, skipped","trace_id":"0f573fa6f35061a106d47bf5eb093c3d","span_id":"17a7f5d766544282"}
[ 22.6s] worker: {"timestamp":"2026-11-20T14:34:37.366Z","severity":"INFO","service.name":"messenger-worker","event":"job.handled","logger":"Worker","job_id":"d619929c-d660-4311-bb5e-0d3fbc1813ef","attachment_id":"6c09adc1-6c17-43ad-a8ed-2fa6582fe5c5","attempt":1,"redelivered":true,"outcome":"skipped","duration_ms":25,"trace_id":"0f573fa6f35061a106d47bf5eb093c3d","span_id":"17a7f5d766544282"}
[ 22.6s] worker: {"timestamp":"2026-11-20T14:34:40.735Z","severity":"INFO","service.name":"messenger-worker","event":"job.handled","logger":"Worker","job_id":"d619929c-d660-4311-bb5e-0d3fbc1813ef","attachment_id":"6c09adc1-6c17-43ad-a8ed-2fa6582fe5c5","attempt":2,"redelivered":false,"outcome":"success","duration_ms":159}
[ 22.7s] relay: {"timestamp":"2026-11-20T14:34:39.557Z","severity":"INFO","service.name":"messenger-relay","event":"outbox_relay","logger":"OutboxRelay","message":"lease expired: 1, stale queued republished: 0"}
## crash-after-commit
$ python3 scripts/job-scenario.py crash-after-commit
[ 0.1s] user 34fae015-fbb7-48eb-ad9d-8f23ee67784c chat 801de767-107d-4eb7-92ff-b6aa38dc95cf
[ 0.1s] $ docker compose stop worker
[ 0.6s] $ docker compose run --no-deps -e WORKER_FAULT=exit-after-commit worker
[ 2.7s] POST attachment -> 202 status=queued id=e02d5568-ce5b-485d-9649-f9b4d7387d41 instance=25ff01cef182
[ 3.4s] one-off: {"timestamp":"2026-11-20T14:34:45.757Z","severity":"WARN","service.name":"messenger-worker","event":"worker","logger":"Worker","message":"fault injection: worker killed after commit, before ack","trace_id":"e15e253daf15a01b76410070a68e7aff","span_id":"b5b5eb6d044c2f38"}
[ 3.5s] one-off process exit code 137 (137 = killed without ack)
[ 3.9s] status after crash: ready staging_key: staging/e02d5568-ce5b-485d-9649-f9b4d7387d41
[ 11.7s] queues (unacked message returned): name type durable messages_ready messages_unacknowledged
attachment.jobs quorum true 1 0
attachment.jobs.dlq quorum true 0 0
[ 11.7s] $ docker compose start worker
[ 17.6s] staging_key after redelivery: NULL
[ 17.6s] original -> 200 sha256 matches upload: True
[ 17.6s] thumbnail -> 200 image/jpeg 256x160 sha256 matches metadata: True
[ 17.6s] crop -> 200 image/jpeg 128x128 sha256 matches metadata: True
[ 17.6s] optimized -> 200 image/webp 320x200 sha256 matches metadata: True
[ 17.8s] job: dbc6450f-1513-4fd0-9923-3c3339985866 | ready | 1 | 5 |
[ 18.0s] variants rows: 3
[ 18.2s] events: enqueued -> processing -> ready
[ 18.5s] worker: {"timestamp":"2026-11-20T14:34:59.494Z","severity":"INFO","service.name":"messenger-worker","event":"job_processor","logger":"JobProcessor","message":"job dbc6450f-1513-4fd0-9923-3c3339985866: ready, not due or leased elsewhere, skipped","trace_id":"e15e253daf15a01b76410070a68e7aff","span_id":"db61235444dfbbbc"}
[ 18.5s] worker: {"timestamp":"2026-11-20T14:34:59.494Z","severity":"INFO","service.name":"messenger-worker","event":"job.handled","logger":"Worker","job_id":"dbc6450f-1513-4fd0-9923-3c3339985866","attachment_id":"e02d5568-ce5b-485d-9649-f9b4d7387d41","attempt":1,"redelivered":true,"outcome":"skipped","duration_ms":50,"trace_id":"e15e253daf15a01b76410070a68e7aff","span_id":"db61235444dfbbbc"}
## outbox-gap
$ python3 scripts/job-scenario.py outbox-gap
[ 0.1s] user 8646554f-5ce9-4e60-951f-9403d8e951d1 chat 196660eb-f3fc-4117-96dd-9287c6c85681
[ 0.1s] $ docker compose stop relay
[ 0.9s] POST attachment -> 202 status=queued id=14e198ac-f973-4692-9267-8dba3a1acbc0 instance=64377e499417
[ 1.1s] outbox after COMMIT: 8 | NOT PUBLISHED | 0
[ 8.8s] queues: name type durable messages_ready messages_unacknowledged
attachment.jobs quorum true 0 0
attachment.jobs.dlq quorum true 0 0
[ 8.8s] $ docker compose kill api
[ 9.4s] API killed (SIGKILL) after COMMIT, nothing was published
[ 9.4s] $ docker compose start api
[ 18.6s] status after API restart: queued
[ 18.6s] $ docker compose start relay
[ 22.9s] GET attachment -> status=queued error_code=None
[ 26.0s] GET attachment -> status=ready error_code=None
[ 26.2s] outbox after relay recovery: 8 | 2026-11-20 14:35:26.35+00 | 1
[ 26.2s] original -> 200 sha256 matches upload: True
[ 26.2s] thumbnail -> 200 image/jpeg 256x160 sha256 matches metadata: True
[ 26.2s] crop -> 200 image/jpeg 128x128 sha256 matches metadata: True
[ 26.3s] optimized -> 200 image/webp 320x200 sha256 matches metadata: True
[ 26.4s] job: 29554704-45b1-4335-87fe-8bcb0621b000 | ready | 1 | 5 |
[ 26.6s] variants rows: 3
[ 26.7s] events: enqueued -> processing -> ready
## relay-crash
$ python3 scripts/job-scenario.py relay-crash
[ 0.1s] user da9df65a-2adb-4552-b09a-063e143f7e42 chat 59b25b73-439e-4eb9-abf5-51a8851b5dd5
[ 0.1s] $ docker compose stop relay
[ 0.7s] $ docker compose run --no-deps -e RELAY_FAULT=exit-after-confirm relay
[ 3.7s] POST attachment -> 202 status=queued id=95f87871-7463-4cec-b73e-556b32ce1f57 instance=64377e499417
[ 4.3s] one-off: {"timestamp":"2026-11-20T14:35:31.985Z","severity":"INFO","service.name":"messenger-relay","event":"outbox_relay","logger":"OutboxRelay","message":"published 1 job message(s), broker confirmed"}
[ 4.3s] one-off: {"timestamp":"2026-11-20T14:35:31.985Z","severity":"WARN","service.name":"messenger-relay","event":"outbox_relay","logger":"OutboxRelay","message":"fault injection: exit after broker confirm, before outbox is marked"}
[ 4.4s] one-off process exit code 137 (137 = killed without ack)
[ 4.8s] outbox after relay crash: 9 | NOT MARKED | 0
[ 4.8s] $ docker compose start relay
[ 9.6s] GET attachment -> status=ready error_code=None
[ 12.8s] outbox after relay restart: 9 | 2026-11-20 14:35:39.707+00 | 1
[ 12.8s] original -> 200 sha256 matches upload: True
[ 12.8s] thumbnail -> 200 image/jpeg 256x160 sha256 matches metadata: True
[ 12.8s] crop -> 200 image/jpeg 128x128 sha256 matches metadata: True
[ 12.8s] optimized -> 200 image/webp 320x200 sha256 matches metadata: True
[ 13.0s] job: 5449c7df-1ee6-435d-a8e4-60b0f672d65e | ready | 1 | 5 |
[ 13.1s] variants rows: 3
[ 13.4s] events: enqueued -> processing -> ready
[ 13.6s] worker: {"timestamp":"2026-11-20T14:35:32.064Z","severity":"INFO","service.name":"messenger-worker","event":"job.handled","logger":"Worker","job_id":"5449c7df-1ee6-435d-a8e4-60b0f672d65e","attachment_id":"95f87871-7463-4cec-b73e-556b32ce1f57","attempt":1,"redelivered":false,"outcome":"success","duration_ms":79,"trace_id":"c24603daec760a57d467aff4b2a1d515","span_id":"99e7c21fe5d815b2"}
[ 13.6s] worker: {"timestamp":"2026-11-20T14:35:39.712Z","severity":"INFO","service.name":"messenger-worker","event":"job_processor","logger":"JobProcessor","message":"job 5449c7df-1ee6-435d-a8e4-60b0f672d65e: ready, not due or leased elsewhere, skipped","trace_id":"c24603daec760a57d467aff4b2a1d515","span_id":"ee8d2b5d22aac1b4"}
[ 13.6s] worker: {"timestamp":"2026-11-20T14:35:39.713Z","severity":"INFO","service.name":"messenger-worker","event":"job.handled","logger":"Worker","job_id":"5449c7df-1ee6-435d-a8e4-60b0f672d65e","attachment_id":"95f87871-7463-4cec-b73e-556b32ce1f57","attempt":1,"redelivered":false,"outcome":"skipped","duration_ms":6,"trace_id":"c24603daec760a57d467aff4b2a1d515","span_id":"ee8d2b5d22aac1b4"}
## retry-replay
$ python3 scripts/job-scenario.py retry-replay
[ 0.1s] user beb0f294-3e65-411c-bda5-ce3f29e2c1d7 chat a253ff82-2c8c-457b-85cf-6b8c9beee260
[ 0.1s] $ docker compose stop worker
[ 0.7s] POST attachment -> 202 status=queued id=9832fbf4-2f69-484e-87e4-463d3e8ef3a7 instance=64377e499417
[ 0.7s] $ docker compose stop s3
[ 1.2s] $ docker start worker (without dependencies)
[ 1.7s] GET attachment -> status=queued error_code=None
[ 2.7s] GET attachment -> status=processing error_code=None
[ 3.7s] GET attachment -> status=queued error_code=None
[ 4.8s] GET attachment -> status=processing error_code=None
[ 5.8s] GET attachment -> status=queued error_code=None
[ 7.8s] GET attachment -> status=processing error_code=None
[ 8.8s] GET attachment -> status=queued error_code=None
[ 12.8s] GET attachment -> status=processing error_code=None
[ 13.8s] GET attachment -> status=queued error_code=None
[ 19.9s] GET attachment -> status=processing error_code=None
[ 20.9s] GET attachment -> status=failed error_code=retry_exhausted
[ 20.9s] POST message with 1 attachment(s) -> 409 id=None
[ 20.9s] failed attachment in a new message -> 409
[ 20.9s] retries:
[ 21.0s] 1 | enqueued | | 14:35:42.501
[ 21.0s] 1 | processing | | 14:35:44.160
[ 21.0s] 1 | retry_scheduled | temporary failure: ApiError; next attempt 2 in 860 ms | 14:35:45.201
[ 21.0s] 2 | processing | | 14:35:46.335
[ 21.0s] 2 | retry_scheduled | temporary failure: ApiError; next attempt 3 in 1537 ms | 14:35:47.349
[ 21.0s] 3 | processing | | 14:35:49.390
[ 21.0s] 3 | retry_scheduled | temporary failure: ApiError; next attempt 4 in 3959 ms | 14:35:50.403
[ 21.0s] 4 | processing | | 14:35:54.463
[ 21.0s] 4 | retry_scheduled | temporary failure: ApiError; next attempt 5 in 5443 ms | 14:35:55.491
[ 21.0s] 5 | processing | | 14:36:01.035
[ 21.0s] 5 | failed | retry_exhausted: temporary failure: ApiError | 14:36:02.047
[ 21.0s] $ replay.js --list
[ 21.4s] replay: {"job_id":"b88d0626-5470-4148-83eb-cac17feb8b4c","attachment_id":"9832fbf4-2f69-484e-87e4-463d3e8ef3a7","error_code":"retry_exhausted","attempt":5,"last_error":"temporary failure: ApiError","finished_at":"2026-11-20T14:36:02.047Z"}
[ 21.4s] $ docker compose start s3
[ 23.7s] $ replay.js 9832fbf4-2f69-484e-87e4-463d3e8ef3a7
[ 24.0s] replay: {"id":"9832fbf4-2f69-484e-87e4-463d3e8ef3a7","job_id":"b88d0626-5470-4148-83eb-cac17feb8b4c","result":"replayed"}
[ 24.0s] GET attachment -> status=queued error_code=None
[ 25.1s] GET attachment -> status=ready error_code=None
[ 25.1s] original -> 200 sha256 matches upload: True
[ 25.1s] thumbnail -> 200 image/jpeg 256x160 sha256 matches metadata: True
[ 25.1s] crop -> 200 image/jpeg 128x128 sha256 matches metadata: True
[ 25.1s] optimized -> 200 image/webp 320x200 sha256 matches metadata: True
[ 25.1s] $ replay.js 9832fbf4-2f69-484e-87e4-463d3e8ef3a7 (already ready)
[ 25.4s] replay: {"id":"9832fbf4-2f69-484e-87e4-463d3e8ef3a7","job_id":"b88d0626-5470-4148-83eb-cac17feb8b4c","result":"already_ready"}
[ 25.6s] job: b88d0626-5470-4148-83eb-cac17feb8b4c | ready | 1 | 5 |
[ 25.7s] variants rows: 3
[ 25.9s] events: enqueued -> processing -> retry_scheduled -> processing -> retry_scheduled -> processing -> retry_scheduled -> processing -> retry_scheduled -> processing -> failed -> replayed -> processing -> ready
## broker
$ python3 scripts/job-scenario.py broker
[ 2.1s] consumers: queue_name ack_required prefetch_count
attachment.jobs true 2
[ 2.2s] worker limits: cpus=1000000000 memory=805306368
[ 2.5s] malformed message published
[ 10.3s] queues: name type durable messages_ready messages_unacknowledged
attachment.jobs quorum true 0 0
attachment.jobs.dlq quorum true 1 0
[ 10.4s] worker: {"timestamp":"2026-11-20T14:36:10.245Z","severity":"WARN","service.name":"messenger-worker","event":"worker","logger":"Worker","message":"malformed job message moved to dead letter queue","trace_id":"733415ad4bdbd66df930af833ef44f18","span_id":"abc299a80e5c1e42"}
## Итог по всем заданиям стенда
$ docker compose exec -T postgres psql -U messenger -d messenger -c SELECT status, count(*) FROM attachment_jobs GROUP BY status -c SELECT count(*) AS unpublished_outbox FROM outbox WHERE published_at IS NULL -c SELECT count(*) AS staging_left_on_ready FROM attachments WHERE status = 'ready' AND staging_key IS NOT NULL
status | count
--------+-------
ready | 9
(1 row)
unpublished_outbox
--------------------
0
(1 row)
staging_left_on_ready
-----------------------
0
(1 row)
## Последний проход reconciler
{"mode":"apply","grace_seconds":3600,"staging_expired":0,"scanned":0,"fresh_skipped":0,"referenced":0,"orphans":0,"deleted":0,"failed":0,"orphan_keys":[],"duration_ms":351}
+26
View File
@@ -0,0 +1,26 @@
# make test-persistence: restart.sh перезапускает postgres, redis, s3, rabbitmq, api, lb с сохранением volumes
# date: 2026-11-20T14:33:01Z
# commit: d83876f
## до
/messenger-lab6-api-1 running started=2026-11-20T14:32:32.185792399Z
/messenger-lab6-api-2 running started=2026-11-20T14:32:32.538220625Z
/messenger-lab6-lb-1 running started=2026-11-20T14:32:38.299124143Z
/messenger-lab6-postgres-1 running started=2026-11-20T14:32:22.796529774Z
/messenger-lab6-rabbitmq-1 running started=2026-11-20T14:32:22.722217174Z
/messenger-lab6-redis-1 running started=2026-11-20T14:32:22.787053855Z
/messenger-lab6-s3-1 running started=2026-11-20T14:32:22.813514565Z
$ make test-persistence ACTION_SCRIPT=scripts/restart.sh
python3 tests/resilience.py --mode persistence --action-script "scripts/restart.sh" --base-url "http://lb:8080"
Executing explicit persistence hook; hook output suppressed to avoid leaking secrets.
PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence.
## после
/messenger-lab6-api-1 running started=2026-11-20T14:33:08.406683646Z
/messenger-lab6-api-2 running started=2026-11-20T14:33:05.908734161Z
/messenger-lab6-lb-1 running started=2026-11-20T14:33:03.32738495Z
/messenger-lab6-postgres-1 running started=2026-11-20T14:33:03.697395758Z
/messenger-lab6-rabbitmq-1 running started=2026-11-20T14:33:04.196322444Z
/messenger-lab6-redis-1 running started=2026-11-20T14:33:03.432938355Z
/messenger-lab6-s3-1 running started=2026-11-20T14:33:03.540481157Z
+37
View File
@@ -0,0 +1,37 @@
# S3 (минимум lab4): идемпотентный bootstrap, приватность мимо API, права пользователя API
# date: 2026-11-20T14:33:34Z
# commit: d83876f
$ docker compose run --rm --no-deps -T s3-init
Bucket created successfully `root/messenger-attachments`.
Access permission for `root/messenger-attachments` is set to `private`
Created policy `messenger-api` successfully.
bucket messenger-attachments ready
$ docker compose run --rm --no-deps -T s3-init
Bucket created successfully `root/messenger-attachments`.
Access permission for `root/messenger-attachments` is set to `private`
Created policy `messenger-api` successfully.
bucket messenger-attachments ready
$ sh scripts/check-s3.sh
## S3 не опубликован наружу, консоль выключена
s3 ports=9000/tcp
MINIO_BROWSER=off
## Права пользователя API: только свой bucket
PUT в свой bucket -> allowed
GET из своего bucket -> allowed
создание другого bucket -> denied
GET из другого bucket -> denied
PUT в другой bucket -> denied
изменение политики bucket -> denied
администрирование пользователей -> denied
## Анонимный доступ напрямую к S3, мимо API
GET messenger-attachments/<object> -> 403
GET messenger-attachments/ -> 403
PUT messenger-attachments/anon.txt -> 403
PASS: bucket приватный, у API нет прав вне своего bucket
+22
View File
@@ -0,0 +1,22 @@
# Публичные проверки через LB; разные реплики обслуживают одну сессию
# date: 2026-11-20T14:32:58Z
# commit: d83876f
$ make check
python3 scripts/check_contract.py
PASS: offline skeleton structure, local refs and Python syntax; application NOT tested.
$ make test
python3 tests/smoke.py --base-url "http://lb:8080"
PASS: lab6 public API scenario; infrastructure and grading evidence remain separate.
$ python3 scripts/probe.py instances 6
register -> 201
login -> 201 expires_at: 2026-11-20T15:33:01.208Z
GET /api/v1/users/me -> 200 X-Instance-Id: 64377e499417
GET /api/v1/users/me -> 200 X-Instance-Id: 25ff01cef182
GET /api/v1/users/me -> 200 X-Instance-Id: 64377e499417
GET /api/v1/users/me -> 200 X-Instance-Id: 25ff01cef182
GET /api/v1/users/me -> 200 X-Instance-Id: 64377e499417
GET /api/v1/users/me -> 200 X-Instance-Id: 25ff01cef182
+158
View File
@@ -0,0 +1,158 @@
# Наблюдаемость: контрольный запрос (trace + логи), dashboard, перцентили, приватность, правила
# date: 2026-11-20T14:36:18Z
# commit: d83876f
## Prometheus: правила загружены (promtool и /api/v1/rules)
$ docker compose exec -T prometheus promtool check rules /etc/prometheus/rules/messenger.yml
Checking /etc/prometheus/rules/messenger.yml
SUCCESS: 13 rules found
$ sh -c wget -qO- http://grafana:3000/api/datasources/proxy/uid/prometheus/api/v1/rules | python3 -c 'import json,sys; [print(r["type"], r["name"], r["health"]) for g in json.load(sys.stdin)["data"]["groups"] for r in g["rules"]]'
alerting MessengerHighErrorRate ok
alerting MessengerApiNotReady ok
alerting MessengerJobQueueLag ok
alerting MessengerSlowApi ok
alerting TelemetryExportFailing ok
recording messenger:http_requests:rate5m ok
recording messenger:http_errors:rate5m ok
recording messenger:http_error_ratio:rate5m ok
recording messenger:http_error_ratio:rate1h ok
recording messenger:http_latency_p95:rate5m ok
recording messenger:http_fast_ratio:rate1h ok
recording messenger:time_to_ready_fast_ratio:rate1h ok
recording messenger:jobs_queue_oldest_age:max ok
## Grafana: datasources с фиксированными UID, dashboard из provisioning
$ sh -c for uid in prometheus loki tempo; do wget -qO- http://grafana:3000/api/datasources/uid/$uid | python3 -c 'import json,sys; d=json.load(sys.stdin); print(d["uid"], d["type"], d["url"])'; done
prometheus prometheus http://prometheus:9090
loki loki http://loki:3100
tempo tempo http://tempo:3200
## Контрольный upload с клиентским traceparent: trace в Tempo и логи в Loki
$ python3 scripts/trace-check.py
upload -> 202 queued traceparent 00-0a9700e372765e5f2774c2c5347f8c1b-0de651da523f691b-01
attachment 0a91d4b0-0624-419a-824e-d4ae671e7d2b status ready after 0.57 s
trace 0a9700e372765e5f2774c2c5347f8c1b: 79 spans (prisma:* internals except db_query omitted below)
messenger-api SPAN_KIND_SERVER POST /api/v1/chats/{chat_id}/attachments 50.36 ms parent=client
messenger-api SPAN_KIND_CLIENT prisma:engine:db_query 0.48 ms parent=prisma:engine:query
messenger-api SPAN_KIND_CLIENT S3.PutObject 25.34 ms parent=POST /api/v1/chats/{chat_id}/attachments
messenger-api SPAN_KIND_INTERNAL prisma:client:transaction 7.80 ms parent=POST /api/v1/chats/{chat_id}/attachments
messenger-api SPAN_KIND_CLIENT prisma:engine:db_query 0.15 ms parent=prisma:engine:start_transaction
messenger-api SPAN_KIND_CLIENT prisma:engine:db_query 0.81 ms parent=prisma:engine:query
messenger-api SPAN_KIND_CLIENT prisma:engine:db_query 0.51 ms parent=prisma:engine:query
messenger-api SPAN_KIND_CLIENT prisma:engine:db_query 0.27 ms parent=prisma:engine:query
messenger-api SPAN_KIND_CLIENT prisma:engine:db_query 0.31 ms parent=prisma:engine:query
messenger-api SPAN_KIND_CLIENT prisma:engine:db_query 1.33 ms parent=prisma:engine:commit_transaction
messenger-relay SPAN_KIND_PRODUCER attachment.jobs publish 5.71 ms parent=prisma:client:transaction
messenger-worker SPAN_KIND_CONSUMER attachment.jobs process 66.86 ms parent=attachment.jobs publish
messenger-worker SPAN_KIND_CLIENT prisma:engine:db_query 0.25 ms parent=prisma:engine:connection
messenger-worker SPAN_KIND_CLIENT prisma:engine:db_query 0.34 ms parent=prisma:engine:query
messenger-worker SPAN_KIND_CLIENT prisma:engine:db_query 0.26 ms parent=prisma:engine:query
messenger-worker SPAN_KIND_INTERNAL prisma:client:transaction 7.26 ms parent=attachment.jobs process
messenger-worker SPAN_KIND_CLIENT prisma:engine:db_query 0.14 ms parent=prisma:engine:start_transaction
messenger-worker SPAN_KIND_CLIENT prisma:engine:db_query 0.48 ms parent=prisma:engine:query
messenger-worker SPAN_KIND_CLIENT prisma:engine:db_query 0.50 ms parent=prisma:engine:query
messenger-worker SPAN_KIND_CLIENT prisma:engine:db_query 0.26 ms parent=prisma:engine:query
messenger-worker SPAN_KIND_CLIENT prisma:engine:db_query 0.27 ms parent=prisma:engine:query
messenger-worker SPAN_KIND_CLIENT prisma:engine:db_query 1.24 ms parent=prisma:engine:commit_transaction
messenger-worker SPAN_KIND_CLIENT S3.GetObject 3.00 ms parent=attachment.jobs process
messenger-worker SPAN_KIND_INTERNAL attachment.render_variants 12.41 ms parent=attachment.jobs process
messenger-worker SPAN_KIND_CLIENT S3.PutObject 7.85 ms parent=attachment.jobs process
messenger-worker SPAN_KIND_CLIENT S3.PutObject 5.94 ms parent=attachment.jobs process
messenger-worker SPAN_KIND_CLIENT S3.PutObject 5.73 ms parent=attachment.jobs process
messenger-worker SPAN_KIND_CLIENT S3.PutObject 5.69 ms parent=attachment.jobs process
messenger-worker SPAN_KIND_INTERNAL prisma:client:transaction 7.42 ms parent=attachment.jobs process
messenger-worker SPAN_KIND_CLIENT prisma:engine:db_query 0.16 ms parent=prisma:engine:start_transaction
messenger-worker SPAN_KIND_CLIENT prisma:engine:db_query 0.61 ms parent=prisma:engine:query
messenger-worker SPAN_KIND_CLIENT prisma:engine:db_query 0.71 ms parent=prisma:engine:query
messenger-worker SPAN_KIND_CLIENT prisma:engine:db_query 0.44 ms parent=prisma:engine:query
messenger-worker SPAN_KIND_CLIENT prisma:engine:db_query 0.26 ms parent=prisma:engine:query
messenger-worker SPAN_KIND_CLIENT prisma:engine:db_query 1.32 ms parent=prisma:engine:commit_transaction
messenger-worker SPAN_KIND_CLIENT S3.DeleteObject 3.23 ms parent=attachment.jobs process
messenger-worker SPAN_KIND_CLIENT prisma:engine:db_query 1.71 ms parent=prisma:engine:query
Loki {service_name=~"messenger-.+"} | trace_id="0a9700e372765e5f2774c2c5347f8c1b"
messenger-worker INFO job.handled span_id=8f3cc8917f60f19d {"timestamp":"2026-11-20T14:36:19.298Z","severity":"INFO","s
messenger-api INFO http.request span_id=70c807e8b4f94da9 {"timestamp":"2026-11-20T14:36:19.155Z","severity":"INFO","s
PASS: client trace continued through api -> relay -> worker, logs linked by trace_id
## Секреты и содержимое не попадают в telemetry; в labels метрик нет ID
$ python3 scripts/telemetry-privacy.py
sent 6 traced requests with known password, token, message text and filename; waiting 25 s for export
Tempo: 6/6 traces fetched (79945 bytes); Loki: 7 log lines carry these trace_ids (control)
password in traces: False log lines in Loki (5 min): 0
message text in traces: False log lines in Loki (5 min): 0
filename in traces: False log lines in Loki (5 min): 0
session token in traces: False log lines in Loki (5 min): 0
basic credentials in traces: False log lines in Loki (5 min): 0
Authorization hdr in traces: False log lines in Loki (5 min): 0
Prometheus: 893 series, 22 metric names from messenger services; label keys: cpu_mode, db_client_connection_state, deployment_environment_name, http_request_method, http_response_status_class, http_route, instance, job, job_type, le, messaging_destination_name, outcome, service_version, state, telemetry_sdk_language, telemetry_sdk_name, telemetry_sdk_version
series with a UUID in any label value: 0
http_route values: /api/v1/attachments/{attachment_id}, /api/v1/attachments/{attachment_id}/content, /api/v1/auth/sessions, /api/v1/auth/sessions/current, /api/v1/chats, /api/v1/chats/{chat_id}, /api/v1/chats/{chat_id}/attachments, /api/v1/chats/{chat_id}/messages, /api/v1/users, /api/v1/users/me, /health/core, /health/live, /health/ready
PASS: no secrets/content in logs and traces, no IDs in metric labels
## Перцентили dashboard против клиентского замера tests/load.py
$ python3 scripts/latency-compare.py --duration 30
waiting 25 s for the metric export interval
client (tests/load.py): 3411 requests, 113.4 rps, errors 0
prometheus: 3661 requests in POST /api/v1/chats/{chat_id}/messages over 60 s
client, ms prometheus, ms
p50 18.33 19.68
p95 89.20 92.20
p99 163.29 176.76
server-side histogram excludes network/client overhead; values are interpolated inside buckets (…, 5, 10, 25, 50, 100 ms, …).
## Все запросы dashboard (пустые панели alerts/WARN-логов допустимы без инцидентов)
$ python3 scripts/dashboard-check.py
Grafana dashboard: Messenger overview folder: Messenger
OK Доступность API, 1 ч (SLO ≥ 99% без 5xx) A 1 =1
OK Запросы быстрее 500 мс, 1 ч (SLO ≥ 95%) A 1 =1
OK Вложения готовы за ≤ 10 с, 1 ч (SLO ≥ 95%) A 1 =0.6666666666666667
EMPTY Alerts (pending / firing) A 0
OK RPS по маршрутам (без health) A 12 =0, =0, =0, =0
OK Ответы по классам статуса (без health) A 2 2xx=61.1782202581422, 4xx=0
OK Доля 5xx среди бизнес-запросов (окно 5m, порог alert 5%) A 1 =0
OK Длительность API p50 / p95 / p99 (все реплики, без health) A 1 =0.019676691515573837
OK Длительность API p50 / p95 / p99 (все реплики, без health) B 1 =0.09219914944691271
OK Длительность API p50 / p95 / p99 (все реплики, без health) C 1 =0.17675359699995996
OK p95 по маршрутам A 12 =NaN, =NaN, =NaN, =NaN
OK Глубина очереди attachment.jobs (из attachment_jobs) A 3 =0, =0, =0
OK Возраст старейшего готового задания (порог alert 60 s) A 1 =0
OK Worker: занято / ёмкость A 1 =0
OK Worker: занято / ёмкость B 1 =2
OK Worker: занято / ёмкость C 1 =0
OK Задания по исходу (в секунду) A 5 failed=0, malformed=0, retry=0, skipped=0.025
OK Длительность обработки задания p50 / p95 / p99 A 1 =NaN
OK Длительность обработки задания p50 / p95 / p99 B 1 =NaN
OK Длительность обработки задания p50 / p95 / p99 C 1 =NaN
OK Время от upload до ready p50 / p95 / p99 A 1 =NaN
OK Время от upload до ready p50 / p95 / p99 B 1 =NaN
OK Время от upload до ready p50 / p95 / p99 C 1 =NaN
OK Пул БД: занято / ожидают соединения A 3 =0, =0, =0
OK Пул БД: занято / ожидают соединения B 3 =0, =0, =0
OK Пул БД: занято / ожидают соединения C 3 =20, =3, =5
OK CPU процессов (доля ядра) A 4 =0.2771702986118047, =0.2803168326733069, =0.006292157529451772, =0.00
OK CPU процессов (доля ядра) B 2 =0.013556760600942309, =0.0006813251539196734
OK RAM процессов (RSS) A 4 =207585280, =208904192, =90546176, =106270720
OK RAM процессов (RSS) B 2 =210243584, =164040704
OK Принято / отправлено / ошибки отправки (в секунду) A 1 =1512.6751237147007
OK Принято / отправлено / ошибки отправки (в секунду) B 1 =1512.6751237147007
OK Принято / отправлено / ошибки отправки (в секунду) C 1 =0
OK Принято / отправлено / ошибки отправки (в секунду) D 1 =75.96230776068984
OK Принято / отправлено / ошибки отправки (в секунду) E 1 =0
OK Принято / отправлено / ошибки отправки (в секунду) F 1 =0
OK Очередь exporter (ограничена queue_size) A 3 =0, =0, =0
OK Очередь exporter (ограничена queue_size) B 1 =200
OK Логи API и worker (WARN и выше) A 20
OK Все логи (http.request, job.handled, …) A 20
OK Последние трассы (HTTP server и consumer spans) A 20
WARN: 1 empty queries
EXIT CODE 1
## Пример структурного лога (stdout API)
{"timestamp":"2026-11-20T14:37:24.827Z","severity":"INFO","service.name":"messenger-api","event":"http.request","logger":"Http","method":"POST","route":"/api/v1/chats/{chat_id}/messages","status":201,"duration_ms":23.22,"request_id":"078e6db3-d15e-4bdf-bac9-5fe5ea48bd35","trace_id":"67197c10f2dc44bc87840224dec07957","span_id":"a4e80137cbbf9edc"}
{"timestamp":"2026-11-20T14:37:24.838Z","severity":"INFO","service.name":"messenger-api","event":"http.request","logger":"Http","method":"POST","route":"/api/v1/chats/{chat_id}/messages","status":201,"duration_ms":20.5,"request_id":"5513c34d-64ea-4250-9cee-a9d9a4b12892","trace_id":"ec0813f680256f525febe0676221457b","span_id":"d20713fac541ea0b"}
+484
View File
@@ -0,0 +1,484 @@
# Собственные e2e-тесты (PostgreSQL + Redis + MinIO из compose.test.yaml); основной стенд на время тестов остановлен ради памяти Docker
# date: 2026-11-20T14:31:15Z
# commit: d83876f
$ docker compose -f compose.test.yaml run --rm tests npx jest --config ./test/jest-e2e.json --verbose
Network messenger-lab6-test_default Creating
Network messenger-lab6-test_default Created
Container messenger-lab6-test-redis-1 Creating
Container messenger-lab6-test-s3-1 Creating
Container messenger-lab6-test-postgres-1 Creating
Container messenger-lab6-test-s3-1 Created
Container messenger-lab6-test-redis-1 Created
Container messenger-lab6-test-postgres-1 Created
Container messenger-lab6-test-s3-1 Starting
Container messenger-lab6-test-redis-1 Starting
Container messenger-lab6-test-postgres-1 Starting
Container messenger-lab6-test-s3-1 Started
Container messenger-lab6-test-postgres-1 Started
Container messenger-lab6-test-redis-1 Started
Prisma schema loaded from prisma/schema.prisma
Datasource "db": PostgreSQL database "messenger_test", schema "public" at "postgres:5432"
5 migrations found in prisma/migrations
Applying migration `20261021000000_init`
Applying migration `20261021010000_auth_sessions`
Applying migration `20261101000000_idempotency_keys`
Applying migration `20261105000000_attachments`
Applying migration `20261115000000_attachment_jobs`
The following migration(s) have been applied:
migrations/
└─ 20261021000000_init/
└─ migration.sql
└─ 20261021010000_auth_sessions/
└─ migration.sql
└─ 20261101000000_idempotency_keys/
└─ migration.sql
└─ 20261105000000_attachments/
└─ migration.sql
└─ 20261115000000_attachment_jobs/
└─ migration.sql
All migrations have been successfully applied.
[Nest] 102 - 11/20/2026, 2:31:59 PM  ERROR [S3Service] get staging/a3987d2f-e0a6-4581-8caa-8670462a94fa failed: NoSuchKey
PASS test/api.e2e-spec.ts (17.729 s)
API мессенджера (e2e) — граничные случаи сверх публичного smoke
POST /api/v1/users
✓ отклоняет некорректный JSON с кодом 400 (112 ms)
✓ отклоняет тело с лишними полями (39 ms)
✓ отклоняет username с заглавными буквами (4 ms)
✓ отклоняет display_name из одних пробелов (4 ms)
✓ отклоняет дубликат username с кодом 409 (183 ms)
✓ никогда не возвращает поле пароля или его хэша (32 ms)
✓ отклоняет username короче 3 символов (3 ms)
✓ отклоняет username длиннее 32 символов (4 ms)
✓ отклоняет username с недопустимыми символами (дефис) (3 ms)
✓ принимает username ровно на границах длины 3 и 32 символа (56 ms)
✓ отклоняет display_name длиннее 100 Unicode code points (3 ms)
✓ принимает display_name ровно из 100 code points, считая эмодзи вне BMP за один символ (27 ms)
✓ отклоняет display_name из 101 code point, состоящий из эмодзи вне BMP (3 ms)
контроль доступа
✓ возвращает 401 раньше 404, если и актор, и ресурс невалидны (5 ms)
✓ возвращает 404 (не 403), когда чат запрашивает не его участник (254 ms)
✓ отклоняет создание чата, если вызывающий не входит в member_ids (155 ms)
✓ отклоняет создание чата с дублирующимися member_ids (83 ms)
✓ считает один UUID в разном регистре дубликатом и узнаёт вызывающего в верхнем регистре (112 ms)
✓ принимает ровно 100 участников и отклоняет 101 (104 ms)
✓ отклоняет создание чата с неизвестным участником (79 ms)
✓ отклоняет создание чата с пустым/пробельным title (75 ms)
✓ возвращает 400 для синтаксически некорректного chat_id (не UUID) (74 ms)
✓ возвращает 404 для корректного по формату, но несуществующего chat_id (75 ms)
GET /api/v1/chats (пагинация)
✓ использует limit=50 по умолчанию, если limit не передан (85 ms)
✓ отклоняет нечисловой limit (75 ms)
✓ отклоняет limit=0 (77 ms)
✓ отклоняет повторённые limit и cursor (72 ms)
✓ отклоняет cursor, не являющийся корректным base64url JSON (67 ms)
✓ отклоняет корректно закодированный cursor с id чата, который никогда не существовал (69 ms)
✓ обходит весь список чатов по страницам без пропусков и дублей (114 ms)
сообщения
✓ не принимает sender_id, переданный клиентом (144 ms)
✓ считает attachment_ids: [] равным отсутствию поля, неизвестный ID — 404, неверный тип — 400 (241 ms)
✓ отклоняет пустое сообщение (148 ms)
✓ отклоняет сообщение из одних пробелов (132 ms)
✓ отклоняет отправку в чат, где вызывающий не участник (199 ms)
✓ отклоняет limit=0 при чтении истории (165 ms)
✓ отклоняет limit=101 при чтении истории (148 ms)
✓ отклоняет нечисловой limit (169 ms)
✓ отклоняет текст сообщения длиннее 4000 Unicode code points (185 ms)
✓ принимает текст сообщения ровно из 4000 Unicode code points (218 ms)
✓ отклоняет текст сообщения свыше 4000 code points, состоящий из эмодзи вне BMP (158 ms)
✓ отклоняет cursor, не являющийся корректным base64url JSON (158 ms)
✓ отклоняет корректно закодированный cursor с id сообщения, которое никогда не существовало в этом чате (168 ms)
✓ возвращает 400 для синтаксически некорректного chat_id при отправке сообщения (80 ms)
✓ возвращает 400 для синтаксически некорректного chat_id при чтении истории (68 ms)
✓ отклоняет cursor, взятый из другого чата (276 ms)
✓ обходит всю историю по страницам без пропусков и дублей (277 ms)
X-Request-Id
✓ присутствует и уникален для независимых запросов (4 ms)
✓ совпадает с request_id внутри тела ошибки (2 ms)
[Nest] 102 - 11/20/2026, 2:32:05 PM  ERROR [S3Service] get staging/02ca6856-dcd8-4bbd-9fe8-272649f2b87c failed: NoSuchKey
[Nest] 102 - 11/20/2026, 2:32:05 PM  ERROR [S3Service] get staging/8007bc99-ed6a-4a68-9ff7-1b65dcb1cd82 failed: NoSuchKey
PASS test/jobs.e2e-spec.ts (19.57 s)
Фоновая обработка вложений (e2e)
до обработки
✓ queued: метаданные доступны, content любого варианта — 409, сообщение с вложением создаётся (250 ms)
результат worker
✓ ready: оригинал неизменен, thumbnail 256×160 JPEG, crop 128×128 JPEG, optimized WebP без увеличения (253 ms)
✓ маленький исходник не увеличивается в thumbnail/optimized, crop — увеличивается (148 ms)
✓ прозрачность компонуется на белый, EXIF orientation применяется (322 ms)
✓ worker ограничивает пиксели до распаковки: превышение — InvalidImageError (18 ms)
✓ размеры считаются целочисленно: 319×200 → 256×160, перебор совпадает с точной формулой (2289 ms)
✓ размеры округляются вниз, минимум 1 px (1 ms)
идемпотентность и повторная доставка
✓ повтор уже ready-задания — no-op: второго набора вариантов нет, объекты не меняются (249 ms)
✓ пока lease действует, второй обработчик того же job не запускается (162 ms)
✓ worker упал после записи объектов, до COMMIT/ack: lease истекает, повтор завершает тот же результат (293 ms)
✓ worker упал после COMMIT, до ack: повтор — no-op, staging убирается при повторной доставке (210 ms)
✓ сообщение потеряно брокером: queued без свежей публикации публикуется повторно один раз (233 ms)
✓ устаревшее сообщение, чужой attachment_id и другая pipeline_version игнорируются (62 ms)
ошибки, повторы и replay
✓ временная ошибка S3: повтор с backoff, до срока задание не берётся, затем ready (208 ms)
✓ попытки исчерпаны — failed retry_exhausted; replay явно возвращает в очередь и доводит до ready (470 ms)
✓ lease истёк на последней попытке — failed retry_exhausted, а не вечный processing (278 ms)
✓ replay и очистка staging не теряют исходник ни при каком порядке (541 ms)
✓ replay задания в очереди не создаёт дубль (76 ms)
✓ нераспаковываемое изображение — постоянная ошибка: сразу failed invalid_image без повторов (165 ms)
✓ staging подменён — failed checksum_mismatch; staging пропал — failed staging_missing (134 ms)
✓ превышение времени обработки — временная ошибка с повтором (112 ms)
формат сообщения и backoff
✓ сообщение соответствует jobs.schema.json, лишние/неверные поля отклоняются (1 ms)
✓ backoff растёт экспоненциально с jitter и ограничен сверху (1 ms)
console.warn
An error was encountered in a non-retryable streaming request.
213 | private async guard<T>(action: string, run: () => Promise<T>): Promise<T> {
214 | try {
> 215 | return await run();
| ^
216 | } catch (error) {
217 | const reason = error instanceof Error ? error.name : 'unknown error';
218 | this.logger.error(`${action} failed: ${reason}`);
at ../node_modules/@smithy/middleware-retry/dist-cjs/index.js:329:125
at ../node_modules/@aws-sdk/middleware-flexible-checksums/dist-cjs/index.js:263:18
at ../node_modules/@aws-sdk/middleware-sdk-s3/dist-cjs/index.js:110:22
at ../node_modules/@aws-sdk/middleware-sdk-s3/dist-cjs/index.js:138:14
at ../node_modules/@aws-sdk/middleware-logger/dist-cjs/index.js:34:22
at S3Service.guard (../src/storage/s3.service.ts:215:11)
at S3Service.putFile (../src/storage/s3.service.ts:64:3)
at AttachmentsService.store (../src/attachments/attachments.service.ts:153:3)
at AttachmentsService.upload (../src/attachments/attachments.service.ts:75:11)
at AttachmentsController.upload (../src/attachments/attachments.controller.ts:36:22)
at ../node_modules/@nestjs/core/router/router-execution-context.js:46:28
at Object.<anonymous> (../node_modules/@nestjs/core/router/router-proxy.js:9:17)
[Nest] 108 - 11/20/2026, 2:32:06 PM  ERROR [S3Service] put staging/1071fe6c-2f2f-4f80-a8b5-904ca05009a2 failed: Error
[Nest] 108 - 11/20/2026, 2:32:06 PM  ERROR [HttpExceptionFilter] Object:
{
"event": "http.error",
"message": "object storage unavailable",
"method": "POST",
"route": "/api/v1/chats/{chat_id}/attachments",
"status": 503,
"request_id": "38769a67-411f-41a7-bca3-d3682a083471"
}
[Nest] 108 - 11/20/2026, 2:32:06 PM  ERROR [HttpExceptionFilter] Object:
{
"event": "http.error",
"message": "Internal server error",
"method": "POST",
"route": "/api/v1/chats/{chat_id}/attachments",
"status": 500,
"request_id": "be749a43-6bee-4629-ac78-3f15b6f77eac"
}
[Nest] 108 - 11/20/2026, 2:32:06 PM  ERROR [AttachmentsService] metadata insert failed for staging/50082f9d-e8dd-4f01-9391-47bf371e52fb
[Nest] 108 - 11/20/2026, 2:32:06 PM  ERROR [HttpExceptionFilter] Object:
{
"event": "http.error",
"message": "database unavailable",
"method": "POST",
"route": "/api/v1/chats/{chat_id}/attachments",
"status": 503,
"request_id": "2a4d5542-1230-4568-acb5-7eb6c4390cdf"
}
[Nest] 108 - 11/20/2026, 2:32:06 PM  ERROR [HttpExceptionFilter] Object:
{
"event": "http.error",
"message": "database unavailable",
"method": "POST",
"route": "/api/v1/chats/{chat_id}/attachments",
"status": 503,
"request_id": "41953d69-f89d-42ba-9f5c-61bd4fa2e40d"
}
PASS test/attachments.e2e-spec.ts (20.033 s)
Загрузка вложений (e2e)
успешная загрузка
✓ возвращает 202 Attachment queued по контракту, staging и задание сохранены до ответа (307 ms)
✓ принимает реальный JPEG (48 ms)
✓ игнорирует параметры и регистр Content-Type (41 ms)
✓ принимает файл ровно 10 MiB, filename ровно 255 символов и PNG с байтами после IEND (724 ms)
✓ принимает ровно 40 млн пикселей (2896 ms)
порядок проверок и доступ
✓ без токена — 401 раньше любых проверок содержимого (19 ms)
✓ посторонний и несуществующий чат — одинаковый 404, чужой файл не сохраняется (17 ms)
✓ неверный UUID чата — 400 (5 ms)
filename
✓ отсутствует — 400 (4 ms)
✓ пустой — 400 (4 ms)
✓ только пробелы — 400 (5 ms)
✓ 256 символов — 400 (5 ms)
✓ путь через / — 400 (5 ms)
✓ путь через \ — 400 (5 ms)
✓ NUL — 400 (8 ms)
✓ перевод строки — 400 (5 ms)
✓ DEL — 400 (5 ms)
✓ C1 U+0085 — 400 (5 ms)
✓ разделитель строк U+2028 — 400 (4 ms)
✓ повторённый параметр — 400 (8 ms)
тип содержимого
✓ Content-Type text/plain — 415 (5 ms)
✓ Content-Type application/json — 415 (35 ms)
✓ Content-Type application/octet-stream — 415 (9 ms)
✓ Content-Type multipart/form-data; boundary=x — 415 (13 ms)
✓ Content-Type image/gif — 415 (13 ms)
✓ Content-Type image/webp — 415 (9 ms)
✓ без Content-Type — 415 (11 ms)
✓ подмена MIME в обе стороны и чужие форматы — 415 (84 ms)
✓ пустое тело — 400 (30 ms)
повреждённые изображения
✓ PNG: первые 33 байта sample.png — 415 (20 ms)
✓ PNG: половина файла — 415 (17 ms)
✓ PNG: без последних 20 байт — 415 (20 ms)
✓ PNG: без IEND — 415 (24 ms)
✓ PNG: неверный CRC — 415 (18 ms)
✓ PNG: фильтр строки 5 — 415 (15 ms)
✓ PNG: фильтр строки 255 — 415 (10 ms)
✓ PNG: неверный фильтр в последней строке — 415 (11 ms)
✓ PNG: мусор в IDAT с верным CRC — 415 (10 ms)
✓ PNG: данных меньше, чем строк — 415 (8 ms)
✓ PNG: палитра на 257 записей — 415 (11 ms)
✓ JPEG: все повреждения — 415 (68 ms)
лимиты
✓ 10 MiB + 1 байт — 413 и без чтения объявленного тела (220 ms)
✓ превышение при chunked без Content-Length — 413 (944 ms)
✓ превышение лимита пикселей — 413 до декодирования (92 ms)
✓ оборванная загрузка не оставляет временных файлов и строк в БД (819 ms)
отказы
✓ S3 недоступен — 503 с заголовками, метаданные не создаются (185 ms)
✓ явный отказ транзакции после PUT удаляет объект staging (56 ms)
✓ необработанная ошибка PostgreSQL при INSERT — 503, объект не удаляется (34 ms)
✓ при неясном исходе COMMIT объект не удаляется (31 ms)
[Nest] 101 - 11/20/2026, 2:32:07 PM  ERROR [S3Service] get attachments/0b88f570-7a00-4713-bc1f-eb84c29804f1/original failed: Error
[Nest] 101 - 11/20/2026, 2:32:07 PM  ERROR [HttpExceptionFilter] Object:
{
"event": "http.error",
"message": "object storage unavailable",
"method": "GET",
"route": "/api/v1/attachments/{attachment_id}/content",
"status": 503,
"request_id": "c2f13d3f-5110-4cc7-83af-7137dd518fea"
}
[Nest] 101 - 11/20/2026, 2:32:07 PM  ERROR [S3Service] get attachments/6ea31e5f-9549-486c-8249-e3ffb6da6413/original failed: NoSuchKey
[Nest] 101 - 11/20/2026, 2:32:07 PM  ERROR [HttpExceptionFilter] Object:
{
"event": "http.error",
"message": "Internal server error",
"method": "GET",
"route": "/api/v1/attachments/{attachment_id}/content",
"status": 500,
"request_id": "e2f0e5e7-e052-4972-8fcc-60b05ae4bb83"
}
[Nest] 102 - 11/20/2026, 2:32:08 PM  ERROR [HttpExceptionFilter] Object:
{
"event": "http.error",
"message": "object storage unavailable",
"method": "GET",
"route": "/health/ready",
"status": 503,
"request_id": "fce039e2-3f6a-4a82-bcc9-516576969c72"
}
PASS test/request-constraints.e2e-spec.ts
Каждое ограничение запроса из контракта отклоняется 400, граница принимается (e2e)
✓ POST /users (263 ms)
✓ POST /chats (91 ms)
✓ POST /chats/{chat_id}/messages (157 ms)
✓ query и path параметры (149 ms)
✓ POST /chats/{chat_id}/attachments: filename и chat_id (91 ms)
PASS test/reconciler.e2e-spec.ts
Сверка объектов S3 с метаданными (e2e)
✓ dry-run сообщает о сироте и ничего не удаляет, apply удаляет, повторный запуск ничего не делает (67 ms)
✓ не трогает свежий объект внутри grace period (24 ms)
✓ не трогает старый объект, на который есть метаданные (19 ms)
✓ при отказе PostgreSQL проход прерывается и ничего не удаляет (27 ms)
✓ сбой удаления считается и повторяется на следующем проходе (50 ms)
✓ обходит все страницы листинга (112 ms)
✓ реальная загрузка остаётся доступной после сверки всего bucket префикса вложений (384 ms)
✓ после grace period объекты вложения и staging в очереди не считаются сиротами (282 ms)
✓ staging failed-вложения удаляется после срока хранения (88 ms)
✓ объекты вариантов без строк у существующего вложения не удаляются: replay после сверки доводит до ready (301 ms)
✓ staging ready-вложения, оставшийся после сбоя worker, удаляется без ожидания срока хранения (161 ms)
Готовность при отказе S3 (e2e)
✓ ready — 503 с заголовками за короткое время, live и core — 200 (64 ms)
✓ при рабочем S3 ready — 200 (61 ms)
PASS test/auth.e2e-spec.ts
Аутентификация и сессии (e2e)
регистрация
✓ требует password (22 ms)
✓ отклоняет password короче 12 code points (3 ms)
✓ хранит пароль как Argon2id-хеш, а не открытым текстом (147 ms)
POST /api/v1/auth/sessions
✓ выдаёт непрозрачный Bearer-токен с expires_at и пользователем (117 ms)
✓ принимает пароль с двоеточием и не-ASCII символами (85 ms)
✓ отвечает одинаковым 401 на неверный пароль и неизвестного пользователя (141 ms)
✓ отклоняет JSON body при входе даже с верным Basic (83 ms)
✓ отвечает 401 без заголовка Authorization (3 ms)
защищённые endpoint
✓ не принимают Basic (68 ms)
✓ не принимают X-User-Id сам по себе (89 ms)
✓ игнорируют X-User-Id рядом с Bearer (150 ms)
✓ отвечают 401 на неизвестный токен (7 ms)
DELETE /api/v1/auth/sessions/current
✓ отзывает только текущий токен: 204, затем 401 на повтор (195 ms)
PASS test/contract.e2e-spec.ts
Все ответы соответствуют контракту: коды, схемы, заголовки (e2e)
✓ health, пользователи и сессии (324 ms)
✓ чаты и сообщения (471 ms)
✓ вложения во всех состояниях (768 ms)
✓ каждая операция контракта вызвана хотя бы раз (1 ms)
PASS test/attachment-access.e2e-spec.ts (7.499 s)
Скачивание и привязка вложений (e2e)
чтение
✓ участник получает метаданные и оригинал побайтно (329 ms)
✓ JPEG отдаётся с истинным Content-Type (210 ms)
✓ посторонний и неизвестный ID получают одинаковый 404 (265 ms)
✓ UUID nil корректного формата — 404, а не 400 (99 ms)
✓ UUID v7 корректного формата — 404, а не 400 (46 ms)
✓ без токена 401, неверный UUID 400 (193 ms)
✓ variant: original явно, отсутствующий 404, неизвестный и повторённый 400 (218 ms)
✓ нет доступа к S3 — метаданные 200, content 503 с заголовками (225 ms)
✓ объект пропал из S3 — 500 без внутренних деталей (168 ms)
attachment_ids в сообщениях
✓ text="" с вложением принимается, Message читается вторым участником (164 ms)
✓ порядок [B, A] сохраняется в ответе, истории и повторе; перестановка с тем же ключом — 409 (430 ms)
✓ отсутствующий attachment_ids и [] равны при повторе, изменение списка — 409 (179 ms)
✓ вложение другого чата и неизвестное — одинаковый 404, сообщение не создаётся (159 ms)
✓ смесь своего и чужого вложения — 404 (316 ms)
✓ посторонний не может отправить вложение в чужой чат (131 ms)
✓ любой участник может сослаться на вложение чата, вложение можно использовать повторно (165 ms)
✓ ровно 10 вложений принимается, 11 — 400 (767 ms)
✓ UUID в другом регистре — тот же ID и дубль (201 ms)
✓ text отсутствует — 400 (152 ms)
✓ text null — 400 (159 ms)
✓ attachment_ids null — 400 (137 ms)
✓ attachment_ids строка — 400 (149 ms)
✓ элемент не UUID — 400 (122 ms)
✓ элемент число — 400 (113 ms)
✓ дубликат — 400 (128 ms)
✓ пустой текст без вложений — 400 (115 ms)
✓ пробельный текст без вложений — 400 (95 ms)
PASS test/concurrency.e2e-spec.ts
Безопасность при конкурентном доступе (e2e)
✓ принимает 100 конкурентных сообщений от 10 клиентов и сохраняет ровно 100 уникальных (1147 ms)
✓ регистрирует ровно одного пользователя при гонке 10 запросов за один и тот же username (192 ms)
[Nest] 102 - 11/20/2026, 2:32:13 PM  ERROR [HttpExceptionFilter] Object:
{
"event": "http.error",
"message": "session cache unavailable",
"method": "DELETE",
"route": "/api/v1/auth/sessions/current",
"status": 503,
"request_id": "f7de03bc-bdf0-42ee-a767-01a1c55c8900"
}
[Nest] 102 - 11/20/2026, 2:32:13 PM  ERROR [HttpExceptionFilter] Object:
{
"event": "http.error",
"message": "session cache unavailable",
"method": "DELETE",
"route": "/api/v1/auth/sessions/current",
"status": 503,
"request_id": "3cb57701-e14e-4d0c-95b1-74125f609d5f"
}
[Nest] 102 - 11/20/2026, 2:32:14 PM  ERROR [HttpExceptionFilter] Object:
{
"event": "http.error",
"message": "database unavailable",
"method": "GET",
"route": "/health/ready",
"status": 503,
"request_id": "c84cbd52-47cb-410d-ae40-7028a9bd486f"
}
[Nest] 102 - 11/20/2026, 2:32:14 PM  ERROR [HttpExceptionFilter] Object:
{
"event": "http.error",
"message": "database unavailable",
"method": "GET",
"route": "/api/v1/chats",
"status": 503,
"request_id": "0f917fec-5b5c-4001-9fad-02c9c41c29a6"
}
[Nest] 102 - 11/20/2026, 2:32:14 PM  ERROR [HttpExceptionFilter] Object:
{
"event": "http.error",
"message": "database unavailable",
"method": "GET",
"route": "/api/v1/chats",
"status": 503,
"request_id": "ce2feb99-927d-4095-971d-d4a91e5674d3"
}
[Nest] 102 - 11/20/2026, 2:32:14 PM  ERROR [HttpExceptionFilter] Object:
{
"event": "http.error",
"message": "database unavailable",
"method": "POST",
"route": "/api/v1/users",
"status": 503,
"request_id": "2e2483c7-5f3d-4218-94f0-5dc4813905f4"
}
PASS test/idempotency.e2e-spec.ts
Идемпотентность отправки и X-Instance-Id (e2e)
✓ требует Idempotency-Key правильного формата (147 ms)
✓ повтор с тем же ключом и телом возвращает тот же 201 и то же сообщение (107 ms)
✓ тот же ключ с другим телом — 409, сообщение не создаётся (106 ms)
✓ 20 параллельных запросов с одним ключом через два экземпляра создают ровно одно сообщение (312 ms)
✓ область ключа — пользователь и чат: одинаковый ключ у разных пользователей и чатов независим (238 ms)
✓ повтор заново проверяет сессию (219 ms)
✓ после IDEMPOTENCY_TTL_SECONDS ключ можно использовать заново (1832 ms)
✓ X-Instance-Id есть в успешных ответах, ошибках и health и стабилен в процессе (58 ms)
[Nest] 102 - 11/20/2026, 2:32:14 PM  ERROR [HttpExceptionFilter] Object:
{
"event": "http.error",
"message": "Internal server error",
"method": "GET",
"route": "/api/v1/chats",
"status": 500,
"request_id": "dd8bae40-6d02-4739-b4ac-194bb45a416c"
}
PASS test/failures.e2e-spec.ts
Уровень 4: expiry, отказы зависимостей, rollback (e2e)
✓ сессия истекает при коротком SESSION_TTL_SECONDS, в том числе из кэша (2882 ms)
✓ без Redis вход и проверка сессии работают через PostgreSQL, logout — 503 без отзыва (160 ms)
✓ logout через реплику без связи с Redis не создаёт расхождения с репликой, у которой сессия в кэше (424 ms)
✓ без PostgreSQL отвечает ограниченным 503 и не пускает анонима (263 ms)
✓ неожиданное исключение — 500 internal_error без деталей, с X-Request-Id и X-Instance-Id (267 ms)
✓ откатывает создание чата целиком, если один из участников не существует (226 ms)
PASS test/docs.e2e-spec.ts
Swagger UI и примеры (e2e)
✓ отдаёт Swagger UI и документ контракта со всеми путями (39 ms)
✓ каждый пример запроса и ответа проходит JSON Schema контракта (407 ms)
A worker process has failed to exit gracefully and has been force exited. This is likely caused by tests leaking due to improper teardown. Try running with --detectOpenHandles to find leaks. Active timers can also cause this, ensure that .unref() was called on them.
Test Suites: 12 passed, 12 total
Tests: 201 passed, 201 total
Snapshots: 0 total
Time: 30.222 s
Ran all test suites.
+428
View File
@@ -0,0 +1,428 @@
{
"uid": "messenger-overview",
"title": "Messenger overview",
"tags": ["messenger", "lab6"],
"timezone": "browser",
"schemaVersion": 39,
"version": 2,
"editable": false,
"graphTooltip": 1,
"refresh": "10s",
"time": { "from": "now-30m", "to": "now" },
"templating": { "list": [] },
"annotations": { "list": [] },
"panels": [
{ "type": "row", "title": "SLO и alerts", "id": 100, "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 } },
{
"type": "stat",
"title": "Доступность API, 1 ч (SLO ≥ 99% без 5xx)",
"id": 20,
"datasource": { "type": "prometheus", "uid": "prometheus" },
"gridPos": { "h": 5, "w": 6, "x": 0, "y": 1 },
"fieldConfig": {
"defaults": {
"unit": "percentunit", "decimals": 2,
"thresholds": { "mode": "absolute", "steps": [{ "color": "red", "value": null }, { "color": "green", "value": 0.99 }] }
},
"overrides": []
},
"options": { "reduceOptions": { "calcs": ["lastNotNull"] }, "colorMode": "background" },
"targets": [{ "refId": "A", "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "1 - messenger:http_error_ratio:rate1h" }]
},
{
"type": "stat",
"title": "Запросы быстрее 500 мс, 1 ч (SLO ≥ 95%)",
"id": 21,
"datasource": { "type": "prometheus", "uid": "prometheus" },
"gridPos": { "h": 5, "w": 6, "x": 6, "y": 1 },
"fieldConfig": {
"defaults": {
"unit": "percentunit", "decimals": 2,
"thresholds": { "mode": "absolute", "steps": [{ "color": "red", "value": null }, { "color": "green", "value": 0.95 }] }
},
"overrides": []
},
"options": { "reduceOptions": { "calcs": ["lastNotNull"] }, "colorMode": "background" },
"targets": [{ "refId": "A", "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "messenger:http_fast_ratio:rate1h" }]
},
{
"type": "stat",
"title": "Вложения готовы за ≤ 10 с, 1 ч (SLO ≥ 95%)",
"id": 22,
"datasource": { "type": "prometheus", "uid": "prometheus" },
"gridPos": { "h": 5, "w": 6, "x": 12, "y": 1 },
"fieldConfig": {
"defaults": {
"unit": "percentunit", "decimals": 2,
"thresholds": { "mode": "absolute", "steps": [{ "color": "red", "value": null }, { "color": "green", "value": 0.95 }] }
},
"overrides": []
},
"options": { "reduceOptions": { "calcs": ["lastNotNull"] }, "colorMode": "background" },
"targets": [{ "refId": "A", "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "messenger:time_to_ready_fast_ratio:rate1h" }]
},
{
"type": "state-timeline",
"title": "Alerts (pending / firing)",
"id": 23,
"datasource": { "type": "prometheus", "uid": "prometheus" },
"gridPos": { "h": 5, "w": 6, "x": 18, "y": 1 },
"fieldConfig": {
"defaults": {
"color": { "mode": "thresholds" },
"thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }, { "color": "orange", "value": 1 }, { "color": "red", "value": 2 }] },
"mappings": [{ "type": "value", "options": { "1": { "text": "pending" }, "2": { "text": "firing" } } }]
},
"overrides": []
},
"options": { "showValue": "never", "mergeValues": true, "legend": { "showLegend": false } },
"targets": [
{
"refId": "A",
"datasource": { "type": "prometheus", "uid": "prometheus" },
"expr": "max by (alertname) (ALERTS{alertstate=\"firing\"}) * 2 or max by (alertname) (ALERTS{alertstate=\"pending\"})",
"legendFormat": "{{alertname}}"
}
]
},
{ "type": "row", "title": "API", "id": 101, "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 6 } },
{
"type": "timeseries",
"title": "RPS по маршрутам (без health)",
"id": 1,
"datasource": { "type": "prometheus", "uid": "prometheus" },
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 7 },
"fieldConfig": { "defaults": { "unit": "reqps" }, "overrides": [] },
"options": { "legend": { "displayMode": "table", "placement": "right", "calcs": ["lastNotNull"] } },
"targets": [
{
"refId": "A",
"datasource": { "type": "prometheus", "uid": "prometheus" },
"expr": "sum by (http_request_method, http_route) (rate(http_server_requests_total{job=\"messenger-api\", http_route!~\"/health.*\"}[$__rate_interval]))",
"legendFormat": "{{http_request_method}} {{http_route}}"
}
]
},
{
"type": "timeseries",
"title": "Ответы по классам статуса (без health)",
"id": 2,
"datasource": { "type": "prometheus", "uid": "prometheus" },
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 7 },
"fieldConfig": { "defaults": { "unit": "reqps" }, "overrides": [] },
"targets": [
{
"refId": "A",
"datasource": { "type": "prometheus", "uid": "prometheus" },
"expr": "sum by (http_response_status_class) (rate(http_server_requests_total{job=\"messenger-api\", http_route!~\"/health.*\"}[$__rate_interval]))",
"legendFormat": "{{http_response_status_class}}"
}
]
},
{
"type": "timeseries",
"title": "Доля 5xx среди бизнес-запросов (окно 5m, порог alert 5%)",
"id": 3,
"datasource": { "type": "prometheus", "uid": "prometheus" },
"gridPos": { "h": 8, "w": 8, "x": 0, "y": 15 },
"fieldConfig": {
"defaults": {
"unit": "percentunit",
"min": 0,
"thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }, { "color": "red", "value": 0.05 }] },
"custom": { "thresholdsStyle": { "mode": "line" } }
},
"overrides": []
},
"targets": [
{ "refId": "A", "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "messenger:http_error_ratio:rate5m", "legendFormat": "5xx ratio" }
]
},
{
"type": "timeseries",
"title": "Длительность API p50 / p95 / p99 (все реплики, без health)",
"id": 4,
"datasource": { "type": "prometheus", "uid": "prometheus" },
"gridPos": { "h": 8, "w": 8, "x": 8, "y": 15 },
"fieldConfig": { "defaults": { "unit": "s", "min": 0 }, "overrides": [] },
"targets": [
{
"refId": "A",
"datasource": { "type": "prometheus", "uid": "prometheus" },
"expr": "histogram_quantile(0.50, sum by (le) (rate(http_server_request_duration_seconds_bucket{job=\"messenger-api\", http_route!~\"/health.*\"}[$__rate_interval])))",
"legendFormat": "p50"
},
{
"refId": "B",
"datasource": { "type": "prometheus", "uid": "prometheus" },
"expr": "histogram_quantile(0.95, sum by (le) (rate(http_server_request_duration_seconds_bucket{job=\"messenger-api\", http_route!~\"/health.*\"}[$__rate_interval])))",
"legendFormat": "p95"
},
{
"refId": "C",
"datasource": { "type": "prometheus", "uid": "prometheus" },
"expr": "histogram_quantile(0.99, sum by (le) (rate(http_server_request_duration_seconds_bucket{job=\"messenger-api\", http_route!~\"/health.*\"}[$__rate_interval])))",
"legendFormat": "p99"
}
]
},
{
"type": "timeseries",
"title": "p95 по маршрутам",
"id": 5,
"datasource": { "type": "prometheus", "uid": "prometheus" },
"gridPos": { "h": 8, "w": 8, "x": 16, "y": 15 },
"fieldConfig": { "defaults": { "unit": "s", "min": 0 }, "overrides": [] },
"targets": [
{
"refId": "A",
"datasource": { "type": "prometheus", "uid": "prometheus" },
"expr": "histogram_quantile(0.95, sum by (le, http_request_method, http_route) (rate(http_server_request_duration_seconds_bucket{job=\"messenger-api\", http_route!~\"/health.*\"}[$__rate_interval])))",
"legendFormat": "{{http_request_method}} {{http_route}}"
}
]
},
{ "type": "row", "title": "Очередь и worker", "id": 102, "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 23 } },
{
"type": "timeseries",
"title": "Глубина очереди attachment.jobs (из attachment_jobs)",
"id": 30,
"datasource": { "type": "prometheus", "uid": "prometheus" },
"gridPos": { "h": 8, "w": 8, "x": 0, "y": 24 },
"fieldConfig": { "defaults": { "unit": "short", "min": 0 }, "overrides": [] },
"targets": [
{
"refId": "A",
"datasource": { "type": "prometheus", "uid": "prometheus" },
"expr": "max by (state) (jobs_queue_depth{job=\"messenger-relay\"})",
"legendFormat": "{{state}}"
}
]
},
{
"type": "timeseries",
"title": "Возраст старейшего готового задания (порог alert 60 s)",
"id": 31,
"datasource": { "type": "prometheus", "uid": "prometheus" },
"gridPos": { "h": 8, "w": 8, "x": 8, "y": 24 },
"fieldConfig": {
"defaults": {
"unit": "s", "min": 0,
"thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }, { "color": "red", "value": 60 }] },
"custom": { "thresholdsStyle": { "mode": "line" } }
},
"overrides": []
},
"targets": [
{ "refId": "A", "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "messenger:jobs_queue_oldest_age:max", "legendFormat": "oldest due job age" }
]
},
{
"type": "timeseries",
"title": "Worker: занято / ёмкость",
"id": 32,
"datasource": { "type": "prometheus", "uid": "prometheus" },
"gridPos": { "h": 8, "w": 8, "x": 16, "y": 24 },
"fieldConfig": { "defaults": { "unit": "short", "min": 0 }, "overrides": [] },
"targets": [
{ "refId": "A", "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "sum(worker_jobs_busy{job=\"messenger-worker\"})", "legendFormat": "busy" },
{ "refId": "B", "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "sum(worker_jobs_capacity{job=\"messenger-worker\"})", "legendFormat": "capacity" },
{ "refId": "C", "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "sum(avg_over_time(worker_jobs_busy{job=\"messenger-worker\"}[5m])) / sum(worker_jobs_capacity{job=\"messenger-worker\"})", "legendFormat": "saturation 5m" }
]
},
{
"type": "timeseries",
"title": "Задания по исходу (в секунду)",
"id": 6,
"datasource": { "type": "prometheus", "uid": "prometheus" },
"gridPos": { "h": 8, "w": 8, "x": 0, "y": 32 },
"fieldConfig": { "defaults": { "unit": "ops" }, "overrides": [] },
"targets": [
{
"refId": "A",
"datasource": { "type": "prometheus", "uid": "prometheus" },
"expr": "sum by (job_type, outcome) (rate(jobs_processed_total{job=\"messenger-worker\"}[$__rate_interval]))",
"legendFormat": "{{job_type}} {{outcome}}"
}
]
},
{
"type": "timeseries",
"title": "Длительность обработки задания p50 / p95 / p99",
"id": 7,
"datasource": { "type": "prometheus", "uid": "prometheus" },
"gridPos": { "h": 8, "w": 8, "x": 8, "y": 32 },
"fieldConfig": { "defaults": { "unit": "s", "min": 0 }, "overrides": [] },
"targets": [
{
"refId": "A",
"datasource": { "type": "prometheus", "uid": "prometheus" },
"expr": "histogram_quantile(0.50, sum by (le) (rate(jobs_duration_seconds_bucket{job=\"messenger-worker\", outcome=\"success\"}[$__rate_interval])))",
"legendFormat": "p50 success"
},
{
"refId": "B",
"datasource": { "type": "prometheus", "uid": "prometheus" },
"expr": "histogram_quantile(0.95, sum by (le) (rate(jobs_duration_seconds_bucket{job=\"messenger-worker\", outcome=\"success\"}[$__rate_interval])))",
"legendFormat": "p95 success"
},
{
"refId": "C",
"datasource": { "type": "prometheus", "uid": "prometheus" },
"expr": "histogram_quantile(0.99, sum by (le) (rate(jobs_duration_seconds_bucket{job=\"messenger-worker\", outcome=\"success\"}[$__rate_interval])))",
"legendFormat": "p99 success"
}
]
},
{
"type": "timeseries",
"title": "Время от upload до ready p50 / p95 / p99",
"id": 33,
"datasource": { "type": "prometheus", "uid": "prometheus" },
"gridPos": { "h": 8, "w": 8, "x": 16, "y": 32 },
"fieldConfig": { "defaults": { "unit": "s", "min": 0 }, "overrides": [] },
"targets": [
{
"refId": "A",
"datasource": { "type": "prometheus", "uid": "prometheus" },
"expr": "histogram_quantile(0.50, sum by (le) (rate(attachments_time_to_ready_seconds_bucket{job=\"messenger-worker\"}[$__rate_interval])))",
"legendFormat": "p50"
},
{
"refId": "B",
"datasource": { "type": "prometheus", "uid": "prometheus" },
"expr": "histogram_quantile(0.95, sum by (le) (rate(attachments_time_to_ready_seconds_bucket{job=\"messenger-worker\"}[$__rate_interval])))",
"legendFormat": "p95"
},
{
"refId": "C",
"datasource": { "type": "prometheus", "uid": "prometheus" },
"expr": "histogram_quantile(0.99, sum by (le) (rate(attachments_time_to_ready_seconds_bucket{job=\"messenger-worker\"}[$__rate_interval])))",
"legendFormat": "p99"
}
]
},
{ "type": "row", "title": "БД и ресурсы", "id": 103, "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 40 } },
{
"type": "timeseries",
"title": "Пул БД: занято / ожидают соединения",
"id": 40,
"datasource": { "type": "prometheus", "uid": "prometheus" },
"gridPos": { "h": 8, "w": 8, "x": 0, "y": 41 },
"fieldConfig": { "defaults": { "unit": "short", "min": 0 }, "overrides": [] },
"targets": [
{ "refId": "A", "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "sum by (job) (db_client_connection_count{db_client_connection_state=\"used\"})", "legendFormat": "{{job}} used" },
{ "refId": "B", "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "sum by (job) (db_client_connection_pending_requests)", "legendFormat": "{{job}} waiting" },
{ "refId": "C", "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "sum by (job) (db_client_connection_max)", "legendFormat": "{{job}} max" }
]
},
{
"type": "timeseries",
"title": "CPU процессов (доля ядра)",
"id": 41,
"datasource": { "type": "prometheus", "uid": "prometheus" },
"gridPos": { "h": 8, "w": 8, "x": 8, "y": 41 },
"fieldConfig": { "defaults": { "unit": "percentunit", "min": 0 }, "overrides": [] },
"targets": [
{
"refId": "A",
"datasource": { "type": "prometheus", "uid": "prometheus" },
"expr": "sum by (job, instance) (rate(process_cpu_time_seconds_total{job=~\"messenger-.+\"}[$__rate_interval]))",
"legendFormat": "{{job}} {{instance}}"
},
{
"refId": "B",
"datasource": { "type": "prometheus", "uid": "prometheus" },
"expr": "rate(otelcol_process_cpu_seconds{job=\"otel-collector\"}[$__rate_interval])",
"legendFormat": "otel-collector"
}
]
},
{
"type": "timeseries",
"title": "RAM процессов (RSS)",
"id": 42,
"datasource": { "type": "prometheus", "uid": "prometheus" },
"gridPos": { "h": 8, "w": 8, "x": 16, "y": 41 },
"fieldConfig": { "defaults": { "unit": "bytes", "min": 0 }, "overrides": [] },
"targets": [
{ "refId": "A", "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "process_memory_usage_bytes{job=~\"messenger-.+\"}", "legendFormat": "{{job}} {{instance}}" },
{ "refId": "B", "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "otelcol_process_memory_rss{job=\"otel-collector\"}", "legendFormat": "otel-collector" }
]
},
{ "type": "row", "title": "Конвейер телеметрии (Collector)", "id": 104, "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 49 } },
{
"type": "timeseries",
"title": "Принято / отправлено / ошибки отправки (в секунду)",
"id": 50,
"datasource": { "type": "prometheus", "uid": "prometheus" },
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 50 },
"fieldConfig": { "defaults": { "unit": "short", "min": 0 }, "overrides": [] },
"targets": [
{ "refId": "A", "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "sum(rate(otelcol_receiver_accepted_spans[$__rate_interval]))", "legendFormat": "spans accepted" },
{ "refId": "B", "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "sum(rate(otelcol_exporter_sent_spans[$__rate_interval]))", "legendFormat": "spans sent" },
{ "refId": "C", "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "sum(rate(otelcol_exporter_send_failed_spans[$__rate_interval])) or vector(0)", "legendFormat": "spans failed" },
{ "refId": "D", "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "sum(rate(otelcol_exporter_sent_log_records[$__rate_interval]))", "legendFormat": "logs sent" },
{ "refId": "E", "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "sum(rate(otelcol_exporter_send_failed_log_records[$__rate_interval])) or vector(0)", "legendFormat": "logs failed" },
{ "refId": "F", "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "sum(rate(otelcol_receiver_refused_spans[$__rate_interval])) + sum(rate(otelcol_receiver_refused_log_records[$__rate_interval])) or vector(0)", "legendFormat": "refused (memory_limiter)" }
]
},
{
"type": "timeseries",
"title": "Очередь exporter (ограничена queue_size)",
"id": 51,
"datasource": { "type": "prometheus", "uid": "prometheus" },
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 50 },
"fieldConfig": { "defaults": { "unit": "short", "min": 0 }, "overrides": [] },
"targets": [
{ "refId": "A", "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "max by (exporter) (otelcol_exporter_queue_size)", "legendFormat": "{{exporter}} size" },
{ "refId": "B", "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "max(otelcol_exporter_queue_capacity)", "legendFormat": "capacity" }
]
},
{ "type": "row", "title": "Логи и трассы", "id": 105, "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 58 } },
{
"type": "logs",
"title": "Логи API и worker (WARN и выше)",
"id": 8,
"datasource": { "type": "loki", "uid": "loki" },
"gridPos": { "h": 10, "w": 12, "x": 0, "y": 59 },
"options": { "showTime": true, "wrapLogMessage": true, "sortOrder": "Descending", "enableLogDetails": true },
"targets": [
{ "refId": "A", "datasource": { "type": "loki", "uid": "loki" }, "expr": "{service_name=~\"messenger-.+\"} | severity_text=~\"WARN|ERROR|FATAL\"" }
]
},
{
"type": "logs",
"title": "Все логи (http.request, job.handled, …)",
"id": 9,
"datasource": { "type": "loki", "uid": "loki" },
"gridPos": { "h": 10, "w": 12, "x": 12, "y": 59 },
"options": { "showTime": true, "wrapLogMessage": true, "sortOrder": "Descending", "enableLogDetails": true },
"targets": [
{ "refId": "A", "datasource": { "type": "loki", "uid": "loki" }, "expr": "{service_name=~\"messenger-.+\"}" }
]
},
{
"type": "table",
"title": "Последние трассы (HTTP server и consumer spans)",
"id": 10,
"datasource": { "type": "tempo", "uid": "tempo" },
"gridPos": { "h": 10, "w": 24, "x": 0, "y": 69 },
"targets": [
{
"refId": "A",
"datasource": { "type": "tempo", "uid": "tempo" },
"queryType": "traceql",
"query": "{resource.service.name=~\"messenger-.+\" && (kind=server || kind=consumer)}",
"limit": 20,
"tableType": "traces"
}
]
}
]
}
@@ -0,0 +1,10 @@
apiVersion: 1
providers:
- name: messenger
folder: Messenger
type: file
disableDeletion: true
allowUiUpdates: false
options:
path: /var/lib/grafana/dashboards
@@ -0,0 +1,45 @@
apiVersion: 1
datasources:
- name: Prometheus
uid: prometheus
type: prometheus
access: proxy
url: http://prometheus:9090
isDefault: true
jsonData:
httpMethod: POST
exemplarTraceIdDestinations:
- name: trace_id
datasourceUid: tempo
- name: Loki
uid: loki
type: loki
access: proxy
url: http://loki:3100
jsonData:
derivedFields:
- name: trace_id
matcherType: label
matcherRegex: trace_id
url: "$${__value.raw}"
datasourceUid: tempo
- name: Tempo
uid: tempo
type: tempo
access: proxy
url: http://tempo:3200
jsonData:
tracesToLogsV2:
datasourceUid: loki
spanStartTimeShift: -5m
spanEndTimeShift: 5m
filterByTraceID: false
customQuery: true
query: '{service_name=~".+"} | trace_id="$${__span.traceId}"'
tracesToMetrics:
datasourceUid: prometheus
serviceMap:
datasourceUid: prometheus
nodeGraph:
enabled: true
+43
View File
@@ -0,0 +1,43 @@
global
log stdout format raw local0 info
maxconn 2000
defaults
mode http
log global
option httplog
option dontlognull
timeout connect 1s
timeout client 30s
timeout server 10s
timeout http-request 10s
timeout queue 5s
timeout check 1s
retries 2
option redispatch 1
retry-on conn-failure
unique-id-format %[uuid()]
http-error status 502 content-type application/json hdr X-Request-Id "%[unique-id]" lf-string "{\"error\":{\"code\":\"unavailable\",\"message\":\"bad gateway\",\"request_id\":\"%[unique-id]\"}}"
http-error status 503 content-type application/json hdr X-Request-Id "%[unique-id]" lf-string "{\"error\":{\"code\":\"unavailable\",\"message\":\"no API replica available\",\"request_id\":\"%[unique-id]\"}}"
http-error status 504 content-type application/json hdr X-Request-Id "%[unique-id]" lf-string "{\"error\":{\"code\":\"unavailable\",\"message\":\"gateway timeout\",\"request_id\":\"%[unique-id]\"}}"
resolvers docker
nameserver dns 127.0.0.11:53
resolve_retries 3
timeout resolve 1s
timeout retry 1s
hold valid 5s
hold obsolete 5s
accepted_payload_size 8192
frontend public
bind :8080
default_backend api
backend api
balance roundrobin
http-request set-timeout server 60s if { method POST } { path_reg ^/api/v1/chats/[^/]+/attachments$ }
option httpchk GET /health/core
http-check expect status 200
default-server check inter 1s fall 2 rise 2 resolvers docker resolve-prefer ipv4 init-addr none
server-template api 1-4 api:8080
+43
View File
@@ -0,0 +1,43 @@
auth_enabled: false
server:
http_listen_address: 0.0.0.0
http_listen_port: 3100
grpc_listen_port: 9096
log_level: warn
common:
instance_addr: 127.0.0.1
path_prefix: /loki
replication_factor: 1
storage:
filesystem:
chunks_directory: /loki/chunks
rules_directory: /loki/rules
ring:
kvstore:
store: inmemory
schema_config:
configs:
- from: 2026-01-01
store: tsdb
object_store: filesystem
schema: v13
index:
prefix: index_
period: 24h
limits_config:
allow_structured_metadata: true
retention_period: 168h
ingestion_rate_mb: 8
ingestion_burst_size_mb: 16
compactor:
working_directory: /loki/compactor
retention_enabled: true
delete_request_store: filesystem
analytics:
reporting_enabled: false
+85
View File
@@ -0,0 +1,85 @@
extensions:
health_check:
endpoint: 0.0.0.0:13133
receivers:
otlp:
protocols:
http:
endpoint: 0.0.0.0:4318
max_request_body_size: 4194304
processors:
memory_limiter:
check_interval: 1s
limit_mib: 160
spike_limit_mib: 40
batch:
send_batch_size: 512
send_batch_max_size: 1024
timeout: 2s
exporters:
otlphttp/prometheus:
endpoint: http://prometheus:9090/api/v1/otlp
timeout: 5s
sending_queue:
enabled: true
num_consumers: 2
queue_size: 200
retry_on_failure:
enabled: true
initial_interval: 1s
max_interval: 10s
max_elapsed_time: 60s
otlphttp/loki:
endpoint: http://loki:3100/otlp
timeout: 5s
sending_queue:
enabled: true
num_consumers: 2
queue_size: 200
retry_on_failure:
enabled: true
initial_interval: 1s
max_interval: 10s
max_elapsed_time: 60s
otlphttp/tempo:
endpoint: http://tempo:4318
timeout: 5s
sending_queue:
enabled: true
num_consumers: 2
queue_size: 200
retry_on_failure:
enabled: true
initial_interval: 1s
max_interval: 10s
max_elapsed_time: 60s
service:
extensions: [health_check]
telemetry:
logs:
level: warn
metrics:
level: normal
readers:
- pull:
exporter:
prometheus:
host: 0.0.0.0
port: 8888
pipelines:
metrics:
receivers: [otlp]
processors: [memory_limiter, batch]
exporters: [otlphttp/prometheus]
logs:
receivers: [otlp]
processors: [memory_limiter, batch]
exporters: [otlphttp/loki]
traces:
receivers: [otlp]
processors: [memory_limiter, batch]
exporters: [otlphttp/tempo]
+23
View File
@@ -0,0 +1,23 @@
global:
scrape_interval: 15s
evaluation_interval: 15s
otlp:
promote_resource_attributes:
- deployment.environment.name
- service.version
storage:
tsdb:
out_of_order_time_window: 30m
rule_files:
- /etc/prometheus/rules/*.yml
scrape_configs:
- job_name: otel-collector
static_configs:
- targets: ["otel-collector:8888"]
- job_name: prometheus
static_configs:
- targets: ["localhost:9090"]
+77
View File
@@ -0,0 +1,77 @@
groups:
- name: messenger-sli
interval: 15s
rules:
- record: messenger:http_requests:rate5m
expr: sum(rate(http_server_requests_total{job="messenger-api", http_route!~"/health.*"}[5m]))
- record: messenger:http_errors:rate5m
expr: sum(rate(http_server_requests_total{job="messenger-api", http_route!~"/health.*", http_response_status_class="5xx"}[5m])) or vector(0)
- record: messenger:http_error_ratio:rate5m
expr: messenger:http_errors:rate5m / messenger:http_requests:rate5m
- record: messenger:http_error_ratio:rate1h
expr: |
(sum(rate(http_server_requests_total{job="messenger-api", http_route!~"/health.*", http_response_status_class="5xx"}[1h])) or vector(0))
/ sum(rate(http_server_requests_total{job="messenger-api", http_route!~"/health.*"}[1h]))
- record: messenger:http_latency_p95:rate5m
expr: histogram_quantile(0.95, sum by (le) (rate(http_server_request_duration_seconds_bucket{job="messenger-api", http_route!~"/health.*"}[5m])))
- record: messenger:http_fast_ratio:rate1h
expr: |
sum(rate(http_server_request_duration_seconds_bucket{job="messenger-api", http_route!~"/health.*", le="0.5"}[1h]))
/ sum(rate(http_server_request_duration_seconds_count{job="messenger-api", http_route!~"/health.*"}[1h]))
- record: messenger:time_to_ready_fast_ratio:rate1h
expr: |
sum(rate(attachments_time_to_ready_seconds_bucket{job="messenger-worker", le="10"}[1h]))
/ sum(rate(attachments_time_to_ready_seconds_count{job="messenger-worker"}[1h]))
- record: messenger:jobs_queue_oldest_age:max
expr: max(jobs_queue_oldest_age_seconds{job="messenger-relay"})
- name: messenger-alerts
rules:
- alert: MessengerHighErrorRate
expr: messenger:http_error_ratio:rate5m > 0.05 and messenger:http_requests:rate5m > 0.05
for: 1m
labels:
severity: critical
annotations:
summary: "Доля 5xx бизнес-запросов API {{ $value | humanizePercentage }} за 5 минут (порог 5%)"
description: "SLO доступности 99% за 1 ч: при 5% ошибок бюджет сгорает в 5 раз быстрее допустимого. Health-запросы не учитываются."
runbook: "docs/runbooks.md#messengerhigherrorrate"
- alert: MessengerApiNotReady
expr: sum(increase(http_server_requests_total{job="messenger-api", http_route="/health/core", http_response_status_class="5xx"}[1m])) > 10
for: 30s
labels:
severity: critical
annotations:
summary: "Реплики API отвечают 503 на /health/core — HAProxy выводит их из балансировки"
description: "Обычно недоступна БД или реплика останавливается. Пока все реплики не готовы, клиенты получают 503 от LB, и такие ответы не попадают в метрики API (MessengerHighErrorRate их не видит)."
runbook: "docs/runbooks.md#messengerapinotready"
- alert: MessengerJobQueueLag
expr: messenger:jobs_queue_oldest_age:max > 60
for: 1m
labels:
severity: warning
annotations:
summary: "Старейшее готовое к обработке задание ждёт {{ $value | humanizeDuration }} (порог 60 s)"
description: "Задания принимаются, но не обрабатываются: worker остановлен, не справляется или не получает сообщения из брокера."
runbook: "docs/runbooks.md#messengerjobqueuelag"
- alert: MessengerSlowApi
expr: messenger:http_latency_p95:rate5m > 1 and messenger:http_requests:rate5m > 0.05
for: 2m
labels:
severity: warning
annotations:
summary: "p95 бизнес-запросов API {{ $value | humanizeDuration }} за 5 минут (порог 1 s)"
description: "Проверьте ожидание пула БД (db_client_connection_pending_requests) и spans prisma:engine:db_query в трассах медленных запросов."
runbook: "docs/runbooks.md#messengerslowapi"
- alert: TelemetryExportFailing
expr: |
sum(rate(otelcol_exporter_send_failed_spans[2m])) + sum(rate(otelcol_exporter_send_failed_log_records[2m]))
+ sum(rate(otelcol_exporter_send_failed_metric_points[2m])) > 0
or max(otelcol_exporter_queue_size / otelcol_exporter_queue_capacity) > 0.5
for: 1m
labels:
severity: warning
annotations:
summary: "Collector не может доставить телеметрию в хранилище"
description: "Растёт очередь exporter или есть отказы отправки. Мессенджер работает, но часть трасс/логов теряется после заполнения ограниченной очереди."
runbook: "docs/runbooks.md#telemetryexportfailing"
+31
View File
@@ -0,0 +1,31 @@
stream_over_http_enabled: true
server:
http_listen_address: 0.0.0.0
http_listen_port: 3200
log_level: warn
distributor:
receivers:
otlp:
protocols:
http:
endpoint: 0.0.0.0:4318
ingester:
max_block_duration: 5m
compactor:
compaction:
block_retention: 168h
storage:
trace:
backend: local
wal:
path: /var/tempo/wal
local:
path: /var/tempo/blocks
usage_report:
reporting_enabled: false
+8
View File
@@ -0,0 +1,8 @@
{
"$schema": "https://json.schemastore.org/nest-cli",
"collection": "@nestjs/schematics",
"sourceRoot": "src",
"compilerOptions": {
"deleteOutDir": true
}
}
+13078
View File
File diff suppressed because it is too large Load Diff
+69
View File
@@ -0,0 +1,69 @@
{
"name": "mtusi-messenger-lab6",
"version": "1.0.0",
"description": "Lab 6: messenger API with S3 attachments and background image processing (NestJS + Fastify)",
"private": true,
"license": "UNLICENSED",
"scripts": {
"build": "nest build",
"start": "node dist/main.js",
"start:dev": "nest start --watch",
"lint": "eslint \"{src,test}/**/*.ts\"",
"test": "jest --config ./test/jest-e2e.json",
"test:watch": "jest --config ./test/jest-e2e.json --watch"
},
"dependencies": {
"@aws-sdk/client-s3": "3.700.0",
"@fastify/static": "7.0.4",
"@nestjs/common": "^10.4.15",
"@nestjs/core": "^10.4.15",
"@nestjs/platform-fastify": "^10.4.15",
"@nestjs/swagger": "7.4.2",
"@opentelemetry/api": "1.9.0",
"@opentelemetry/api-logs": "0.57.2",
"@opentelemetry/core": "1.30.1",
"@opentelemetry/exporter-logs-otlp-proto": "0.57.2",
"@opentelemetry/exporter-metrics-otlp-proto": "0.57.2",
"@opentelemetry/exporter-trace-otlp-proto": "0.57.2",
"@opentelemetry/instrumentation-aws-sdk": "0.49.1",
"@opentelemetry/instrumentation-http": "0.57.2",
"@opentelemetry/resources": "1.30.1",
"@opentelemetry/sdk-logs": "0.57.2",
"@opentelemetry/sdk-metrics": "1.30.1",
"@opentelemetry/sdk-node": "0.57.2",
"@opentelemetry/sdk-trace-base": "1.30.1",
"@opentelemetry/semantic-conventions": "1.28.0",
"@prisma/client": "6.1.0",
"@prisma/instrumentation": "6.1.0",
"amqplib": "0.10.5",
"argon2": "0.41.1",
"class-transformer": "^0.5.1",
"class-validator": "^0.14.1",
"ioredis": "5.4.1",
"reflect-metadata": "^0.2.2",
"rxjs": "^7.8.1",
"sharp": "0.33.5"
},
"devDependencies": {
"@nestjs/cli": "^10.4.9",
"@nestjs/schematics": "^10.2.3",
"@nestjs/testing": "^10.4.15",
"@types/amqplib": "0.10.6",
"@types/jest": "^29.5.14",
"@types/node": "^20.17.9",
"@types/supertest": "^6.0.2",
"@typescript-eslint/eslint-plugin": "^8.18.1",
"@typescript-eslint/parser": "^8.18.1",
"ajv": "8.12.0",
"eslint": "^8.57.1",
"jest": "^29.7.0",
"prisma": "6.1.0",
"source-map-support": "^0.5.21",
"supertest": "^7.0.0",
"ts-jest": "^29.2.5",
"ts-loader": "^9.5.1",
"ts-node": "^10.9.2",
"tsconfig-paths": "^4.2.0",
"typescript": "^5.7.2"
}
}
@@ -0,0 +1,50 @@
CREATE TABLE "users" (
"id" UUID NOT NULL,
"username" VARCHAR(32) NOT NULL,
"display_name" VARCHAR(100) NOT NULL,
"created_at" TIMESTAMPTZ(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
CONSTRAINT "users_pkey" PRIMARY KEY ("id"),
CONSTRAINT "users_username_format" CHECK ("username" ~ '^[a-z0-9_]{3,32}$'),
CONSTRAINT "users_display_name_not_blank" CHECK (btrim("display_name") <> '')
);
CREATE TABLE "chats" (
"id" UUID NOT NULL,
"title" VARCHAR(100) NOT NULL,
"created_at" TIMESTAMPTZ(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
CONSTRAINT "chats_pkey" PRIMARY KEY ("id"),
CONSTRAINT "chats_title_not_blank" CHECK (btrim("title") <> '')
);
CREATE TABLE "chat_members" (
"chat_id" UUID NOT NULL,
"user_id" UUID NOT NULL,
CONSTRAINT "chat_members_pkey" PRIMARY KEY ("chat_id","user_id")
);
CREATE TABLE "messages" (
"id" UUID NOT NULL,
"chat_id" UUID NOT NULL,
"sender_id" UUID NOT NULL,
"text" VARCHAR(4000) NOT NULL,
"created_at" TIMESTAMPTZ(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
CONSTRAINT "messages_pkey" PRIMARY KEY ("id")
);
CREATE UNIQUE INDEX "users_username_key" ON "users"("username");
CREATE INDEX "chat_members_user_id_idx" ON "chat_members"("user_id");
CREATE INDEX "messages_chat_id_created_at_id_idx" ON "messages"("chat_id", "created_at", "id");
ALTER TABLE "chat_members" ADD CONSTRAINT "chat_members_chat_id_fkey" FOREIGN KEY ("chat_id") REFERENCES "chats"("id") ON DELETE CASCADE ON UPDATE CASCADE;
ALTER TABLE "chat_members" ADD CONSTRAINT "chat_members_user_id_fkey" FOREIGN KEY ("user_id") REFERENCES "users"("id") ON DELETE RESTRICT ON UPDATE CASCADE;
ALTER TABLE "messages" ADD CONSTRAINT "messages_chat_id_fkey" FOREIGN KEY ("chat_id") REFERENCES "chats"("id") ON DELETE CASCADE ON UPDATE CASCADE;
ALTER TABLE "messages" ADD CONSTRAINT "messages_sender_id_fkey" FOREIGN KEY ("sender_id") REFERENCES "users"("id") ON DELETE RESTRICT ON UPDATE CASCADE;
@@ -0,0 +1,14 @@
ALTER TABLE "users" ADD COLUMN "password_hash" TEXT NOT NULL;
CREATE TABLE "sessions" (
"token_hash" CHAR(64) NOT NULL,
"user_id" UUID NOT NULL,
"expires_at" TIMESTAMPTZ(3) NOT NULL,
"created_at" TIMESTAMPTZ(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
CONSTRAINT "sessions_pkey" PRIMARY KEY ("token_hash")
);
CREATE INDEX "sessions_user_id_idx" ON "sessions"("user_id");
ALTER TABLE "sessions" ADD CONSTRAINT "sessions_user_id_fkey" FOREIGN KEY ("user_id") REFERENCES "users"("id") ON DELETE CASCADE ON UPDATE CASCADE;
@@ -0,0 +1,13 @@
CREATE TABLE "idempotency_keys" (
"user_id" UUID NOT NULL,
"chat_id" UUID NOT NULL,
"key" VARCHAR(128) NOT NULL,
"request_hash" CHAR(64) NOT NULL,
"message_id" UUID NOT NULL,
"created_at" TIMESTAMPTZ(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
CONSTRAINT "idempotency_keys_pkey" PRIMARY KEY ("user_id","chat_id","key"),
CONSTRAINT "idempotency_keys_key_format" CHECK ("key" ~ '^[A-Za-z0-9._:-]{1,128}$')
);
ALTER TABLE "idempotency_keys" ADD CONSTRAINT "idempotency_keys_message_id_fkey" FOREIGN KEY ("message_id") REFERENCES "messages"("id") ON DELETE CASCADE ON UPDATE CASCADE;
@@ -0,0 +1,45 @@
CREATE TABLE "attachments" (
"id" UUID NOT NULL,
"chat_id" UUID NOT NULL,
"owner_id" UUID NOT NULL,
"filename" VARCHAR(255) NOT NULL,
"content_type" VARCHAR(32) NOT NULL,
"size_bytes" INTEGER NOT NULL,
"sha256" CHAR(64) NOT NULL,
"object_key" VARCHAR(128) NOT NULL,
"created_at" TIMESTAMPTZ(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
"updated_at" TIMESTAMPTZ(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
CONSTRAINT "attachments_pkey" PRIMARY KEY ("id"),
CONSTRAINT "attachments_filename_not_blank" CHECK (btrim("filename") <> ''),
CONSTRAINT "attachments_content_type_allowed" CHECK ("content_type" IN ('image/png', 'image/jpeg')),
CONSTRAINT "attachments_size_range" CHECK ("size_bytes" BETWEEN 1 AND 10485760),
CONSTRAINT "attachments_sha256_format" CHECK ("sha256" ~ '^[a-f0-9]{64}$')
);
CREATE TABLE "message_attachments" (
"message_id" UUID NOT NULL,
"position" SMALLINT NOT NULL,
"attachment_id" UUID NOT NULL,
CONSTRAINT "message_attachments_pkey" PRIMARY KEY ("message_id","position"),
CONSTRAINT "message_attachments_position_range" CHECK ("position" BETWEEN 0 AND 9)
);
CREATE UNIQUE INDEX "attachments_object_key_key" ON "attachments"("object_key");
CREATE INDEX "attachments_chat_id_idx" ON "attachments"("chat_id");
CREATE INDEX "attachments_owner_id_idx" ON "attachments"("owner_id");
CREATE UNIQUE INDEX "message_attachments_message_id_attachment_id_key" ON "message_attachments"("message_id", "attachment_id");
CREATE INDEX "message_attachments_attachment_id_idx" ON "message_attachments"("attachment_id");
ALTER TABLE "attachments" ADD CONSTRAINT "attachments_chat_id_fkey" FOREIGN KEY ("chat_id") REFERENCES "chats"("id") ON DELETE CASCADE ON UPDATE CASCADE;
ALTER TABLE "attachments" ADD CONSTRAINT "attachments_owner_id_fkey" FOREIGN KEY ("owner_id") REFERENCES "users"("id") ON DELETE RESTRICT ON UPDATE CASCADE;
ALTER TABLE "message_attachments" ADD CONSTRAINT "message_attachments_message_id_fkey" FOREIGN KEY ("message_id") REFERENCES "messages"("id") ON DELETE CASCADE ON UPDATE CASCADE;
ALTER TABLE "message_attachments" ADD CONSTRAINT "message_attachments_attachment_id_fkey" FOREIGN KEY ("attachment_id") REFERENCES "attachments"("id") ON DELETE NO ACTION ON UPDATE CASCADE;
@@ -0,0 +1,97 @@
ALTER TABLE "attachments"
ADD COLUMN "staging_key" VARCHAR(128),
ADD COLUMN "status" VARCHAR(16) NOT NULL DEFAULT 'ready',
ADD COLUMN "error_code" VARCHAR(64);
ALTER TABLE "attachments" ALTER COLUMN "status" DROP DEFAULT;
ALTER TABLE "attachments"
ADD CONSTRAINT "attachments_status_allowed" CHECK ("status" IN ('queued', 'processing', 'ready', 'failed')),
ADD CONSTRAINT "attachments_error_code_when_failed" CHECK (("status" = 'failed') = ("error_code" IS NOT NULL));
CREATE UNIQUE INDEX "attachments_staging_key_key" ON "attachments"("staging_key");
CREATE TABLE "attachment_variants" (
"attachment_id" UUID NOT NULL,
"pipeline_version" INTEGER NOT NULL,
"name" VARCHAR(16) NOT NULL,
"content_type" VARCHAR(32) NOT NULL,
"object_key" VARCHAR(128) NOT NULL,
"size_bytes" INTEGER NOT NULL,
"width" INTEGER NOT NULL,
"height" INTEGER NOT NULL,
"sha256" CHAR(64) NOT NULL,
"created_at" TIMESTAMPTZ(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
CONSTRAINT "attachment_variants_pkey" PRIMARY KEY ("attachment_id","pipeline_version","name"),
CONSTRAINT "attachment_variants_name_allowed" CHECK ("name" IN ('thumbnail', 'crop', 'optimized')),
CONSTRAINT "attachment_variants_content_type_allowed" CHECK ("content_type" IN ('image/jpeg', 'image/png', 'image/webp')),
CONSTRAINT "attachment_variants_positive" CHECK ("pipeline_version" >= 1 AND "size_bytes" >= 1 AND "width" >= 1 AND "height" >= 1),
CONSTRAINT "attachment_variants_sha256_format" CHECK ("sha256" ~ '^[a-f0-9]{64}$')
);
CREATE TABLE "attachment_jobs" (
"id" UUID NOT NULL,
"attachment_id" UUID NOT NULL,
"pipeline_version" INTEGER NOT NULL,
"status" VARCHAR(16) NOT NULL,
"attempt" INTEGER NOT NULL,
"max_attempts" INTEGER NOT NULL,
"next_attempt_at" TIMESTAMPTZ(3) NOT NULL,
"lease_owner" UUID,
"lease_until" TIMESTAMPTZ(3),
"error_code" VARCHAR(64),
"last_error" VARCHAR(200),
"created_at" TIMESTAMPTZ(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
"updated_at" TIMESTAMPTZ(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
"finished_at" TIMESTAMPTZ(3),
CONSTRAINT "attachment_jobs_pkey" PRIMARY KEY ("id"),
CONSTRAINT "attachment_jobs_status_allowed" CHECK ("status" IN ('queued', 'processing', 'ready', 'failed')),
CONSTRAINT "attachment_jobs_attempt_range" CHECK ("pipeline_version" >= 1 AND "attempt" >= 1 AND "attempt" <= "max_attempts"),
CONSTRAINT "attachment_jobs_lease_when_processing" CHECK ("status" <> 'processing' OR ("lease_owner" IS NOT NULL AND "lease_until" IS NOT NULL)),
CONSTRAINT "attachment_jobs_error_code_when_failed" CHECK (("status" = 'failed') = ("error_code" IS NOT NULL))
);
CREATE TABLE "job_events" (
"id" BIGSERIAL NOT NULL,
"job_id" UUID NOT NULL,
"attempt" INTEGER NOT NULL,
"event" VARCHAR(32) NOT NULL,
"detail" VARCHAR(200),
"created_at" TIMESTAMPTZ(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
CONSTRAINT "job_events_pkey" PRIMARY KEY ("id")
);
CREATE TABLE "outbox" (
"id" BIGSERIAL NOT NULL,
"job_id" UUID NOT NULL,
"payload" JSONB NOT NULL,
"available_at" TIMESTAMPTZ(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
"published_at" TIMESTAMPTZ(3),
"publish_attempts" INTEGER NOT NULL DEFAULT 0,
"created_at" TIMESTAMPTZ(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
CONSTRAINT "outbox_pkey" PRIMARY KEY ("id")
);
CREATE UNIQUE INDEX "attachment_variants_object_key_key" ON "attachment_variants"("object_key");
CREATE UNIQUE INDEX "attachment_jobs_attachment_id_pipeline_version_key" ON "attachment_jobs"("attachment_id", "pipeline_version");
CREATE INDEX "attachment_jobs_status_lease_until_idx" ON "attachment_jobs"("status", "lease_until");
CREATE INDEX "job_events_job_id_idx" ON "job_events"("job_id");
CREATE INDEX "outbox_published_at_available_at_idx" ON "outbox"("published_at", "available_at");
CREATE INDEX "outbox_job_id_idx" ON "outbox"("job_id");
ALTER TABLE "attachment_variants" ADD CONSTRAINT "attachment_variants_attachment_id_fkey" FOREIGN KEY ("attachment_id") REFERENCES "attachments"("id") ON DELETE CASCADE ON UPDATE CASCADE;
ALTER TABLE "attachment_jobs" ADD CONSTRAINT "attachment_jobs_attachment_id_fkey" FOREIGN KEY ("attachment_id") REFERENCES "attachments"("id") ON DELETE CASCADE ON UPDATE CASCADE;
ALTER TABLE "job_events" ADD CONSTRAINT "job_events_job_id_fkey" FOREIGN KEY ("job_id") REFERENCES "attachment_jobs"("id") ON DELETE CASCADE ON UPDATE CASCADE;
ALTER TABLE "outbox" ADD CONSTRAINT "outbox_job_id_fkey" FOREIGN KEY ("job_id") REFERENCES "attachment_jobs"("id") ON DELETE CASCADE ON UPDATE CASCADE;
+3
View File
@@ -0,0 +1,3 @@
# Please do not edit this file manually
# It should be added in your version-control system (i.e. Git)
provider = "postgresql"
+191
View File
@@ -0,0 +1,191 @@
generator client {
provider = "prisma-client-js"
previewFeatures = ["metrics"]
}
datasource db {
provider = "postgresql"
url = env("DATABASE_URL")
}
model User {
id String @id @default(uuid()) @db.Uuid
username String @unique @db.VarChar(32)
displayName String @map("display_name") @db.VarChar(100)
passwordHash String @map("password_hash")
createdAt DateTime @default(now()) @map("created_at") @db.Timestamptz(3)
memberships ChatMember[]
messages Message[]
sessions Session[]
attachments Attachment[]
@@map("users")
}
model Session {
tokenHash String @id @map("token_hash") @db.Char(64)
userId String @map("user_id") @db.Uuid
expiresAt DateTime @map("expires_at") @db.Timestamptz(3)
createdAt DateTime @default(now()) @map("created_at") @db.Timestamptz(3)
user User @relation(fields: [userId], references: [id], onDelete: Cascade)
@@index([userId])
@@map("sessions")
}
model Chat {
id String @id @default(uuid()) @db.Uuid
title String @db.VarChar(100)
createdAt DateTime @default(now()) @map("created_at") @db.Timestamptz(3)
members ChatMember[]
messages Message[]
attachments Attachment[]
@@map("chats")
}
model ChatMember {
chatId String @map("chat_id") @db.Uuid
userId String @map("user_id") @db.Uuid
chat Chat @relation(fields: [chatId], references: [id], onDelete: Cascade)
user User @relation(fields: [userId], references: [id], onDelete: Restrict)
@@id([chatId, userId])
@@index([userId])
@@map("chat_members")
}
model Message {
id String @id @default(uuid()) @db.Uuid
chatId String @map("chat_id") @db.Uuid
senderId String @map("sender_id") @db.Uuid
text String @db.VarChar(4000)
createdAt DateTime @default(now()) @map("created_at") @db.Timestamptz(3)
chat Chat @relation(fields: [chatId], references: [id], onDelete: Cascade)
sender User @relation(fields: [senderId], references: [id], onDelete: Restrict)
idempotencyKeys IdempotencyKey[]
attachments MessageAttachment[]
@@index([chatId, createdAt, id])
@@map("messages")
}
model Attachment {
id String @id @default(uuid()) @db.Uuid
chatId String @map("chat_id") @db.Uuid
ownerId String @map("owner_id") @db.Uuid
filename String @db.VarChar(255)
contentType String @map("content_type") @db.VarChar(32)
sizeBytes Int @map("size_bytes")
sha256 String @db.Char(64)
objectKey String @unique @map("object_key") @db.VarChar(128)
stagingKey String? @unique @map("staging_key") @db.VarChar(128)
status String @db.VarChar(16)
errorCode String? @map("error_code") @db.VarChar(64)
createdAt DateTime @default(now()) @map("created_at") @db.Timestamptz(3)
updatedAt DateTime @default(now()) @updatedAt @map("updated_at") @db.Timestamptz(3)
chat Chat @relation(fields: [chatId], references: [id], onDelete: Cascade)
owner User @relation(fields: [ownerId], references: [id], onDelete: Restrict)
messages MessageAttachment[]
variants AttachmentVariant[]
jobs AttachmentJob[]
@@index([chatId])
@@index([ownerId])
@@map("attachments")
}
model AttachmentVariant {
attachmentId String @map("attachment_id") @db.Uuid
pipelineVersion Int @map("pipeline_version")
name String @db.VarChar(16)
contentType String @map("content_type") @db.VarChar(32)
objectKey String @unique @map("object_key") @db.VarChar(128)
sizeBytes Int @map("size_bytes")
width Int
height Int
sha256 String @db.Char(64)
createdAt DateTime @default(now()) @map("created_at") @db.Timestamptz(3)
attachment Attachment @relation(fields: [attachmentId], references: [id], onDelete: Cascade)
@@id([attachmentId, pipelineVersion, name])
@@map("attachment_variants")
}
model AttachmentJob {
id String @id @db.Uuid
attachmentId String @map("attachment_id") @db.Uuid
pipelineVersion Int @map("pipeline_version")
status String @db.VarChar(16)
attempt Int
maxAttempts Int @map("max_attempts")
nextAttemptAt DateTime @map("next_attempt_at") @db.Timestamptz(3)
leaseOwner String? @map("lease_owner") @db.Uuid
leaseUntil DateTime? @map("lease_until") @db.Timestamptz(3)
errorCode String? @map("error_code") @db.VarChar(64)
lastError String? @map("last_error") @db.VarChar(200)
createdAt DateTime @default(now()) @map("created_at") @db.Timestamptz(3)
updatedAt DateTime @default(now()) @updatedAt @map("updated_at") @db.Timestamptz(3)
finishedAt DateTime? @map("finished_at") @db.Timestamptz(3)
attachment Attachment @relation(fields: [attachmentId], references: [id], onDelete: Cascade)
events JobEvent[]
outbox OutboxEntry[]
@@unique([attachmentId, pipelineVersion])
@@index([status, leaseUntil])
@@map("attachment_jobs")
}
model JobEvent {
id BigInt @id @default(autoincrement())
jobId String @map("job_id") @db.Uuid
attempt Int
event String @db.VarChar(32)
detail String? @db.VarChar(200)
createdAt DateTime @default(now()) @map("created_at") @db.Timestamptz(3)
job AttachmentJob @relation(fields: [jobId], references: [id], onDelete: Cascade)
@@index([jobId])
@@map("job_events")
}
model OutboxEntry {
id BigInt @id @default(autoincrement())
jobId String @map("job_id") @db.Uuid
payload Json
availableAt DateTime @default(now()) @map("available_at") @db.Timestamptz(3)
publishedAt DateTime? @map("published_at") @db.Timestamptz(3)
publishAttempts Int @default(0) @map("publish_attempts")
createdAt DateTime @default(now()) @map("created_at") @db.Timestamptz(3)
job AttachmentJob @relation(fields: [jobId], references: [id], onDelete: Cascade)
@@index([publishedAt, availableAt])
@@index([jobId])
@@map("outbox")
}
model MessageAttachment {
messageId String @map("message_id") @db.Uuid
position Int @db.SmallInt
attachmentId String @map("attachment_id") @db.Uuid
message Message @relation(fields: [messageId], references: [id], onDelete: Cascade)
attachment Attachment @relation(fields: [attachmentId], references: [id], onDelete: NoAction)
@@id([messageId, position])
@@unique([messageId, attachmentId])
@@index([attachmentId])
@@map("message_attachments")
}
model IdempotencyKey {
userId String @map("user_id") @db.Uuid
chatId String @map("chat_id") @db.Uuid
key String @db.VarChar(128)
requestHash String @map("request_hash") @db.Char(64)
messageId String @map("message_id") @db.Uuid
createdAt DateTime @default(now()) @map("created_at") @db.Timestamptz(3)
message Message @relation(fields: [messageId], references: [id], onDelete: Cascade)
@@id([userId, chatId, key])
@@map("idempotency_keys")
}
+21
View File
@@ -0,0 +1,21 @@
#!/usr/bin/env python3
import sys
import urllib.error
import urllib.request
bucket, key = sys.argv[1:3]
base = 'http://s3:9000/' + bucket
attempts = (
('GET', '/<object>', base + '/' + key, None),
('GET', '/', base + '/', None),
('PUT', '/anon.txt', base + '/anon.txt', b'x'),
)
failed = False
for method, label, url, data in attempts:
try:
code = urllib.request.urlopen(urllib.request.Request(url, data=data, method=method), timeout=5).status
except urllib.error.HTTPError as error:
code = error.code
print(method, bucket + label, '->', code)
failed = failed or code != 403
sys.exit(1 if failed else 0)
+46
View File
@@ -0,0 +1,46 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
bucket="${S3_BUCKET:-messenger-attachments}"
object="check/$(date +%s)-$$.txt"
as_api() {
docker compose run --rm --no-deps -T -e BUCKET="$bucket" -e OBJECT="$object" --entrypoint sh s3-init -c "$1"
}
echo "## S3 не опубликован наружу, консоль выключена"
docker compose ps s3 --format '{{.Service}} ports={{.Ports}}'
echo "MINIO_BROWSER=$(docker compose exec -T s3 printenv MINIO_BROWSER)"
echo
echo "## Права пользователя API: только свой bucket"
as_api '
set -e
cd /run/secrets
export MC_HOST_api="http://$(cat s3_access_key_id):$(cat s3_secret_access_key)@s3:9000"
export MC_HOST_root="http://$(cat s3_root_user):$(cat s3_root_password)@s3:9000"
expect() { want=$1; label=$2; shift 2; if "$@" >/dev/null 2>&1; then got=allowed; else got=denied; fi; echo "$label -> $got"; [ "$got" = "$want" ]; }
echo probe > /tmp/probe.txt
mc mb --ignore-existing root/check-other-bucket >/dev/null
mc cp /tmp/probe.txt root/check-other-bucket/probe.txt >/dev/null
trap "mc rb --force root/check-other-bucket >/dev/null" EXIT
expect allowed "PUT в свой bucket" mc cp /tmp/probe.txt "api/$BUCKET/$OBJECT"
expect allowed "GET из своего bucket" mc cat "api/$BUCKET/$OBJECT"
expect denied "создание другого bucket" mc mb api/forbidden-bucket
expect denied "GET из другого bucket" mc cat api/check-other-bucket/probe.txt
expect denied "PUT в другой bucket" mc cp /tmp/probe.txt api/check-other-bucket/new.txt
expect denied "изменение политики bucket" mc anonymous set public "api/$BUCKET"
expect denied "администрирование пользователей" mc admin user list api
'
echo
echo "## Анонимный доступ напрямую к S3, мимо API"
python3 scripts/check-s3-anonymous.py "$bucket" "$object"
as_api '
cd /run/secrets
export MC_HOST_api="http://$(cat s3_access_key_id):$(cat s3_secret_access_key)@s3:9000"
mc rm "api/$BUCKET/$OBJECT" >/dev/null
'
echo
echo "PASS: bucket приватный, у API нет прав вне своего bucket"
+161
View File
@@ -0,0 +1,161 @@
#!/bin/sh
set -u
cd "$(dirname "$0")/.."
mkdir -p evidence
COMMIT="$(git rev-parse --short HEAD)$(git diff --quiet HEAD -- src prisma test scripts infra compose.yaml Dockerfile || echo ' (dirty)')"
header() {
printf '# %s\n# date: %s\n# commit: %s\n\n' "$1" "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "$COMMIT"
}
run() {
echo "\$ $*"
"$@" 2>&1 || echo "EXIT CODE $?"
echo
}
started() {
docker inspect -f '{{.Name}} {{.State.Status}} started={{.State.StartedAt}}' $(docker compose ps -aq postgres redis s3 rabbitmq api lb)
}
api_states() {
docker inspect -f '{{.Name}} id={{slice .Id 0 12}} {{.State.Status}} exit={{.State.ExitCode}} finished={{.State.FinishedAt}}' $(docker compose ps -aq api)
}
restore_api() {
for id in $(docker compose ps -aq --status exited api); do
docker start "$id" >/dev/null
done
python3 scripts/probe.py ready >/dev/null
sleep 3
}
echo 'container.txt'
{
header 'Стенд: сервисы, порты, лимиты, секреты, миграции, топология брокера'
run docker compose ps -a --format 'table {{.Service}}\t{{.Name}}\t{{.Image}}\t{{.Status}}\t{{.Ports}}'
echo '## CPU/RAM-лимиты (NanoCpus: 1e9 = 1 CPU; Memory в байтах)'
run sh -c "docker inspect -f '{{.Name}} cpus={{.HostConfig.NanoCpus}} memory={{.HostConfig.Memory}}' \$(docker compose ps -aq)"
echo '## Опубликованные порты: только LB и Grafana на 127.0.0.1; OTLP, Prometheus, Loki, Tempo наружу не открыты'
run sh -c "docker compose ps --format '{{.Service}} {{.Ports}}' | grep -v '^\\S* *$' || true"
echo '## Сети телеметрии: хранилища только во внутренней сети telemetry'
run sh -c "for s in otel-collector prometheus loki tempo grafana api worker; do printf '%s: ' \$s; docker inspect -f '{{range \$k, \$v := .NetworkSettings.Networks}}{{\$k}} {{end}}' \$(docker compose ps -q \$s | head -1); done"
run docker network inspect -f '{{.Name}} internal={{.Internal}}' messenger-lab6_telemetry
echo '## Секреты: каждый сервис видит только свои файлы (содержимое не выводится)'
for service in postgres redis s3 api reconciler relay worker grafana; do
run docker compose exec -T "$service" ls /run/secrets
done
run docker compose exec -T rabbitmq ls /etc/rabbitmq/conf.d
run docker compose exec -T lb sh -c 'ls /run/secrets 2>/dev/null || echo "no /run/secrets"'
echo '## Версии'
run docker compose exec -T lb haproxy -v
run docker compose exec -T postgres postgres --version
run docker compose exec -T redis redis-server --version
run docker compose exec -T s3 minio --version
run docker compose exec -T rabbitmq rabbitmqctl -q version
run sh -c "docker compose images otel-collector prometheus loki tempo grafana"
echo '## Топология брокера: durable quorum-очереди, delivery-limit и DLQ'
run docker compose exec -T rabbitmq rabbitmqctl -q list_queues name type durable arguments messages
run docker compose exec -T rabbitmq rabbitmqctl -q list_consumers queue_name prefetch_count ack_required
echo '## Таблицы'
run docker compose exec -T postgres psql -U messenger -d messenger -c '\d attachments' -c '\d attachment_variants' -c '\d attachment_jobs' -c '\d outbox' -c '\d job_events'
echo '## Миграции: отдельный one-shot сервис, повторный запуск ничего не применяет, схема совпадает с БД'
run docker compose run --rm migrate
run docker compose run --rm --no-deps migrate sh -c 'export DATABASE_URL="$(cat "$DATABASE_URL_FILE")"; npx prisma migrate diff --from-url "$DATABASE_URL" --to-schema-datamodel prisma/schema.prisma --exit-code'
run docker compose exec -T postgres psql -U messenger -d messenger -c 'SELECT migration_name, finished_at FROM _prisma_migrations ORDER BY started_at'
} > evidence/container.txt
echo 'tests.txt'
{
header 'Собственные e2e-тесты (PostgreSQL + Redis + MinIO из compose.test.yaml); основной стенд на время тестов остановлен ради памяти Docker'
docker compose stop lb api worker relay reconciler otel-collector prometheus loki tempo grafana rabbitmq s3 redis postgres >/dev/null 2>&1
docker compose -f compose.test.yaml build -q tests
run docker compose -f compose.test.yaml run --rm tests npx jest --config ./test/jest-e2e.json --verbose
docker compose -f compose.test.yaml down -v >/dev/null 2>&1
docker compose start postgres redis s3 rabbitmq api worker relay reconciler lb otel-collector prometheus loki tempo grafana >/dev/null 2>&1
python3 scripts/probe.py ready >/dev/null
sleep 20
} > evidence/tests.txt
echo 'smoke.txt'
{
header 'Публичные проверки через LB; разные реплики обслуживают одну сессию'
run make check
run make test
run python3 scripts/probe.py instances 6
} > evidence/smoke.txt
echo 'persistence.txt'
{
header 'make test-persistence: restart.sh перезапускает postgres, redis, s3, rabbitmq, api, lb с сохранением volumes'
echo '## до'
started
echo
run make test-persistence ACTION_SCRIPT=scripts/restart.sh
echo '## после'
started
} > evidence/persistence.txt
echo 'failover.txt'
{
header 'make test-failover: stop-one.sh останавливает реплику, обслужившую тест'
echo '## до'
api_states
echo
run make test-failover ACTION_SCRIPT=scripts/stop-one.sh
echo '## после: остановленная реплика Exited (0) — штатное завершение по SIGTERM'
api_states
restore_api
} > evidence/failover.txt
echo 's3.txt'
{
header 'S3 (минимум lab4): идемпотентный bootstrap, приватность мимо API, права пользователя API'
run docker compose run --rm --no-deps -T s3-init
run docker compose run --rm --no-deps -T s3-init
run sh scripts/check-s3.sh
} > evidence/s3.txt
echo 'jobs.txt'
{
header 'Фоновая обработка: сценарии защиты уровней 3 и 4 (scripts/job-scenario.py)'
for scenario in restart crash-after-store crash-after-commit outbox-gap relay-crash retry-replay broker; do
echo "## $scenario"
run python3 scripts/job-scenario.py "$scenario"
done
echo '## Итог по всем заданиям стенда'
run docker compose exec -T postgres psql -U messenger -d messenger -c 'SELECT status, count(*) FROM attachment_jobs GROUP BY status' -c 'SELECT count(*) AS unpublished_outbox FROM outbox WHERE published_at IS NULL' -c 'SELECT count(*) AS staging_left_on_ready FROM attachments WHERE status = '"'ready'"' AND staging_key IS NOT NULL'
echo '## Последний проход reconciler'
docker compose logs --no-log-prefix --tail 1 reconciler 2>&1
} > evidence/jobs.txt
echo 'telemetry.txt'
{
header 'Наблюдаемость: контрольный запрос (trace + логи), dashboard, перцентили, приватность, правила'
echo '## Prometheus: правила загружены (promtool и /api/v1/rules)'
run docker compose exec -T prometheus promtool check rules /etc/prometheus/rules/messenger.yml
run sh -c "wget -qO- http://grafana:3000/api/datasources/proxy/uid/prometheus/api/v1/rules | python3 -c 'import json,sys; [print(r[\"type\"], r[\"name\"], r[\"health\"]) for g in json.load(sys.stdin)[\"data\"][\"groups\"] for r in g[\"rules\"]]'"
echo '## Grafana: datasources с фиксированными UID, dashboard из provisioning'
run sh -c "for uid in prometheus loki tempo; do wget -qO- http://grafana:3000/api/datasources/uid/\$uid | python3 -c 'import json,sys; d=json.load(sys.stdin); print(d[\"uid\"], d[\"type\"], d[\"url\"])'; done"
echo '## Контрольный upload с клиентским traceparent: trace в Tempo и логи в Loki'
run python3 scripts/trace-check.py
echo '## Секреты и содержимое не попадают в telemetry; в labels метрик нет ID'
run python3 scripts/telemetry-privacy.py
echo '## Перцентили dashboard против клиентского замера tests/load.py'
run python3 scripts/latency-compare.py --duration 30
echo '## Все запросы dashboard (пустые панели alerts/WARN-логов допустимы без инцидентов)'
run python3 scripts/dashboard-check.py
echo '## Пример структурного лога (stdout API)'
docker compose logs --no-log-prefix --tail 200 api 2>&1 | grep '"event":"http.request"' | tail -2
} > evidence/telemetry.txt
for scenario in retry-trace worker-stop db telemetry; do
echo "incident-$scenario.txt"
{
header "Инцидент: scripts/incidents.py $scenario"
run python3 scripts/incidents.py "$scenario"
} > "evidence/incident-$scenario.txt"
done
echo 'done'
+57
View File
@@ -0,0 +1,57 @@
#!/usr/bin/env python3
"""Runs every query of the provisioned dashboard through Grafana datasource proxies and reports empty panels.
Run inside the tools container: docker compose run --rm tools python3 scripts/dashboard-check.py
"""
import json
import os
import sys
import time
import urllib.parse
import urllib.request
GRAFANA = os.environ.get('GRAFANA_URL', 'http://grafana:3000')
DASHBOARD = os.path.join(os.path.dirname(__file__), '..', 'infra', 'grafana', 'dashboards', 'messenger.json')
def get(path, **params):
url = GRAFANA + path + ('?' + urllib.parse.urlencode(params) if params else '')
with urllib.request.urlopen(url, timeout=15) as response:
return json.loads(response.read())
def run(target):
uid = target['datasource']['uid']
now = int(time.time())
if uid == 'prometheus':
expr = target['expr'].replace('$__rate_interval', '1m')
body = get('/api/datasources/proxy/uid/prometheus/api/v1/query_range', query=expr, start=now - 1800, end=now, step=30)
series = body['data']['result']
last = [s['values'][-1][1] for s in series if s['values']]
return len(series), ', '.join('%s=%s' % (s['metric'].get('outcome') or s['metric'].get('http_response_status_class') or '', v)
for s, v in zip(series[:4], last[:4]))
if uid == 'loki':
body = get('/api/datasources/proxy/uid/loki/loki/api/v1/query_range', query=target['expr'], limit=20, since='30m')
return sum(len(s['values']) for s in body['data']['result']), ''
if uid == 'tempo':
body = get('/api/datasources/proxy/uid/tempo/api/search', q=target['query'], limit=target.get('limit', 20), start=now - 1800, end=now)
return len(body.get('traces', [])), ''
return 0, 'unknown datasource'
def main():
grafana = get('/api/dashboards/uid/messenger-overview')
print('Grafana dashboard:', grafana['dashboard']['title'], 'folder:', grafana['meta'].get('folderTitle'))
dashboard = json.load(open(DASHBOARD, encoding='utf-8'))
empty = 0
for panel in dashboard['panels']:
for target in panel.get('targets', []):
count, sample = run(target)
empty += count == 0
print('%-4s %-62s %-3s %4d %s' % ('OK' if count else 'EMPTY', panel['title'][:62], target['refId'], count, sample[:70]))
print('\n%s: %d empty queries' % ('PASS' if empty == 0 else 'WARN', empty))
sys.exit(0 if empty == 0 else 1)
if __name__ == '__main__':
main()
+389
View File
@@ -0,0 +1,389 @@
#!/usr/bin/env python3
"""Controlled incidents for lab6 observability; run inside the tools container.
docker compose run --rm tools python3 scripts/incidents.py worker-stop|db|telemetry|retry-trace
Every scenario injects a fault, keeps synthetic traffic going, samples Prometheus/Tempo/Loki through Grafana
and waits for the related alert to become firing and then resolved after recovery.
"""
import base64
import importlib.util
import json
import os
import statistics
import struct
import subprocess
import sys
import threading
import time
import urllib.parse
import urllib.request
import uuid
import zlib
spec = importlib.util.spec_from_file_location('job_scenario', os.path.join(os.path.dirname(__file__), 'job-scenario.py'))
js = importlib.util.module_from_spec(spec)
spec.loader.exec_module(js)
log = js.log
GRAFANA = os.environ.get('GRAFANA_URL', 'http://grafana:3000')
PROM = GRAFANA + '/api/datasources/proxy/uid/prometheus/api/v1/'
TEMPO = GRAFANA + '/api/datasources/proxy/uid/tempo/api/'
LOKI = GRAFANA + '/api/datasources/proxy/uid/loki/loki/api/v1/'
def get(url, **params):
with urllib.request.urlopen(url + ('?' + urllib.parse.urlencode(params) if params else ''), timeout=20) as r:
return json.loads(r.read())
def prom(expr):
try:
result = get(PROM + 'query', query=expr)['data']['result']
except OSError:
return None
return float(result[0]['value'][1]) if result else None
def alert_state(name):
try:
alerts = get(PROM + 'alerts')['data']['alerts']
except OSError:
return 'unknown'
states = [a['state'] for a in alerts if a['labels'].get('alertname') == name]
return 'firing' if 'firing' in states else ('pending' if 'pending' in states else 'inactive')
def fmt(value, unit=''):
if value is None:
return '-'
if unit == '%':
return '%.1f%%' % (value * 100)
if unit == 'ms':
return '%.0fms' % (value * 1000)
if unit == 'MiB':
return '%.0fMiB' % (value / 2 ** 20)
return ('%.1f' % value).rstrip('0').rstrip('.')
def sample(columns, alerts):
parts = ['%s=%s' % (label, fmt(prom(expr), unit)) for label, expr, unit in columns]
parts += ['%s=%s' % (name, alert_state(name)) for name in alerts]
log(' '.join(parts))
def watch(columns, alerts, until, timeout, every=10):
deadline = time.monotonic() + timeout
while time.monotonic() < deadline:
sample(columns, alerts)
if until():
return True
time.sleep(every)
sample(columns, alerts)
return False
class Traffic:
"""Closed-loop synthetic traffic through the LB: POST message + GET history, counts statuses per phase."""
def __init__(self, auth, chat, threads=4, pause=0.2):
self.auth, self.chat, self.threads, self.pause = auth, chat, threads, pause
self.stop = threading.Event()
self.lock = threading.Lock()
self.reset()
def reset(self):
with self.lock:
self.codes, self.latencies = {}, []
def snapshot(self, phase):
with self.lock:
codes, latencies = dict(self.codes), sorted(self.latencies)
p = lambda q: latencies[min(len(latencies) - 1, int(q * len(latencies)))] * 1000 if latencies else 0
log('client [%s]: %s; p50=%.0fms p95=%.0fms p99=%.0fms' % (
phase, ' '.join('%s:%d' % kv for kv in sorted(codes.items())), p(.5), p(.95), p(.99)))
self.reset()
def run(self):
def loop(n):
i = 0
while not self.stop.is_set():
started = time.monotonic()
if i % 2 == 0:
code = js.call('POST', '/api/v1/chats/%s/messages' % self.chat, {'text': 'incident %d-%d' % (n, i)},
{**self.auth, 'Idempotency-Key': uuid.uuid4().hex})[0]
else:
code = js.call('GET', '/api/v1/chats/%s/messages?limit=20' % self.chat, headers=self.auth)[0]
with self.lock:
self.codes[code] = self.codes.get(code, 0) + 1
if 200 <= code < 300:
self.latencies.append(time.monotonic() - started)
i += 1
time.sleep(self.pause)
self.workers = [threading.Thread(target=loop, args=(n,), daemon=True) for n in range(self.threads)]
for worker in self.workers:
worker.start()
return self
def close(self):
self.stop.set()
for worker in self.workers:
worker.join(20)
def b64hex(value):
return base64.b64decode(value).hex() if value else ''
def trace_spans(trace_id):
body = get(TEMPO + 'traces/' + trace_id)
spans = []
for batch in body.get('batches', []):
service = {a['key']: a['value'] for a in batch['resource']['attributes']}['service.name']['stringValue']
for scope in batch.get('scopeSpans', []):
for span in scope.get('spans', []):
span['service'] = service
span['ms'] = (int(span['endTimeUnixNano']) - int(span['startTimeUnixNano'])) / 1e6
span['attrs'] = {a['key']: list(a['value'].values())[0] for a in span.get('attributes', [])}
spans.append(span)
return spans
def find_traces(query, since, limit=5):
try:
return get(TEMPO + 'search', q=query, limit=limit, start=int(since), end=int(time.time()) + 60).get('traces', [])
except OSError:
return []
def breakdown(trace_id, title):
spans = trace_spans(trace_id)
server = next((s for s in spans if s.get('kind') == 'SPAN_KIND_SERVER'), None)
if not server:
log('trace %s: no server span' % trace_id)
return
names = {s['spanId']: s['name'] for s in spans}
def total(prefix, top_level=False):
return sum(s['ms'] for s in spans if s['name'].startswith(prefix)
and not (top_level and names.get(s.get('parentSpanId'), '').startswith('prisma:')))
db, s3 = total('prisma:', top_level=True), total('S3.')
log('%s: trace %s %s -> %s, %.0f ms' % (title, trace_id, server['name'], server['attrs'].get('http.status_code'), server['ms']))
log(' DB calls %.0f ms (pool wait %.0f ms, SQL %.0f ms); S3 %.0f ms; handler itself %.0f ms' % (
db, total('prisma:engine:connection'), total('prisma:engine:db_query'), s3, max(0, server['ms'] - db - s3)))
errors = sorted({s['name'] for s in spans if s.get('status', {}).get('code') == 'STATUS_CODE_ERROR'})
if errors:
log(' spans with error status:', ', '.join(errors))
def noise_png(width, height):
raw = b''.join(b'\0' + os.urandom(width * 3) for _ in range(height))
def chunk(kind, data):
return struct.pack('>I', len(data)) + kind + data + struct.pack('>I', zlib.crc32(kind + data))
header = struct.pack('>IIBBBBB', width, height, 8, 2, 0, 0, 0)
return b'\x89PNG\r\n\x1a\n' + chunk(b'IHDR', header) + chunk(b'IDAT', zlib.compress(raw, 1)) + chunk(b'IEND', b'')
def docker(*args):
log('$ docker', ' '.join(args))
subprocess.run(['docker', *args], check=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
def container(service):
return subprocess.run(['docker', 'compose', 'ps', '-q', service], check=True, capture_output=True, text=True).stdout.split()
def scenario_worker_stop():
"""Incident 1: worker stopped -> jobs age, MessengerJobQueueLag fires -> worker started -> backlog drains, resolved."""
js.wait_ready()
auth, chat = js.session()
columns = [
('due', 'max(jobs_queue_depth{state="due"})', ''),
('processing', 'max(jobs_queue_depth{state="processing"})', ''),
('oldest_age_s', 'messenger:jobs_queue_oldest_age:max', ''),
('worker_busy', 'sum(worker_jobs_busy)', ''),
('worker_up', 'count(timestamp(worker_jobs_capacity) > time() - 30)', ''),
('ready_per_s', 'sum(rate(jobs_processed_total{outcome="success"}[1m]))', ''),
]
alerts = ['MessengerJobQueueLag']
js.compose('stop', 'worker')
stopped = time.time()
uploads = []
def upload_more():
if len(uploads) < 12 and time.time() - stopped < 150:
uploads.append(js.upload(auth, chat))
return alert_state('MessengerJobQueueLag') == 'firing'
fired = watch(columns, alerts, upload_more, timeout=300)
log('alert MessengerJobQueueLag firing after %.0f s of worker downtime: %s' % (time.time() - stopped, fired))
js.start_only('worker')
started = time.time()
resolved = watch(columns, alerts, lambda: alert_state('MessengerJobQueueLag') == 'inactive', timeout=240)
log('backlog drained and alert resolved %.0f s after worker start: %s' % (time.time() - started, resolved))
ready = sum(1 for a in uploads if js.metadata(auth, a)[1]['status'] == 'ready')
log('uploads during downtime: %d, ready now: %d' % (len(uploads), ready))
time.sleep(15)
log('time to ready, histogram over 5m: p50=%s s, p99=%s s' % tuple(fmt(prom(
'histogram_quantile(%s, sum by (le) (increase(attachments_time_to_ready_seconds_bucket[5m])))' % q)) for q in (0.5, 0.99)))
log('time to ready from the database (finished_at - created_at), s:', js.psql(
"SELECT round(min(extract(epoch FROM j.finished_at - a.created_at))::numeric, 1) || ' .. ' || "
"round(max(extract(epoch FROM j.finished_at - a.created_at))::numeric, 1) FROM attachments a "
"JOIN attachment_jobs j ON j.attachment_id = a.id WHERE a.id IN (%s)" % ','.join("'%s'" % a for a in uploads)))
if not (fired and resolved and ready == len(uploads)):
sys.exit('FAIL')
log('PASS')
def scenario_db():
"""Incident 2: slow DB (CPU-throttled postgres) vs slow handler (large upload); then DB unavailable (paused)."""
js.wait_ready()
auth, chat = js.session()
columns = [
('p95_1m', 'histogram_quantile(0.95, sum by (le) (rate(http_server_request_duration_seconds_bucket{job="messenger-api",http_route!~"/health.*"}[1m])))', 'ms'),
('5xx_1m', '(sum(rate(http_server_requests_total{job="messenger-api",http_route!~"/health.*",http_response_status_class="5xx"}[1m])) or vector(0)) / sum(rate(http_server_requests_total{job="messenger-api",http_route!~"/health.*"}[1m]))', '%'),
('pool_wait', 'sum(db_client_connection_pending_requests{job="messenger-api"})', ''),
('pool_used', 'sum(db_client_connection_count{job="messenger-api",db_client_connection_state="used"})', ''),
('core_5xx_1m', 'sum(increase(http_server_requests_total{http_route="/health/core",http_response_status_class="5xx"}[1m]))', ''),
('api_cpu', 'sum(rate(process_cpu_time_seconds_total{job="messenger-api"}[1m]))', ''),
]
alerts = ['MessengerSlowApi', 'MessengerHighErrorRate', 'MessengerApiNotReady']
traffic = Traffic(auth, chat).run()
try:
log('== baseline: 60 s of traffic')
watch(columns, alerts, lambda: False, timeout=60, every=20)
traffic.snapshot('baseline')
log('== slow handler (not DB): 1700x1700 noise PNG upload — decoding and S3 dominate')
big = noise_png(1700, 1700)
since = time.time()
code, raw, _ = js.call('POST', '/api/v1/chats/%s/attachments?filename=big.png' % chat,
headers={**auth, 'Content-Type': 'image/png'}, raw=big)
log('POST %.1f MiB attachment -> %s' % (len(big) / 2 ** 20, code))
time.sleep(15)
for t in find_traces('{resource.service.name="messenger-api" && kind=server && span.http.route="/api/v1/chats/{chat_id}/attachments"}', since, 1):
breakdown(t['traceID'], 'slow handler')
log('== slow DB: postgres limited to 0.05 CPU for 150 s')
postgres = container('postgres')
since = time.time()
docker('update', '--cpus', '0.05', *postgres)
watch(columns, alerts, lambda: alert_state('MessengerSlowApi') == 'firing', timeout=150, every=15)
traffic.snapshot('slow DB')
slow = find_traces('{resource.service.name="messenger-api" && kind=server && duration > 500ms}', since, 3)
for t in slow[:2]:
breakdown(t['traceID'], 'slow DB')
docker('update', '--cpus', '1.0', *postgres)
watch(columns, alerts, lambda: alert_state('MessengerSlowApi') != 'firing', timeout=240, every=15)
traffic.snapshot('after slow DB')
log('== DB unavailable: postgres paused for 120 s')
since = time.time()
docker('pause', *postgres)
not_ready = watch(columns, alerts, lambda: alert_state('MessengerApiNotReady') == 'firing', timeout=120, every=15)
traffic.snapshot('DB paused')
for t in find_traces('{resource.service.name="messenger-api" && kind=server && status=error}', since, 2):
breakdown(t['traceID'], 'DB unavailable')
logs = get(LOKI + 'query_range', query='{service_name="messenger-api"} | event="http.error"', limit=3, since='3m')
for stream in logs['data']['result'][:3]:
log(' loki:', stream['values'][0][1][:200])
docker('unpause', *postgres)
recovered = watch(columns, alerts, lambda: alert_state('MessengerApiNotReady') == 'inactive', timeout=240, every=15)
traffic.snapshot('after DB unpaused')
finally:
traffic.close()
subprocess.run(['docker', 'unpause', *container('postgres')], capture_output=True)
subprocess.run(['docker', 'update', '--cpus', '1.0', *container('postgres')], capture_output=True)
log('MessengerApiNotReady fired: %s, resolved after recovery: %s' % (not_ready, recovered))
log('PASS' if not_ready and recovered else 'FAIL')
def scenario_telemetry():
"""Incident 3: Loki and Tempo stopped, then the Collector stopped; API keeps serving, memory stays bounded."""
js.wait_ready()
auth, chat = js.session()
columns = [
('spans_failed_1m', 'sum(increase(otelcol_exporter_send_failed_spans[1m]))', ''),
('logs_failed_1m', 'sum(increase(otelcol_exporter_send_failed_log_records[1m]))', ''),
('queue', 'max(otelcol_exporter_queue_size)', ''),
('queue_cap', 'max(otelcol_exporter_queue_capacity)', ''),
('refused_1m', 'sum(increase(otelcol_receiver_refused_spans[1m])) + sum(increase(otelcol_receiver_refused_log_records[1m]))', ''),
('collector_rss', 'max(otelcol_process_memory_rss)', 'MiB'),
('api_rss', 'max(process_memory_usage_bytes{job="messenger-api"})', 'MiB'),
]
alerts = ['TelemetryExportFailing', 'MessengerHighErrorRate']
traffic = Traffic(auth, chat, threads=6, pause=0.05).run()
try:
watch(columns, alerts, lambda: False, timeout=30, every=15)
traffic.snapshot('baseline')
log('== Loki and Tempo stopped until TelemetryExportFailing fires (max 240 s)')
js.compose('stop', 'loki', 'tempo')
fired = watch(columns, alerts, lambda: alert_state('TelemetryExportFailing') == 'firing', timeout=240, every=15)
traffic.snapshot('Loki+Tempo down')
js.compose('start', 'loki', 'tempo')
resolved = watch(columns, alerts, lambda: alert_state('TelemetryExportFailing') == 'inactive', timeout=240, every=15)
traffic.snapshot('after start')
log('== Collector stopped for 300 s: SDK export fails fast (5 s timeout, bounded queues); RSS saw-tooth until GC')
js.compose('stop', 'otel-collector')
for _ in range(15):
time.sleep(20)
log('docker stats:', subprocess.run(
['docker', 'stats', '--no-stream', '--format', '{{.Name}} {{.MemUsage}}', *container('api'), *container('worker')],
capture_output=True, text=True).stdout.replace('\n', '; '))
traffic.snapshot('collector down')
js.compose('start', 'otel-collector')
time.sleep(30)
traffic.snapshot('collector back')
finally:
traffic.close()
subprocess.run(['docker', 'compose', 'start', 'loki', 'tempo', 'otel-collector'], capture_output=True)
restarts = subprocess.run(['docker', 'inspect', '-f', '{{.Name}} restarts={{.RestartCount}} oom={{.State.OOMKilled}}',
*container('api'), *container('worker')], capture_output=True, text=True).stdout.split('\n')
log('containers:', '; '.join(r for r in restarts if r))
log('TelemetryExportFailing fired: %s, resolved: %s' % (fired, resolved))
log('PASS' if fired and resolved else 'FAIL')
def scenario_retry_trace():
"""Level 4 trace: one upload whose first attempt fails on S3, then succeeds; all attempts in one trace."""
js.wait_ready()
auth, chat = js.session()
trace_id = os.urandom(16).hex()
js.compose('stop', 'worker')
code, raw, _ = js.call('POST', '/api/v1/chats/%s/attachments?filename=sample.png' % chat,
headers={**auth, 'Content-Type': 'image/png', 'traceparent': '00-%s-%s-01' % (trace_id, os.urandom(8).hex())},
raw=js.SAMPLE)
attachment = js.json_of(raw)['id']
log('upload ->', code, attachment, 'trace', trace_id)
js.compose('stop', 's3')
js.start_only('worker')
time.sleep(5)
js.start_only('s3')
js.wait_status(auth, attachment, ('ready', 'failed'))
js.job_state(attachment)
time.sleep(15)
spans = trace_spans(trace_id)
for span in sorted(spans, key=lambda s: int(s['startTimeUnixNano'])):
if span.get('kind') in ('SPAN_KIND_SERVER', 'SPAN_KIND_PRODUCER', 'SPAN_KIND_CONSUMER') or span['name'].startswith('S3.'):
log(' %-17s %-20s %-36s attempt=%-2s %8.1f ms %s' % (
span['service'], span.get('kind', '')[10:], span['name'], span['attrs'].get('messaging.job.attempt', ''),
span['ms'], span.get('status', {}).get('code', '')[12:]))
consumers = [s for s in spans if s.get('kind') == 'SPAN_KIND_CONSUMER']
log('consumer spans in one trace: %d (attempts %s)' % (
len(consumers), sorted({s['attrs'].get('messaging.job.attempt') for s in consumers})))
log('PASS' if len(consumers) >= 2 else 'FAIL')
SCENARIOS = {
'worker-stop': scenario_worker_stop,
'db': scenario_db,
'telemetry': scenario_telemetry,
'retry-trace': scenario_retry_trace,
}
if __name__ == '__main__':
if len(sys.argv) != 2 or sys.argv[1] not in SCENARIOS:
sys.exit('usage: incidents.py %s' % '|'.join(SCENARIOS))
SCENARIOS[sys.argv[1]]()
+394
View File
@@ -0,0 +1,394 @@
#!/usr/bin/env python3
"""Defense scenarios for background attachment processing; run inside the tools container."""
import atexit
import base64
import hashlib
import json
import os
import re
import subprocess
import sys
import time
import urllib.error
import urllib.request
import uuid
BASE = os.environ.get('BASE_URL', 'http://localhost:8080')
SAMPLE = open(os.path.join(os.path.dirname(__file__), '..', 'tests', 'fixtures', 'sample.png'), 'rb').read()
STARTED = time.monotonic()
ANSI = re.compile(chr(27) + re.escape('[') + '[0-9;]*m')
def log(*parts):
print('[%6.1fs]' % (time.monotonic() - STARTED), *parts, flush=True)
def call(method, path, body=None, headers=None, raw=None):
data = raw if raw is not None else (json.dumps(body).encode() if body is not None else None)
all_headers = dict(headers or {})
if body is not None:
all_headers['Content-Type'] = 'application/json'
request = urllib.request.Request(BASE + path, data=data, method=method, headers=all_headers)
try:
with urllib.request.urlopen(request, timeout=15) as response:
return response.status, response.read(), response.headers
except urllib.error.HTTPError as error:
return error.code, error.read(), error.headers
except OSError as error:
return 0, str(error).encode(), {}
def json_of(raw):
try:
return json.loads(raw or b'null')
except ValueError:
return None
def compose(*args):
log('$ docker compose', ' '.join(args))
subprocess.run(['docker', 'compose', *args], check=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
def start_only(service):
"""docker compose start also starts dependencies (worker -> s3-init -> s3); start just this container."""
log('$ docker start', service, '(without dependencies)')
ids = subprocess.run(['docker', 'compose', 'ps', '-aq', service], check=True, capture_output=True, text=True).stdout.split()
subprocess.run(['docker', 'start', *ids], check=True, stdout=subprocess.DEVNULL)
def psql(sql):
out = subprocess.run(
['docker', 'compose', 'exec', '-T', 'postgres', 'psql', '-U', 'messenger', '-d', 'messenger', '-tA', '-F', ' | ', '-c', sql],
check=True, capture_output=True, text=True,
)
return out.stdout.strip()
def queue_depth():
time.sleep(6)
out = subprocess.run(
['docker', 'compose', 'exec', '-T', 'rabbitmq', 'rabbitmqctl', '-q', 'list_queues', 'name', 'type', 'durable', 'messages_ready', 'messages_unacknowledged'],
capture_output=True, text=True,
)
return out.stdout.strip() or out.stderr.strip()
def wait_ready(timeout=90):
deadline = time.monotonic() + timeout
while time.monotonic() < deadline:
if call('GET', '/health/ready')[0] == 200:
return
time.sleep(1)
sys.exit('API did not become ready')
def session():
username = 'job_' + uuid.uuid4().hex[:8]
password = 'Job-scenario-' + uuid.uuid4().hex
code, raw, _ = call('POST', '/api/v1/users', {'username': username, 'display_name': 'Job scenario', 'password': password})
if code != 201:
sys.exit('register failed: %s' % code)
user = json_of(raw)
credential = base64.b64encode((username + ':' + password).encode()).decode()
code, raw, _ = call('POST', '/api/v1/auth/sessions', headers={'Authorization': 'Basic ' + credential})
if code != 201:
sys.exit('login failed: %s' % code)
auth = {'Authorization': 'Bearer ' + json_of(raw)['token']}
code, raw, _ = call('POST', '/api/v1/chats', {'title': 'Job scenario', 'member_ids': [user['id']]}, auth)
if code != 201:
sys.exit('chat failed: %s' % code)
chat = json_of(raw)['id']
log('user', user['id'], 'chat', chat)
return auth, chat
def upload(auth, chat, body=SAMPLE):
code, raw, headers = call(
'POST', '/api/v1/chats/%s/attachments?filename=sample.png' % chat,
headers={**auth, 'Content-Type': 'image/png'}, raw=body,
)
attachment = json_of(raw)
log('POST attachment ->', code, 'status=%s' % attachment.get('status'), 'id=%s' % attachment.get('id'),
'instance=%s' % headers.get('X-Instance-Id'))
if code != 202:
sys.exit('upload failed')
return attachment['id']
def message(auth, chat, text, attachments=()):
code, raw, _ = call(
'POST', '/api/v1/chats/%s/messages' % chat, {'text': text, 'attachment_ids': list(attachments)},
{**auth, 'Idempotency-Key': uuid.uuid4().hex},
)
log('POST message', 'with %d attachment(s)' % len(attachments), '->', code, 'id=%s' % (json_of(raw) or {}).get('id'))
return code
def metadata(auth, attachment):
code, raw, _ = call('GET', '/api/v1/attachments/' + attachment, headers=auth)
return code, json_of(raw)
def content_status(auth, attachment, variant='original'):
return call('GET', '/api/v1/attachments/%s/content?variant=%s' % (attachment, variant), headers=auth)[0]
def wait_status(auth, attachment, wanted, timeout=150):
deadline = time.monotonic() + timeout
last = None
while time.monotonic() < deadline:
code, meta = metadata(auth, attachment)
if code == 200 and meta['status'] != last:
last = meta['status']
log('GET attachment -> status=%s error_code=%s' % (last, meta['error_code']))
if code == 200 and meta['status'] in wanted:
return meta
time.sleep(1)
sys.exit('attachment did not reach %s' % '/'.join(wanted))
def check_ready(auth, attachment, meta):
code, raw, _ = call('GET', '/api/v1/attachments/%s/content' % attachment, headers=auth)
log('original ->', code, 'sha256 matches upload:', code == 200 and hashlib.sha256(raw).hexdigest() == hashlib.sha256(SAMPLE).hexdigest())
for variant in meta['variants']:
code, raw, headers = call('GET', '/api/v1/attachments/%s/content?variant=%s' % (attachment, variant['name']), headers=auth)
log('%-9s -> %s %s %dx%d sha256 matches metadata: %s' % (
variant['name'], code, headers.get('Content-Type'), variant['width'], variant['height'],
hashlib.sha256(raw).hexdigest() == variant['sha256']))
def job_state(attachment):
log('job:', psql("SELECT id, status, attempt, max_attempts, error_code FROM attachment_jobs WHERE attachment_id = '%s'" % attachment))
log('variants rows:', psql("SELECT count(*) FROM attachment_variants WHERE attachment_id = '%s'" % attachment))
log('events:', psql("SELECT string_agg(e.event, ' -> ' ORDER BY e.id) FROM job_events e JOIN attachment_jobs j ON j.id = e.job_id WHERE j.attachment_id = '%s'" % attachment))
def scenario_restart():
"""Level 3: worker down -> upload 202 -> restart API and broker -> worker up -> ready."""
wait_ready()
auth, chat = session()
compose('stop', 'worker')
attachment = upload(auth, chat)
log('content while queued ->', content_status(auth, attachment))
message(auth, chat, '', [attachment])
message(auth, chat, 'other messages keep working')
log('queues:', queue_depth())
compose('restart', 'api', 'rabbitmq')
wait_ready()
log('queues after broker restart:', queue_depth())
log('status after restarts:', metadata(auth, attachment)[1]['status'])
compose('start', 'worker')
meta = wait_status(auth, attachment, ['ready', 'failed'])
check_ready(auth, attachment, meta)
job_state(attachment)
def remove_container(name):
subprocess.run(['docker', 'rm', '-f', name], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
def plain(line):
return ANSI.sub('', line).split('] ')[-1].strip()
def faulty(service, env):
"""Run a one-off copy of the service with fault injection; no restart policy, so it dies once."""
name = 'lab6-fault-%s-%s' % (service, uuid.uuid4().hex[:8])
atexit.register(remove_container, name)
args = ['docker', 'compose', 'run', '-d', '--no-deps', '--name', name]
for key, value in env.items():
args += ['-e', '%s=%s' % (key, value)]
log('$ docker compose run --no-deps', ' '.join('-e %s=%s' % item for item in env.items()), service)
subprocess.run(args + [service], check=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
deadline = time.monotonic() + 30
while time.monotonic() < deadline:
if 'connected to broker' in subprocess.run(['docker', 'logs', name], capture_output=True, text=True).stdout:
return name
time.sleep(0.5)
sys.exit('%s with fault did not start' % service)
def wait_exit(name, timeout=90):
try:
code = subprocess.run(['docker', 'wait', name], capture_output=True, text=True, timeout=timeout).stdout.strip()
except subprocess.TimeoutExpired:
code = 'timeout'
for line in subprocess.run(['docker', 'logs', name], capture_output=True, text=True).stdout.splitlines():
if 'fault injection' in line or 'published' in line or 'job ' in line:
log(' one-off:', plain(line))
remove_container(name)
log('one-off process exit code', code, '(137 = killed without ack)')
def service_logs(service, since, needle):
out = subprocess.run(['docker', 'compose', 'logs', '--no-log-prefix', '--no-color', '--since', since, service],
capture_output=True, text=True).stdout
for line in out.splitlines():
if needle in line:
log(' %s:' % service, plain(line))
def utc_now():
return time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime())
def staging_key(attachment):
return psql("SELECT coalesce(staging_key, 'NULL') FROM attachments WHERE id = '%s'" % attachment)
def job_id(attachment):
return psql("SELECT id FROM attachment_jobs WHERE attachment_id = '%s'" % attachment)
def scenario_crash_after_store():
"""Level 4: worker killed after writing S3 objects, before COMMIT and ack; lease expires; same result."""
wait_ready()
since = utc_now()
auth, chat = session()
compose('stop', 'worker')
proc = faulty('worker', {'WORKER_FAULT': 'exit-after-store', 'JOB_LEASE_SECONDS': '15'})
attachment = upload(auth, chat)
wait_exit(proc)
log('status after crash:', metadata(auth, attachment)[1]['status'])
log('lease:', psql("SELECT status, attempt, lease_until > now() AS lease_active FROM attachment_jobs WHERE attachment_id = '%s'" % attachment))
log('queues (unacked message returned):', queue_depth())
compose('start', 'worker')
meta = wait_status(auth, attachment, ['ready', 'failed'])
check_ready(auth, attachment, meta)
job_state(attachment)
service_logs('worker', since, job_id(attachment))
service_logs('relay', since, 'lease expired')
def scenario_crash_after_commit():
"""Level 4: worker killed after COMMIT ready, before ack; redelivery is a no-op and cleans staging."""
wait_ready()
since = utc_now()
auth, chat = session()
compose('stop', 'worker')
proc = faulty('worker', {'WORKER_FAULT': 'exit-after-commit'})
attachment = upload(auth, chat)
wait_exit(proc)
log('status after crash:', metadata(auth, attachment)[1]['status'], 'staging_key:', staging_key(attachment))
log('queues (unacked message returned):', queue_depth())
compose('start', 'worker')
deadline = time.monotonic() + 30
while staging_key(attachment) != 'NULL' and time.monotonic() < deadline:
time.sleep(1)
log('staging_key after redelivery:', staging_key(attachment))
check_ready(auth, attachment, metadata(auth, attachment)[1])
job_state(attachment)
service_logs('worker', since, job_id(attachment))
def scenario_outbox_gap():
"""Level 4: crash between COMMIT and publish; the outbox row is published after recovery."""
wait_ready()
auth, chat = session()
compose('stop', 'relay')
attachment = upload(auth, chat)
outbox = "SELECT o.id, coalesce(o.published_at::text, 'NOT PUBLISHED'), o.publish_attempts FROM outbox o JOIN attachment_jobs j ON j.id = o.job_id WHERE j.attachment_id = '%s'" % attachment
log('outbox after COMMIT:', psql(outbox))
log('queues:', queue_depth())
compose('kill', 'api')
log('API killed (SIGKILL) after COMMIT, nothing was published')
compose('start', 'api')
wait_ready()
log('status after API restart:', metadata(auth, attachment)[1]['status'])
compose('start', 'relay')
meta = wait_status(auth, attachment, ['ready', 'failed'])
log('outbox after relay recovery:', psql(outbox))
check_ready(auth, attachment, meta)
job_state(attachment)
def scenario_relay_crash():
"""Level 4: relay killed after broker confirm, before marking outbox; duplicate delivery is a no-op."""
wait_ready()
since = utc_now()
auth, chat = session()
compose('stop', 'relay')
proc = faulty('relay', {'RELAY_FAULT': 'exit-after-confirm'})
attachment = upload(auth, chat)
wait_exit(proc)
outbox = "SELECT o.id, coalesce(o.published_at::text, 'NOT MARKED'), o.publish_attempts FROM outbox o JOIN attachment_jobs j ON j.id = o.job_id WHERE j.attachment_id = '%s'" % attachment
log('outbox after relay crash:', psql(outbox))
compose('start', 'relay')
meta = wait_status(auth, attachment, ['ready', 'failed'])
time.sleep(3)
log('outbox after relay restart:', psql(outbox))
check_ready(auth, attachment, meta)
job_state(attachment)
service_logs('worker', since, job_id(attachment))
def replay(*args):
out = subprocess.run(['docker', 'compose', 'exec', '-T', '-e', 'OPERATOR=defense-demo', 'worker', 'node', 'dist/replay.js', *args],
capture_output=True, text=True)
for line in (out.stdout.strip() or out.stderr.strip()).splitlines():
log(' replay:', line)
def scenario_retry_replay():
"""Level 4: S3 down -> retries with backoff and jitter -> failed retry_exhausted -> explicit replay -> ready."""
wait_ready()
auth, chat = session()
compose('stop', 'worker')
attachment = upload(auth, chat)
compose('stop', 's3')
start_only('worker')
meta = wait_status(auth, attachment, ['failed'], timeout=120)
log('failed attachment in a new message ->', message(auth, chat, '', [attachment]))
log('retries:')
for line in psql("SELECT e.attempt, e.event, coalesce(e.detail, ''), to_char(e.created_at, 'HH24:MI:SS.MS') FROM job_events e JOIN attachment_jobs j ON j.id = e.job_id WHERE j.attachment_id = '%s' ORDER BY e.id" % attachment).splitlines():
log(' ', line)
log('$ replay.js --list')
replay('--list')
compose('start', 's3')
wait_ready()
log('$ replay.js', attachment)
replay(attachment)
meta = wait_status(auth, attachment, ['ready', 'failed'])
check_ready(auth, attachment, meta)
log('$ replay.js', attachment, '(already ready)')
replay(attachment)
job_state(attachment)
def scenario_broker():
"""Level 4: prefetch/concurrency, limits and dead letter queue for a malformed message."""
wait_ready()
since = utc_now()
log('consumers:', subprocess.run(
['docker', 'compose', 'exec', '-T', 'rabbitmq', 'rabbitmqctl', '-q', 'list_consumers', 'queue_name', 'prefetch_count', 'ack_required'],
capture_output=True, text=True).stdout.strip())
log('worker limits:', subprocess.run(
['sh', '-c', "docker inspect -f 'cpus={{.HostConfig.NanoCpus}} memory={{.HostConfig.Memory}}' $(docker compose ps -q worker)"],
capture_output=True, text=True).stdout.strip())
publish = (
"const amqp=require('amqplib');const fs=require('fs');(async()=>{"
"const c=await amqp.connect(fs.readFileSync(process.env.BROKER_URL_FILE,'utf8').trim());"
"const ch=await c.createConfirmChannel();"
"ch.sendToQueue('attachment.jobs',Buffer.from(JSON.stringify({job_id:'not-a-uuid',url:'http://example.invalid/x'})),{persistent:true});"
"await ch.waitForConfirms();await c.close();console.log('malformed message published');})()"
)
log(subprocess.run(['docker', 'compose', 'exec', '-T', 'worker', 'node', '-e', publish], capture_output=True, text=True).stdout.strip())
log('queues:', queue_depth())
service_logs('worker', since, 'malformed')
SCENARIOS = {
'restart': scenario_restart,
'crash-after-store': scenario_crash_after_store,
'crash-after-commit': scenario_crash_after_commit,
'outbox-gap': scenario_outbox_gap,
'relay-crash': scenario_relay_crash,
'retry-replay': scenario_retry_replay,
'broker': scenario_broker,
}
if __name__ == '__main__':
if len(sys.argv) != 2 or sys.argv[1] not in SCENARIOS:
sys.exit('usage: job-scenario.py %s' % '|'.join(SCENARIOS))
SCENARIOS[sys.argv[1]]()
+52
View File
@@ -0,0 +1,52 @@
#!/usr/bin/env python3
"""Compares client-side percentiles of tests/load.py with histogram_quantile over the same window in Prometheus.
Run inside the tools container: docker compose run --rm tools python3 scripts/latency-compare.py [--duration 30]
"""
import json
import os
import subprocess
import sys
import tempfile
import time
import urllib.parse
import urllib.request
GRAFANA = os.environ.get('GRAFANA_URL', 'http://grafana:3000')
BASE = os.environ.get('BASE_URL', 'http://lb:8080')
SERIES = 'http_server_request_duration_seconds_bucket{job="messenger-api",http_request_method="POST",http_route="/api/v1/chats/{chat_id}/messages"}'
def prom(expr, at):
params = urllib.parse.urlencode({'query': expr, 'time': at})
with urllib.request.urlopen(GRAFANA + '/api/datasources/proxy/uid/prometheus/api/v1/query?' + params, timeout=15) as r:
result = json.loads(r.read())['data']['result']
return float(result[0]['value'][1]) if result else None
def main():
duration = sys.argv[sys.argv.index('--duration') + 1] if '--duration' in sys.argv else '30'
output = os.path.join(tempfile.mkdtemp(), 'load.json')
started = time.time()
subprocess.run([sys.executable, 'tests/load.py', '--base-url', BASE, '--duration', duration, '--output', output],
check=True, stdout=subprocess.DEVNULL)
report = json.load(open(output))
print('waiting 25 s for the metric export interval')
time.sleep(25)
at = time.time()
window = int(at - started) + 5
print('client (tests/load.py): %d requests, %.1f rps, errors %d' % (
report['successful_requests'], report['successful_rps'], report['failed_requests']))
count = prom('sum(increase(%s[%ds]))' % (SERIES.replace('_bucket{', '_count{'), window), at)
print('prometheus: %.0f requests in POST /api/v1/chats/{chat_id}/messages over %d s' % (count or 0, window))
print('%-5s %12s %16s' % ('', 'client, ms', 'prometheus, ms'))
for p in ('p50', 'p95', 'p99'):
q = int(p[1:]) / 100
server = prom('histogram_quantile(%s, sum by (le) (increase(%s[%ds])))' % (q, SERIES, window), at)
print('%-5s %12.2f %16.2f' % (p, report['successful_latency_ms'][p], (server or 0) * 1000))
print('server-side histogram excludes network/client overhead; values are interpolated inside buckets '
'(…, 5, 10, 25, 50, 100 ms, …).')
if __name__ == '__main__':
main()
+88
View File
@@ -0,0 +1,88 @@
#!/usr/bin/env python3
import base64
import json
import os
import sys
import time
import urllib.error
import urllib.request
import uuid
BASE = os.environ.get('BASE_URL', 'http://localhost:8080')
def call(method, path, body=None, headers=None, with_headers=False):
data = json.dumps(body).encode() if body is not None else None
content_type = {'Content-Type': 'application/json'} if data else {}
request = urllib.request.Request(BASE + path, data=data, method=method, headers={**content_type, **(headers or {})})
try:
with urllib.request.urlopen(request, timeout=15) as response:
result = response.status, json.loads(response.read() or b'null'), response.headers
except urllib.error.HTTPError as error:
result = error.code, json.loads(error.read() or b'null'), error.headers
except OSError as error:
result = 0, str(error), {}
return result if with_headers else result[:2]
def login():
username = 'probe_' + uuid.uuid4().hex[:8]
password = 'Probe-check-' + uuid.uuid4().hex
print('register ->', call('POST', '/api/v1/users', {'username': username, 'display_name': 'Probe', 'password': password})[0])
credential = base64.b64encode((username + ':' + password).encode()).decode()
code, session = call('POST', '/api/v1/auth/sessions', headers={'Authorization': 'Basic ' + credential})
if code != 201:
sys.exit('login failed: %s %s' % (code, session))
print('login ->', code, 'expires_at:', session['expires_at'])
return {'Authorization': 'Bearer ' + session['token']}
def instances(count):
auth = login()
for _ in range(count):
code, _, headers = call('GET', '/api/v1/users/me', headers=auth, with_headers=True)
print('GET /api/v1/users/me ->', code, 'X-Instance-Id:', headers.get('X-Instance-Id'))
def logout():
auth = login()
print('GET /api/v1/users/me ->', call('GET', '/api/v1/users/me', headers=auth)[0])
code, body = call('DELETE', '/api/v1/auth/sessions/current', headers=auth)
print('DELETE /api/v1/auth/sessions/current ->', code, json.dumps(body, ensure_ascii=False) if body else '')
print('GET /api/v1/users/me ->', call('GET', '/api/v1/users/me', headers=auth)[0])
def ready(timeout=90):
deadline = time.monotonic() + timeout
while time.monotonic() < deadline:
if call('GET', '/health/ready')[0] == 200:
print('ready')
return
time.sleep(0.5)
sys.exit('not ready within %ds' % timeout)
def status(method, path, authorization=None):
code, body = call(method, path, headers={'Authorization': authorization} if authorization else None)
print(method, path, '(%s)' % (authorization or 'anonymous'), '->', code, json.dumps(body, ensure_ascii=False))
def expiry(ttl):
auth = login()
print('GET /api/v1/users/me (fresh) ->', call('GET', '/api/v1/users/me', headers=auth)[0])
time.sleep(ttl + 1)
print('GET /api/v1/users/me (after %ds) ->' % (ttl + 1), call('GET', '/api/v1/users/me', headers=auth)[0])
if __name__ == '__main__':
command = sys.argv[1]
if command == 'ready':
ready()
elif command == 'status':
status(*sys.argv[2:5])
elif command == 'expiry':
expiry(int(sys.argv[2]))
elif command == 'instances':
instances(int(sys.argv[2]))
elif command == 'logout':
logout()
+9
View File
@@ -0,0 +1,9 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
started() {
docker inspect -f '{{.Name}} {{.State.Status}} started={{.State.StartedAt}}' $(docker compose ps -aq postgres redis s3 rabbitmq api lb)
}
started
docker compose restart postgres redis s3 rabbitmq api lb
started
+11
View File
@@ -0,0 +1,11 @@
#!/bin/sh
set -eu
: "${TARGET_INSTANCE_ID:?test provides target API instance alias}"
project="${COMPOSE_PROJECT_NAME:-messenger-lab6}"
labels=$(docker inspect -f '{{index .Config.Labels "com.docker.compose.project"}}/{{index .Config.Labels "com.docker.compose.service"}}' "$TARGET_INSTANCE_ID")
if [ "$labels" != "$project/api" ]; then
echo "refusing to stop $TARGET_INSTANCE_ID: not an api replica of $project ($labels)" >&2
exit 1
fi
docker stop -t 8 "$TARGET_INSTANCE_ID" >/dev/null
docker inspect -f '{{.Name}} {{.State.Status}} finished={{.State.FinishedAt}}' "$TARGET_INSTANCE_ID"
+102
View File
@@ -0,0 +1,102 @@
#!/usr/bin/env python3
"""Checks that passwords, tokens, message text and filenames do not reach Loki/Tempo and that metric labels hold no IDs.
Run inside the tools container: docker compose run --rm tools python3 scripts/telemetry-privacy.py
"""
import base64
import json
import os
import re
import sys
import time
import urllib.parse
import urllib.request
import uuid
BASE = os.environ.get('BASE_URL', 'http://lb:8080')
GRAFANA = os.environ.get('GRAFANA_URL', 'http://grafana:3000')
SAMPLE = open(os.path.join(os.path.dirname(__file__), '..', 'tests', 'fixtures', 'sample.png'), 'rb').read()
UUID = re.compile(r'[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}')
def call(method, path, body=None, headers=None, raw=None):
data = raw if raw is not None else (json.dumps(body).encode() if body is not None else None)
headers = dict(headers or {})
if body is not None:
headers['Content-Type'] = 'application/json'
with urllib.request.urlopen(urllib.request.Request(BASE + path, data=data, method=method, headers=headers), timeout=15) as r:
return json.loads(r.read() or b'null')
def grafana(path, **params):
with urllib.request.urlopen(GRAFANA + path + '?' + urllib.parse.urlencode(params), timeout=30) as r:
return r.read().decode()
def main():
marker = uuid.uuid4().hex[:12]
secrets = {
'password': 'Privacy-password-' + marker,
'message text': 'privacy message body ' + marker,
'filename': 'privacy-file-%s.png' % marker,
}
trace_ids = []
def traced():
trace_id = os.urandom(16).hex()
trace_ids.append(trace_id)
return {'traceparent': '00-%s-%s-01' % (trace_id, os.urandom(8).hex())}
username = 'privacy_' + marker
user = call('POST', '/api/v1/users', {'username': username, 'display_name': 'Privacy', 'password': secrets['password']}, traced())
basic = base64.b64encode(('%s:%s' % (username, secrets['password'])).encode()).decode()
token = call('POST', '/api/v1/auth/sessions', headers={'Authorization': 'Basic ' + basic, **traced()})['token']
secrets['session token'] = token
secrets['basic credentials'] = basic
auth = {'Authorization': 'Bearer ' + token}
chat = call('POST', '/api/v1/chats', {'title': 'Privacy', 'member_ids': [user['id']]}, {**auth, **traced()})['id']
call('POST', '/api/v1/chats/%s/messages' % chat, {'text': secrets['message text']},
{**auth, 'Idempotency-Key': marker, **traced()})
call('POST', '/api/v1/chats/%s/attachments?filename=%s' % (chat, secrets['filename']), headers={**auth, 'Content-Type': 'image/png', **traced()},
raw=SAMPLE)
call('GET', '/api/v1/chats/%s/messages?limit=5' % chat, headers={**auth, **traced()})
print('sent %d traced requests with known password, token, message text and filename; waiting 25 s for export' % len(trace_ids))
time.sleep(25)
traces = ''
found_traces = 0
for trace_id in trace_ids:
try:
traces += grafana('/api/datasources/proxy/uid/tempo/api/traces/' + trace_id)
found_traces += 1
except OSError:
pass
def loki_lines(query):
body = grafana('/api/datasources/proxy/uid/loki/loki/api/v1/query_range', query=query, limit=100, since='5m')
return sum(len(s['values']) for s in json.loads(body)['data']['result'])
own = loki_lines('{service_name=~"messenger-.+"} |~ "%s"' % '|'.join(trace_ids))
print('Tempo: %d/%d traces fetched (%d bytes); Loki: %d log lines carry these trace_ids (control)' % (
found_traces, len(trace_ids), len(traces), own))
leaks = 0
for name, value in list(secrets.items()) + [('Authorization hdr', 'authorization')]:
in_traces = value.lower() in traces.lower()
in_logs = loki_lines('{service_name=~"messenger-.+"} |~ "(?i)%s"' % re.escape(value).replace('\\', '\\\\'))
leaks += in_traces + in_logs
print(' %-18s in traces: %-5s log lines in Loki (5 min): %d' % (name, in_traces, in_logs))
series = json.loads(grafana('/api/datasources/proxy/uid/prometheus/api/v1/series', **{'match[]': '{job=~"messenger-.+"}'}))['data']
names = {s['__name__'] for s in series}
with_ids = [s for s in series if any(UUID.search(v) for k, v in s.items() if k != 'instance')]
labels = sorted({k for s in series for k in s} - {'__name__'})
print('Prometheus: %d series, %d metric names from messenger services; label keys: %s' % (len(series), len(names), ', '.join(labels)))
print(' series with a UUID in any label value: %d' % len(with_ids))
routes = sorted({s.get('http_route') for s in series if s.get('http_route')})
print(' http_route values: %s' % ', '.join(routes))
ok = found_traces == len(trace_ids) and own > 0 and leaks == 0 and not with_ids
print('%s: no secrets/content in logs and traces, no IDs in metric labels' % ('PASS' if ok else 'FAIL'))
sys.exit(0 if ok else 1)
if __name__ == '__main__':
main()
+134
View File
@@ -0,0 +1,134 @@
#!/usr/bin/env python3
"""Control request for lab6: upload with a client traceparent, then find its trace in Tempo and logs in Loki.
Run inside the tools container: docker compose run --rm tools python3 scripts/trace-check.py
"""
import base64
import json
import os
import sys
import time
import urllib.error
import urllib.parse
import urllib.request
import uuid
BASE = os.environ.get('BASE_URL', 'http://localhost:8080')
GRAFANA = os.environ.get('GRAFANA_URL', 'http://grafana:3000')
SAMPLE = open(os.path.join(os.path.dirname(__file__), '..', 'tests', 'fixtures', 'sample.png'), 'rb').read()
def call(method, url, body=None, headers=None, raw=None):
data = raw if raw is not None else (json.dumps(body).encode() if body is not None else None)
all_headers = dict(headers or {})
if body is not None:
all_headers['Content-Type'] = 'application/json'
request = urllib.request.Request(url, data=data, method=method, headers=all_headers)
try:
with urllib.request.urlopen(request, timeout=15) as response:
return response.status, json.loads(response.read() or b'null')
except urllib.error.HTTPError as error:
return error.code, None
def session():
username = 'trace_' + uuid.uuid4().hex[:8]
password = 'Trace-check-' + uuid.uuid4().hex
_, user = call('POST', BASE + '/api/v1/users', {'username': username, 'display_name': 'Trace check', 'password': password})
credential = base64.b64encode((username + ':' + password).encode()).decode()
_, login = call('POST', BASE + '/api/v1/auth/sessions', headers={'Authorization': 'Basic ' + credential})
auth = {'Authorization': 'Bearer ' + login['token']}
_, chat = call('POST', BASE + '/api/v1/chats', {'title': 'Trace check', 'member_ids': [user['id']]}, auth)
return auth, chat['id']
def wait_trace(trace_id, services, timeout=60):
deadline = time.monotonic() + timeout
spans = []
while time.monotonic() < deadline:
code, body = call('GET', GRAFANA + '/api/datasources/proxy/uid/tempo/api/traces/' + trace_id)
if code == 200 and body:
spans = []
for batch in body.get('batches', []):
attrs = {a['key']: a['value'] for a in batch['resource']['attributes']}
service = attrs['service.name']['stringValue']
for scope in batch.get('scopeSpans', []):
for span in scope.get('spans', []):
spans.append((service, span))
if services <= {service for service, _ in spans}:
return spans
time.sleep(3)
return spans
def b64hex(value):
return base64.b64decode(value).hex() if value else ''
def main():
trace_id = os.urandom(16).hex()
client_span = os.urandom(8).hex()
traceparent = '00-%s-%s-01' % (trace_id, client_span)
auth, chat = session()
started = time.monotonic()
code, attachment = call(
'POST', BASE + '/api/v1/chats/%s/attachments?filename=sample.png' % chat,
headers={**auth, 'Content-Type': 'image/png', 'traceparent': traceparent}, raw=SAMPLE,
)
print('upload ->', code, attachment and attachment['status'], 'traceparent', traceparent)
status = None
while time.monotonic() - started < 60:
_, body = call('GET', BASE + '/api/v1/attachments/' + attachment['id'], headers=auth)
status = body['status']
if status in ('ready', 'failed'):
break
time.sleep(0.5)
print('attachment', attachment['id'], 'status', status, 'after %.2f s' % (time.monotonic() - started))
spans = wait_trace(trace_id, {'messenger-api', 'messenger-relay', 'messenger-worker'})
by_id = {b64hex(s['spanId']): (svc, s) for svc, s in spans}
print('\ntrace %s: %d spans (prisma:* internals except db_query omitted below)' % (trace_id, len(spans)))
for service, span in sorted(spans, key=lambda item: int(item[1]['startTimeUnixNano'])):
if span['name'].startswith('prisma:') and span['name'] not in ('prisma:engine:db_query', 'prisma:client:transaction'):
continue
parent = b64hex(span.get('parentSpanId'))
parent_name = 'client' if parent == client_span else by_id.get(parent, ('', {'name': '-'}))[1]['name']
duration = (int(span['endTimeUnixNano']) - int(span['startTimeUnixNano'])) / 1e6
print(' %-17s %-22s %-48s %8.2f ms parent=%s' % (
service, span.get('kind', ''), span['name'][:48], duration, parent_name))
query = '{service_name=~"messenger-.+"} | trace_id="%s"' % trace_id
params = urllib.parse.urlencode({'query': query, 'limit': 50, 'since': '15m'})
streams = []
deadline = time.monotonic() + 30
while time.monotonic() < deadline:
_, logs = call('GET', GRAFANA + '/api/datasources/proxy/uid/loki/loki/api/v1/query_range?' + params)
streams = (logs or {}).get('data', {}).get('result', [])
if {'messenger-api', 'messenger-worker'} <= {s['stream'].get('service_name') for s in streams}:
break
time.sleep(3)
print('\nLoki %s' % query)
lines = 0
for stream in streams:
labels = stream['stream']
for _, line in stream['values']:
lines += 1
print(' %-17s %-8s %-13s span_id=%s %s' % (
labels.get('service_name'), labels.get('severity_text', ''), labels.get('event', ''),
labels.get('span_id', ''), line[:60]))
kinds = {(service, span.get('kind')) for service, span in spans}
ok = (
status == 'ready'
and ('messenger-api', 'SPAN_KIND_SERVER') in kinds
and ('messenger-relay', 'SPAN_KIND_PRODUCER') in kinds
and ('messenger-worker', 'SPAN_KIND_CONSUMER') in kinds
and any(b64hex(s.get('parentSpanId')) == client_span for _, s in spans)
and {'messenger-api', 'messenger-worker'} <= {s['stream'].get('service_name') for s in streams}
)
print('\n%s: client trace continued through api -> relay -> worker, logs linked by trace_id' % ('PASS' if ok else 'FAIL'))
sys.exit(0 if ok else 1)
if __name__ == '__main__':
main()
+25
View File
@@ -0,0 +1,25 @@
import { Module } from '@nestjs/common';
import { AttachmentsModule } from './attachments/attachments.module';
import { AuthModule } from './auth/auth.module';
import { CacheModule } from './cache/cache.module';
import { ChatsModule } from './chats/chats.module';
import { DbModule } from './db/db.module';
import { HealthModule } from './health/health.module';
import { MessagesModule } from './messages/messages.module';
import { StorageModule } from './storage/storage.module';
import { UsersModule } from './users/users.module';
@Module({
imports: [
DbModule,
CacheModule,
StorageModule,
AuthModule,
HealthModule,
UsersModule,
ChatsModule,
MessagesModule,
AttachmentsModule,
],
})
export class AppModule {}
+79
View File
@@ -0,0 +1,79 @@
import {
Controller,
Get,
HttpCode,
Param,
Post,
Query,
Req,
Res,
UseGuards,
} from '@nestjs/common';
import { FastifyReply, FastifyRequest } from 'fastify';
import { Readable } from 'stream';
import { AuthGuard } from '../auth/auth.guard';
import { CurrentUser } from '../auth/current-user.decorator';
import { User } from '../common/domain';
import { AttachmentsService } from './attachments.service';
import {
AttachmentResponse,
toAttachmentResponse,
} from './dto/attachment.response';
@Controller()
@UseGuards(AuthGuard)
export class AttachmentsController {
constructor(private readonly attachments: AttachmentsService) {}
@Post('api/v1/chats/:chatId/attachments')
@HttpCode(202)
async upload(
@Param('chatId') chatId: string,
@Query('filename') filename: unknown,
@Req() request: FastifyRequest,
@CurrentUser() user: User,
): Promise<AttachmentResponse> {
const attachment = await this.attachments.upload({
chatId,
ownerId: user.id,
filename,
contentType: request.uploadContentType,
contentLength: request.headers['content-length'],
body: request.body as Readable | undefined,
});
return toAttachmentResponse(attachment);
}
@Get('api/v1/attachments/:attachmentId')
async metadata(
@Param('attachmentId') attachmentId: string,
@CurrentUser() user: User,
): Promise<AttachmentResponse> {
return toAttachmentResponse(
await this.attachments.getVisible(attachmentId, user.id),
);
}
@Get('api/v1/attachments/:attachmentId/content')
async content(
@Param('attachmentId') attachmentId: string,
@Query('variant') variant: unknown,
@CurrentUser() user: User,
@Res() reply: FastifyReply,
): Promise<void> {
const download = await this.attachments.openContent(
attachmentId,
user.id,
variant,
);
await reply
.header('content-type', download.contentType)
.header('content-length', download.sizeBytes)
.header('x-content-type-options', 'nosniff')
.header(
'content-disposition',
`inline; filename*=UTF-8''${encodeURIComponent(download.filename)}`,
)
.send(download.content);
}
}
+9
View File
@@ -0,0 +1,9 @@
import { Module } from '@nestjs/common';
import { AttachmentsController } from './attachments.controller';
import { AttachmentsService } from './attachments.service';
@Module({
controllers: [AttachmentsController],
providers: [AttachmentsService],
})
export class AttachmentsModule {}
+244
View File
@@ -0,0 +1,244 @@
import { Injectable, Logger, OnModuleInit } from '@nestjs/common';
import { Attachment, AttachmentVariant, Prisma } from '@prisma/client';
import { randomUUID } from 'crypto';
import { mkdtemp, readdir, rm, stat } from 'fs/promises';
import { tmpdir } from 'os';
import { join } from 'path';
import { Readable } from 'stream';
import { assertChatMember } from '../chats/chat-access';
import { codePointLength } from '../common/dto/code-point-length.validator';
import { ApiError } from '../common/filters/api-error';
import { validate as isUuid } from '../common/uuid';
import { isDatabaseUnavailable, PrismaService } from '../db/prisma.service';
import { PIPELINE_VERSION } from '../jobs/job-message';
import { enqueueJob } from '../jobs/job-store';
import { S3Service } from '../storage/s3.service';
import { assertValidImage, isSupportedContentType } from './image-validation';
import { receiveBody } from './upload-body';
const TEMP_PREFIX = 'messenger-upload-';
const STALE_TEMP_MS = 60 * 60 * 1000;
const MAX_FILENAME_CODE_POINTS = 255;
const FORBIDDEN_FILENAME = /[/\\\p{Cc}\p{Zl}\p{Zp}]/u;
const KNOWN_VARIANTS = new Set(['original', 'thumbnail', 'crop', 'optimized']);
export type AttachmentWithVariants = Attachment & { variants: AttachmentVariant[] };
export interface DownloadableContent {
filename: string;
contentType: string;
sizeBytes: number;
content: Readable;
}
export interface UploadInput {
chatId: string;
ownerId: string;
filename: unknown;
contentType: string | undefined;
contentLength: string | undefined;
body: Readable | undefined;
}
@Injectable()
export class AttachmentsService implements OnModuleInit {
private readonly logger = new Logger('AttachmentsService');
private readonly maxBytes = Number(process.env.MAX_UPLOAD_BYTES ?? 10485760);
private readonly maxPixels = Number(process.env.MAX_IMAGE_PIXELS ?? 40000000);
private readonly maxConcurrent = Number(process.env.MAX_CONCURRENT_UPLOADS ?? 16);
private readonly bodyTimeoutMs = Number(process.env.UPLOAD_BODY_TIMEOUT_MS ?? 60000);
private readonly tempRoot = process.env.UPLOAD_TMP_DIR ?? tmpdir();
private activeUploads = 0;
constructor(
private readonly prisma: PrismaService,
private readonly s3: S3Service,
) {}
async onModuleInit(): Promise<void> {
await this.removeStaleTempDirs();
}
async upload(input: UploadInput): Promise<AttachmentWithVariants> {
await assertChatMember(this.prisma, input.chatId, input.ownerId);
const filename = parseFilename(input.filename);
const contentType = parseContentType(input.contentType);
if (Number(input.contentLength) > this.maxBytes) {
throw new ApiError('payload_too_large', 'attachment exceeds size limit');
}
if (this.activeUploads >= this.maxConcurrent) {
throw new ApiError('unavailable', 'too many concurrent uploads');
}
this.activeUploads++;
const dir = await mkdtemp(join(this.tempRoot, TEMP_PREFIX));
try {
return await this.store(input, filename, contentType, join(dir, 'body'));
} finally {
this.activeUploads--;
await rm(dir, { recursive: true, force: true });
}
}
async getVisible(
attachmentId: string,
userId: string,
): Promise<AttachmentWithVariants> {
if (!isUuid(attachmentId)) {
throw new ApiError('invalid_request', 'attachmentId must be a valid UUID');
}
const row = await this.prisma.attachment.findFirst({
where: {
id: attachmentId,
chat: { members: { some: { userId } } },
},
include: { variants: { where: { pipelineVersion: PIPELINE_VERSION } } },
});
if (!row) {
throw new ApiError('not_found', 'attachment not found');
}
return row;
}
async openContent(
attachmentId: string,
userId: string,
variant: unknown,
): Promise<DownloadableContent> {
const attachment = await this.getVisible(attachmentId, userId);
const name = parseVariant(variant);
if (attachment.status !== 'ready') {
throw new ApiError('conflict', `attachment is ${attachment.status}`);
}
if (name === 'original') {
return {
filename: attachment.filename,
contentType: attachment.contentType,
sizeBytes: attachment.sizeBytes,
content: await this.s3.getStream(attachment.objectKey),
};
}
const stored = attachment.variants.find((v) => v.name === name);
if (!stored) {
throw new ApiError('not_found', 'attachment variant not found');
}
return {
filename: attachment.filename,
contentType: stored.contentType,
sizeBytes: stored.sizeBytes,
content: await this.s3.getStream(stored.objectKey),
};
}
private async store(
input: UploadInput,
filename: string,
contentType: string,
path: string,
): Promise<AttachmentWithVariants> {
if (!input.body) {
throw new ApiError('invalid_request', 'request body must not be empty');
}
const { size, sha256 } = await receiveBody(
input.body,
path,
this.maxBytes,
this.bodyTimeoutMs,
);
if (size === 0) {
throw new ApiError('invalid_request', 'request body must not be empty');
}
await assertValidImage(path, size, contentType, this.maxPixels);
const id = randomUUID();
const stagingKey = `staging/${id}`;
await this.s3.putFile({ key: stagingKey, path, size, contentType, sha256 });
const now = new Date();
try {
return await this.prisma.$transaction(async (tx) => {
const attachment = await tx.attachment.create({
data: {
id,
chatId: input.chatId,
ownerId: input.ownerId,
filename,
contentType,
sizeBytes: size,
sha256,
objectKey: `attachments/${id}/original`,
stagingKey,
status: 'queued',
createdAt: now,
updatedAt: now,
},
});
await enqueueJob(tx, id, now);
return { ...attachment, variants: [] };
});
} catch (error) {
if (isDefiniteRejection(error)) {
await this.s3.delete(stagingKey).catch(() => {
this.logger.warn(`compensation failed for ${stagingKey}`);
});
}
if (error instanceof Prisma.PrismaClientUnknownRequestError) {
this.logger.error(`metadata insert failed for ${stagingKey}`);
throw new ApiError('unavailable', 'database unavailable');
}
throw error;
}
}
private async removeStaleTempDirs(): Promise<void> {
const root = this.tempRoot;
const names = await readdir(root);
for (const name of names.filter((n) => n.startsWith(TEMP_PREFIX))) {
const path = join(root, name);
const info = await stat(path).catch(() => null);
if (info && Date.now() - info.mtimeMs > STALE_TEMP_MS) {
await rm(path, { recursive: true, force: true });
}
}
}
}
function parseVariant(raw: unknown): string {
if (raw === undefined) {
return 'original';
}
if (typeof raw === 'string' && KNOWN_VARIANTS.has(raw)) {
return raw;
}
throw new ApiError('invalid_request', 'unknown attachment variant');
}
function isDefiniteRejection(error: unknown): boolean {
return (
error instanceof Prisma.PrismaClientKnownRequestError &&
!isDatabaseUnavailable(error)
);
}
function parseFilename(raw: unknown): string {
if (
typeof raw !== 'string' ||
raw.trim() === '' ||
codePointLength(raw) > MAX_FILENAME_CODE_POINTS ||
FORBIDDEN_FILENAME.test(raw)
) {
throw new ApiError(
'invalid_request',
'filename must be 1-255 characters without path separators or control characters',
);
}
return raw;
}
function parseContentType(raw: string | undefined): string {
const contentType = raw?.split(';')[0].trim().toLowerCase() ?? '';
if (!isSupportedContentType(contentType)) {
throw new ApiError(
'unsupported_media_type',
'Content-Type must be image/png or image/jpeg',
);
}
return contentType;
}
@@ -0,0 +1,55 @@
import { Attachment, AttachmentVariant } from '@prisma/client';
const VARIANT_ORDER = ['thumbnail', 'crop', 'optimized'];
export interface VariantResponse {
name: string;
content_type: string;
size_bytes: number;
width: number;
height: number;
sha256: string;
}
export interface AttachmentResponse {
id: string;
chat_id: string;
owner_id: string;
filename: string;
content_type: string;
size_bytes: number;
sha256: string;
status: string;
variants: VariantResponse[];
error_code: string | null;
created_at: string;
updated_at: string;
}
export function toAttachmentResponse(
row: Attachment & { variants: AttachmentVariant[] },
): AttachmentResponse {
const variants = row.status === 'ready' ? [...row.variants] : [];
variants.sort((a, b) => VARIANT_ORDER.indexOf(a.name) - VARIANT_ORDER.indexOf(b.name));
return {
id: row.id,
chat_id: row.chatId,
owner_id: row.ownerId,
filename: row.filename,
content_type: row.contentType,
size_bytes: row.sizeBytes,
sha256: row.sha256,
status: row.status,
variants: variants.map((variant) => ({
name: variant.name,
content_type: variant.contentType,
size_bytes: variant.sizeBytes,
width: variant.width,
height: variant.height,
sha256: variant.sha256,
})),
error_code: row.errorCode,
created_at: row.createdAt.toISOString(),
updated_at: row.updatedAt.toISOString(),
};
}
+105
View File
@@ -0,0 +1,105 @@
import { open } from 'fs/promises';
import sharp from 'sharp';
import { ApiError } from '../common/filters/api-error';
sharp.cache(false);
sharp.concurrency(1);
const PNG_SIGNATURE = Buffer.from('89504e470d0a1a0a', 'hex');
const JPEG_SIGNATURE = Buffer.from('ffd8ff', 'hex');
const PNG_END_CHUNK = Buffer.from('0000000049454e44ae426082', 'hex');
const PNG_TAIL_BYTES = 64 * 1024;
const MAX_CONCURRENT_DECODES = 2;
const FORMAT_BY_CONTENT_TYPE: Record<string, string> = {
'image/png': 'png',
'image/jpeg': 'jpeg',
};
let activeDecodes = 0;
const decodeQueue: (() => void)[] = [];
export function isSupportedContentType(contentType: string): boolean {
return contentType in FORMAT_BY_CONTENT_TYPE;
}
export async function assertValidImage(
path: string,
size: number,
contentType: string,
maxPixels: number,
): Promise<void> {
const invalid = new ApiError(
'unsupported_media_type',
'file is not a valid image of the declared type',
);
const format = FORMAT_BY_CONTENT_TYPE[contentType];
if (!(await hasExpectedFraming(path, size, format))) {
throw invalid;
}
const { width, height, format: actual } = await sharp(path, {
limitInputPixels: false,
})
.metadata()
.catch(() => {
throw invalid;
});
if (!width || !height || actual !== format) {
throw invalid;
}
if (width * height > maxPixels) {
throw new ApiError('payload_too_large', 'image exceeds pixel limit');
}
await acquireDecodeSlot();
try {
await sharp(path, {
failOn: 'warning',
sequentialRead: true,
limitInputPixels: maxPixels,
}).stats();
} catch {
throw invalid;
} finally {
releaseDecodeSlot();
}
}
async function hasExpectedFraming(
path: string,
size: number,
format: string,
): Promise<boolean> {
const file = await open(path, 'r');
try {
const head = Buffer.alloc(PNG_SIGNATURE.length);
await file.read(head, 0, head.length, 0);
if (format === 'jpeg') {
return head.subarray(0, JPEG_SIGNATURE.length).equals(JPEG_SIGNATURE);
}
if (!head.equals(PNG_SIGNATURE)) {
return false;
}
const tail = Buffer.alloc(Math.min(size, PNG_TAIL_BYTES));
await file.read(tail, 0, tail.length, size - tail.length);
return tail.includes(PNG_END_CHUNK);
} finally {
await file.close();
}
}
function acquireDecodeSlot(): Promise<void> {
if (activeDecodes < MAX_CONCURRENT_DECODES) {
activeDecodes++;
return Promise.resolve();
}
return new Promise((resolve) => decodeQueue.push(resolve));
}
function releaseDecodeSlot(): void {
const next = decodeQueue.shift();
if (next) {
next();
} else {
activeDecodes--;
}
}
+163
View File
@@ -0,0 +1,163 @@
import { PrismaClient } from '@prisma/client';
import { S3Service } from '../storage/s3.service';
const MAX_REPORTED_KEYS = 20;
const DEFAULT_PREFIXES = ['attachments/', 'staging/'];
type ReconcilePrisma = Pick<PrismaClient, 'attachment' | 'attachmentVariant'>;
export interface ReconcileOptions {
apply: boolean;
graceMs: number;
stagingRetentionMs?: number;
prefix?: string;
now?: Date;
pageSize?: number;
}
export interface ReconcileReport {
mode: 'apply' | 'dry-run';
grace_seconds: number;
staging_expired: number;
scanned: number;
fresh_skipped: number;
referenced: number;
orphans: number;
deleted: number;
failed: number;
orphan_keys: string[];
duration_ms: number;
}
export async function reconcile(
prisma: ReconcilePrisma,
s3: S3Service,
options: ReconcileOptions,
): Promise<ReconcileReport> {
const started = Date.now();
const cutoff = (options.now ?? new Date()).getTime() - options.graceMs;
const report: ReconcileReport = {
mode: options.apply ? 'apply' : 'dry-run',
grace_seconds: options.graceMs / 1000,
staging_expired: 0,
scanned: 0,
fresh_skipped: 0,
referenced: 0,
orphans: 0,
deleted: 0,
failed: 0,
orphan_keys: [],
duration_ms: 0,
};
if (options.stagingRetentionMs !== undefined) {
report.staging_expired = await expireFailedStaging(
prisma,
s3,
(options.now ?? new Date()).getTime() - options.stagingRetentionMs,
options.apply,
options.prefix,
);
}
for (const prefix of options.prefix ? [options.prefix] : DEFAULT_PREFIXES) {
await reconcilePrefix(prisma, s3, prefix, cutoff, options, report);
}
report.duration_ms = Date.now() - started;
return report;
}
async function expireFailedStaging(
prisma: ReconcilePrisma,
s3: S3Service,
before: number,
apply: boolean,
prefix?: string,
): Promise<number> {
const rows = await prisma.attachment.findMany({
where: {
stagingKey: { not: null, startsWith: prefix },
OR: [{ status: 'ready' }, { status: 'failed', updatedAt: { lt: new Date(before) } }],
},
select: { id: true, stagingKey: true },
take: 500,
});
if (!apply) {
return rows.length;
}
let expired = 0;
for (const row of rows) {
const released = await prisma.attachment.updateMany({
where: {
id: row.id,
stagingKey: row.stagingKey,
OR: [{ status: 'ready' }, { status: 'failed', updatedAt: { lt: new Date(before) } }],
},
data: { stagingKey: null },
});
if (released.count === 0) {
continue;
}
await s3.delete(row.stagingKey!).then(
() => expired++,
() => undefined,
);
}
return expired;
}
function attachmentIdOf(key: string): string | null {
const match = /^attachments\/([0-9a-f-]{36})(\/|$)/.exec(key);
return match ? match[1] : null;
}
async function reconcilePrefix(
prisma: ReconcilePrisma,
s3: S3Service,
prefix: string,
cutoff: number,
options: ReconcileOptions,
report: ReconcileReport,
): Promise<void> {
let token: string | undefined;
do {
const page = await s3.listObjects(prefix, options.pageSize ?? 500, token);
token = page.nextToken;
report.scanned += page.objects.length;
const old = page.objects.filter((object) => object.lastModified.getTime() < cutoff);
report.fresh_skipped += page.objects.length - old.length;
const keys = old.map((object) => object.key);
const rows = await prisma.attachment.findMany({
where: { OR: [{ objectKey: { in: keys } }, { stagingKey: { in: keys } }] },
select: { objectKey: true, stagingKey: true },
});
const variants = await prisma.attachmentVariant.findMany({
where: { objectKey: { in: keys } },
select: { objectKey: true },
});
const ids = [...new Set(keys.map(attachmentIdOf).filter((id): id is string => id !== null))];
const owners = ids.length
? await prisma.attachment.findMany({ where: { id: { in: ids } }, select: { id: true } })
: [];
const owned = new Set(owners.map((row) => row.id));
const known = new Set([
...rows.flatMap((row) => [row.objectKey, row.stagingKey]),
...variants.map((row) => row.objectKey),
]);
for (const { key } of old) {
const owner = attachmentIdOf(key);
if (known.has(key) || (owner !== null && owned.has(owner))) {
report.referenced++;
continue;
}
report.orphans++;
if (report.orphan_keys.length < MAX_REPORTED_KEYS) {
report.orphan_keys.push(key);
}
if (options.apply) {
await s3.delete(key).then(
() => report.deleted++,
() => report.failed++,
);
}
}
} while (token);
}
+98
View File
@@ -0,0 +1,98 @@
import { createHash } from 'crypto';
import { FastifyInstance, FastifyRequest } from 'fastify';
import { createWriteStream } from 'fs';
import { Readable } from 'stream';
import { ApiError } from '../common/filters/api-error';
const UPLOAD_ROUTE = '/api/v1/chats/:chatId/attachments';
const PASS_THROUGH_TYPE = 'application/x-messenger-upload';
declare module 'fastify' {
interface FastifyRequest {
uploadContentType?: string;
}
}
export function registerUploadBody(instance: FastifyInstance): void {
instance.addContentTypeParser(PASS_THROUGH_TYPE, (_request, payload, done) => {
done(null, payload);
});
instance.addHook('onRequest', (request: FastifyRequest, _reply, done) => {
if (request.method === 'POST' && request.routeOptions.url === UPLOAD_ROUTE) {
request.uploadContentType = request.headers['content-type'];
request.headers['content-type'] = PASS_THROUGH_TYPE;
}
done();
});
}
export interface ReceivedBody {
size: number;
sha256: string;
}
export function receiveBody(
body: Readable,
path: string,
maxBytes: number,
timeoutMs: number,
): Promise<ReceivedBody> {
return new Promise((resolve, reject) => {
const hash = createHash('sha256');
const out = createWriteStream(path, { flags: 'wx', mode: 0o600 });
let size = 0;
let settled = false;
const timer = setTimeout(() => body.destroy(), timeoutMs);
const settle = (done: () => void): void => {
if (settled) {
return;
}
settled = true;
clearTimeout(timer);
body.off('data', onData);
body.off('end', onEnd);
body.off('close', onClose);
done();
};
const fail = (error: ApiError): void => {
settle(() => {
out.destroy();
reject(error);
});
};
const onData = (chunk: Buffer): void => {
size += chunk.length;
if (size > maxBytes) {
fail(new ApiError('payload_too_large', 'attachment exceeds size limit'));
return;
}
hash.update(chunk);
if (!out.write(chunk)) {
body.pause();
out.once('drain', () => {
if (!settled) {
body.resume();
}
});
}
};
const onEnd = (): void => {
settle(() => {
out.end(() =>
resolve({ size, sha256: hash.digest('hex') }),
);
});
};
const onClose = (): void => {
fail(new ApiError('invalid_request', 'upload was interrupted'));
};
out.on('error', () => {
fail(new ApiError('unavailable', 'temporary storage unavailable'));
});
body.on('data', onData);
body.on('end', onEnd);
body.on('close', onClose);
});
}
+51
View File
@@ -0,0 +1,51 @@
import {
Controller,
Delete,
Headers,
HttpCode,
Post,
Req,
UseGuards,
} from '@nestjs/common';
import { FastifyRequest } from 'fastify';
import { ApiError } from '../common/filters/api-error';
import { toUserResponse, UserResponse } from '../users/dto/user.response';
import { AuthGuard } from './auth.guard';
import { AuthService } from './auth.service';
interface SessionResponse {
token: string;
token_type: 'Bearer';
expires_at: string;
user: UserResponse;
}
@Controller('api/v1/auth/sessions')
export class AuthController {
constructor(private readonly auth: AuthService) {}
@Post()
@HttpCode(201)
async create(
@Headers('authorization') authorization: string | undefined,
@Req() request: FastifyRequest,
): Promise<SessionResponse> {
if (request.body !== undefined) {
throw new ApiError('invalid_request', 'request body is not allowed');
}
const session = await this.auth.login(authorization);
return {
token: session.token,
token_type: 'Bearer',
expires_at: session.expiresAt.toISOString(),
user: toUserResponse(session.user),
};
}
@Delete('current')
@HttpCode(204)
@UseGuards(AuthGuard)
async revoke(@Req() request: FastifyRequest): Promise<void> {
await this.auth.revoke(request.sessionTokenHash!);
}
}
+25
View File
@@ -0,0 +1,25 @@
import { CanActivate, ExecutionContext, Injectable } from '@nestjs/common';
import { FastifyRequest } from 'fastify';
import { ApiError } from '../common/filters/api-error';
import { AuthService } from './auth.service';
const BEARER_CHALLENGE = 'Bearer realm="messenger"';
@Injectable()
export class AuthGuard implements CanActivate {
constructor(private readonly auth: AuthService) {}
async canActivate(context: ExecutionContext): Promise<boolean> {
const request = context.switchToHttp().getRequest<FastifyRequest>();
const match = /^Bearer (\S+)$/i.exec(request.headers.authorization ?? '');
const session = match ? await this.auth.resolve(match[1]) : null;
if (!session) {
throw new ApiError('unauthorized', 'missing or invalid session', {
'WWW-Authenticate': BEARER_CHALLENGE,
});
}
request.currentUser = session.user;
request.sessionTokenHash = session.tokenHash;
return true;
}
}
+12
View File
@@ -0,0 +1,12 @@
import { Global, Module } from '@nestjs/common';
import { AuthController } from './auth.controller';
import { AuthGuard } from './auth.guard';
import { AuthService } from './auth.service';
@Global()
@Module({
controllers: [AuthController],
providers: [AuthService, AuthGuard],
exports: [AuthService, AuthGuard],
})
export class AuthModule {}
+155
View File
@@ -0,0 +1,155 @@
import { Injectable } from '@nestjs/common';
import { createHash, randomBytes } from 'crypto';
import { CacheService } from '../cache/cache.service';
import { codePointLength } from '../common/dto/code-point-length.validator';
import { User } from '../common/domain';
import { ApiError } from '../common/filters/api-error';
import { PrismaService } from '../db/prisma.service';
import { toUser } from '../users/users.service';
import { verifyPassword } from './password';
const BASIC_CHALLENGE = 'Basic realm="messenger", charset="UTF-8"';
const USERNAME_PATTERN = /^[a-z0-9_]{3,32}$/;
const CACHE_TTL_MS = 60_000;
const REVOKED = 'revoked';
interface CachedSession {
user: User;
expiresAt: number;
}
export interface CreatedSession {
token: string;
expiresAt: Date;
user: User;
}
export interface ResolvedSession {
tokenHash: string;
user: User;
}
@Injectable()
export class AuthService {
private readonly ttlSeconds = readSessionTtl();
constructor(
private readonly prisma: PrismaService,
private readonly cache: CacheService,
) {}
async login(authorization: string | undefined): Promise<CreatedSession> {
const credentials = parseBasic(authorization);
if (!credentials) {
throw invalidCredentials();
}
const row = await this.prisma.user.findUnique({
where: { username: credentials.username },
});
const valid = await verifyPassword(
row?.passwordHash ?? null,
credentials.password,
);
if (!row || !valid) {
throw invalidCredentials();
}
const token = randomBytes(32).toString('base64url');
const expiresAt = new Date(Date.now() + this.ttlSeconds * 1000);
await this.prisma.session.create({
data: { tokenHash: hashToken(token), userId: row.id, expiresAt },
});
return { token, expiresAt, user: toUser(row) };
}
async resolve(token: string): Promise<ResolvedSession | null> {
const tokenHash = hashToken(token);
const key = cacheKey(tokenHash);
const cached = await this.cache.get(key);
if (cached === REVOKED) {
return null;
}
if (cached) {
const entry = JSON.parse(cached) as CachedSession;
if (entry.expiresAt > Date.now()) {
return { tokenHash, user: entry.user };
}
}
const session = await this.prisma.session.findUnique({
where: { tokenHash },
include: { user: true },
});
const remainingMs = session
? session.expiresAt.getTime() - Date.now()
: 0;
if (!session || remainingMs <= 0) {
return null;
}
const user = toUser(session.user);
const entry: CachedSession = {
user,
expiresAt: session.expiresAt.getTime(),
};
await this.cache.set(
key,
JSON.stringify(entry),
Math.min(remainingMs, CACHE_TTL_MS),
true,
);
return { tokenHash, user };
}
async revoke(tokenHash: string): Promise<void> {
const marked = await this.cache.setStrict(
cacheKey(tokenHash),
REVOKED,
CACHE_TTL_MS,
);
if (!marked) {
throw new ApiError('unavailable', 'session cache unavailable');
}
await this.prisma.session.deleteMany({ where: { tokenHash } });
}
}
function cacheKey(tokenHash: string): string {
return `session:v1:${tokenHash}`;
}
function parseBasic(
header: string | undefined,
): { username: string; password: string } | null {
const match = /^Basic ([A-Za-z0-9+/]+={0,2})$/i.exec(header ?? '');
if (!match) {
return null;
}
const decoded = Buffer.from(match[1], 'base64').toString('utf8');
const separator = decoded.indexOf(':');
if (separator < 0) {
return null;
}
const username = decoded.slice(0, separator);
const password = decoded.slice(separator + 1);
const length = codePointLength(password);
if (!USERNAME_PATTERN.test(username) || length < 12 || length > 128) {
return null;
}
return { username, password };
}
function hashToken(token: string): string {
return createHash('sha256').update(token).digest('hex');
}
function invalidCredentials(): ApiError {
return new ApiError('unauthorized', 'invalid credentials', {
'WWW-Authenticate': BASIC_CHALLENGE,
});
}
function readSessionTtl(): number {
const ttl = Number(process.env.SESSION_TTL_SECONDS ?? 3600);
if (!Number.isInteger(ttl) || ttl <= 0) {
throw new Error('SESSION_TTL_SECONDS must be a positive integer');
}
return ttl;
}
+20
View File
@@ -0,0 +1,20 @@
import { createParamDecorator, ExecutionContext } from '@nestjs/common';
import { FastifyRequest } from 'fastify';
import { User } from '../common/domain';
declare module 'fastify' {
interface FastifyRequest {
currentUser?: User;
sessionTokenHash?: string;
}
}
export const CurrentUser = createParamDecorator(
(_data: unknown, ctx: ExecutionContext): User => {
const request = ctx.switchToHttp().getRequest<FastifyRequest>();
if (!request.currentUser) {
throw new Error('CurrentUser used without AuthGuard');
}
return request.currentUser;
},
);
+17
View File
@@ -0,0 +1,17 @@
import { argon2id, hash, verify } from 'argon2';
const OPTIONS = { type: argon2id, memoryCost: 19456, timeCost: 2, parallelism: 1 };
const dummyHash = hash('timing-equalizer-password', OPTIONS);
export function hashPassword(password: string): Promise<string> {
return hash(password, OPTIONS);
}
export async function verifyPassword(
passwordHash: string | null,
password: string,
): Promise<boolean> {
const valid = await verify(passwordHash ?? (await dummyHash), password);
return passwordHash !== null && valid;
}
+57
View File
@@ -0,0 +1,57 @@
import { Logger } from '@nestjs/common';
import amqp, { Channel, Connection } from 'amqplib';
import { readFileSync } from 'fs';
export const JOB_QUEUE = 'attachment.jobs';
export const DEAD_LETTER_QUEUE = 'attachment.jobs.dlq';
const DELIVERY_LIMIT = 20;
const RECONNECT_MAX_MS = 10000;
export function brokerUrl(): string {
const file = process.env.BROKER_URL_FILE;
const url = file ? readFileSync(file, 'utf8').trim() : process.env.BROKER_URL;
if (!url) {
throw new Error('BROKER_URL_FILE or BROKER_URL must be set');
}
return url;
}
export async function assertTopology(channel: Channel): Promise<void> {
await channel.assertQueue(DEAD_LETTER_QUEUE, {
durable: true,
arguments: { 'x-queue-type': 'quorum' },
});
await channel.assertQueue(JOB_QUEUE, {
durable: true,
arguments: {
'x-queue-type': 'quorum',
'x-delivery-limit': DELIVERY_LIMIT,
'x-dead-letter-exchange': '',
'x-dead-letter-routing-key': DEAD_LETTER_QUEUE,
},
});
}
export async function connectWithRetry(
logger: Logger,
stopping: () => boolean,
): Promise<Connection | null> {
let delay = 500;
while (!stopping()) {
try {
const connection = await amqp.connect(brokerUrl(), { timeout: 5000 });
logger.log('connected to broker');
return connection;
} catch (error) {
const reason = error instanceof Error ? error.message.split(' ')[0] : 'unknown';
logger.warn(`broker unavailable (${reason}), retry in ${delay} ms`);
await sleep(delay);
delay = Math.min(RECONNECT_MAX_MS, delay * 2);
}
}
return null;
}
export function sleep(ms: number): Promise<void> {
return new Promise((resolve) => setTimeout(resolve, ms));
}
+9
View File
@@ -0,0 +1,9 @@
import { Global, Module } from '@nestjs/common';
import { CacheService } from './cache.service';
@Global()
@Module({
providers: [CacheService],
exports: [CacheService],
})
export class CacheModule {}
+98
View File
@@ -0,0 +1,98 @@
import { Injectable, Logger, OnModuleDestroy } from '@nestjs/common';
import { readFileSync } from 'fs';
import Redis from 'ioredis';
@Injectable()
export class CacheService implements OnModuleDestroy {
private readonly logger = new Logger('CacheService');
private readonly client: Redis | null;
private trusted = false;
constructor() {
const url = readCacheUrl();
this.client = url
? new Redis(url, {
enableOfflineQueue: false,
maxRetriesPerRequest: 0,
connectTimeout: 1000,
commandTimeout: 200,
})
: null;
this.client?.on('error', (error) =>
this.logger.warn(`redis unavailable: ${error.message}`),
);
this.client?.on('close', () => {
this.trusted = false;
});
this.client?.on('ready', () => {
this.client!.flushdb().then(
() => {
this.trusted = true;
},
() => this.distrust(),
);
});
}
async get(key: string): Promise<string | null> {
if (!this.client || !this.trusted) {
return null;
}
try {
return await this.client.get(key);
} catch {
this.distrust();
return null;
}
}
async set(
key: string,
value: string,
ttlMs: number,
onlyIfAbsent = false,
): Promise<void> {
if (!this.client || !this.trusted) {
return;
}
try {
if (onlyIfAbsent) {
await this.client.set(key, value, 'PX', ttlMs, 'NX');
} else {
await this.client.set(key, value, 'PX', ttlMs);
}
} catch {
this.distrust();
}
}
async setStrict(key: string, value: string, ttlMs: number): Promise<boolean> {
if (!this.client) {
return true;
}
if (!this.trusted) {
return false;
}
try {
await this.client.set(key, value, 'PX', ttlMs);
return true;
} catch {
this.distrust();
return false;
}
}
onModuleDestroy(): void {
this.client?.disconnect();
}
private distrust(): void {
this.trusted = false;
this.client?.disconnect(true);
}
}
function readCacheUrl(): string | undefined {
const file = process.env.CACHE_URL_FILE;
return file ? readFileSync(file, 'utf8').trim() : process.env.CACHE_URL;
}
+19
View File
@@ -0,0 +1,19 @@
import { ApiError } from '../common/filters/api-error';
import { validate as isUuid } from '../common/uuid';
import { PrismaService } from '../db/prisma.service';
export async function assertChatMember(
prisma: PrismaService,
chatId: string,
userId: string,
): Promise<void> {
if (!isUuid(chatId)) {
throw new ApiError('invalid_request', 'chatId must be a valid UUID');
}
const member = await prisma.chatMember.findUnique({
where: { chatId_userId: { chatId, userId } },
});
if (!member) {
throw new ApiError('not_found', 'chat not found');
}
}
+57
View File
@@ -0,0 +1,57 @@
import {
Body,
Controller,
Get,
HttpCode,
Param,
Post,
Query,
UseGuards,
} from '@nestjs/common';
import { AuthGuard } from '../auth/auth.guard';
import { CurrentUser } from '../auth/current-user.decorator';
import { parseLimit, rawCursorParam } from '../common/dto/pagination-query.dto';
import { User } from '../common/domain';
import { ChatsService } from './chats.service';
import {
ChatPageResponse,
ChatResponse,
toChatResponse,
} from './dto/chat.response';
import { CreateChatDto } from './dto/create-chat.dto';
@Controller('api/v1/chats')
@UseGuards(AuthGuard)
export class ChatsController {
constructor(private readonly chats: ChatsService) {}
@Post()
@HttpCode(201)
async create(
@Body() dto: CreateChatDto,
@CurrentUser() user: User,
): Promise<ChatResponse> {
const chat = await this.chats.create(dto.title, dto.member_ids, user.id);
return toChatResponse(chat);
}
@Get()
async list(
@CurrentUser() user: User,
@Query('limit') limitRaw: unknown,
@Query('cursor') cursorRaw: unknown,
): Promise<ChatPageResponse> {
const limit = parseLimit(limitRaw);
const cursor = rawCursorParam(cursorRaw);
return this.chats.listForMember(user.id, limit, cursor);
}
@Get(':chatId')
async getOne(
@Param('chatId') chatId: string,
@CurrentUser() user: User,
): Promise<ChatResponse> {
const chat = await this.chats.getForMember(chatId, user.id);
return toChatResponse(chat);
}
}
+10
View File
@@ -0,0 +1,10 @@
import { Module } from '@nestjs/common';
import { ChatsController } from './chats.controller';
import { ChatsService } from './chats.service';
@Module({
controllers: [ChatsController],
providers: [ChatsService],
exports: [ChatsService],
})
export class ChatsModule {}
+131
View File
@@ -0,0 +1,131 @@
import { Injectable } from '@nestjs/common';
import { Prisma } from '@prisma/client';
import { decodeChatCursor, encodeChatCursor } from '../common/cursor';
import { Chat } from '../common/domain';
import { ApiError } from '../common/filters/api-error';
import { validate as isUuid } from '../common/uuid';
import { PrismaService, isForeignKeyViolation } from '../db/prisma.service';
import { ChatPageResponse, toChatResponse } from './dto/chat.response';
interface ChatRow {
id: string;
title: string;
createdAt: Date;
memberIds: string[];
}
@Injectable()
export class ChatsService {
constructor(private readonly prisma: PrismaService) {}
async create(
title: string,
rawMemberIds: string[],
callerId: string,
): Promise<Chat> {
const invalid = rawMemberIds.find((id) => !isUuid(id));
if (invalid) {
throw new ApiError('invalid_request', `unknown member_id: ${invalid}`);
}
const memberIds = rawMemberIds.map((id) => id.toLowerCase());
if (new Set(memberIds).size !== memberIds.length) {
throw new ApiError(
'invalid_request',
"All member_ids's elements must be unique",
);
}
if (!memberIds.includes(callerId)) {
throw new ApiError(
'invalid_request',
'member_ids must include the caller',
);
}
try {
const chat = await this.prisma.$transaction(async (tx) => {
const created = await tx.chat.create({
data: { title: title.trim() },
});
await tx.chatMember.createMany({
data: memberIds.map((userId) => ({
chatId: created.id,
userId,
})),
});
return created;
});
return toChat({ ...chat, memberIds });
} catch (error) {
if (isForeignKeyViolation(error)) {
throw new ApiError(
'invalid_request',
'member_ids contain an unknown user',
);
}
throw error;
}
}
async getForMember(chatId: string, callerId: string): Promise<Chat> {
if (!isUuid(chatId)) {
throw new ApiError('invalid_request', 'chatId must be a valid UUID');
}
const chat = await this.prisma.chat.findFirst({
where: { id: chatId, members: { some: { userId: callerId } } },
include: { members: { select: { userId: true } } },
});
if (!chat) {
throw new ApiError('not_found', 'chat not found');
}
return toChat({
...chat,
memberIds: chat.members.map((m) => m.userId),
});
}
async listForMember(
callerId: string,
limit: number,
rawCursor: string | null,
): Promise<ChatPageResponse> {
let after = Prisma.empty;
if (rawCursor !== null) {
const cursor = decodeChatCursor(rawCursor);
const member = cursor
? await this.prisma.chatMember.findUnique({
where: {
chatId_userId: { chatId: cursor.id, userId: callerId },
},
include: { chat: { select: { createdAt: true } } },
})
: null;
if (!member) {
throw new ApiError('invalid_request', 'invalid cursor');
}
after = Prisma.sql`AND (c.created_at, c.id) > (${member.chat.createdAt}::timestamptz, ${member.chatId}::uuid)`;
}
const rows = await this.prisma.$queryRaw<ChatRow[]>`
SELECT c.id, c.title, c.created_at AS "createdAt",
ARRAY(SELECT m.user_id::text FROM chat_members m WHERE m.chat_id = c.id) AS "memberIds"
FROM chat_members cm
JOIN chats c ON c.id = cm.chat_id
WHERE cm.user_id = ${callerId}::uuid ${after}
ORDER BY c.created_at, c.id
LIMIT ${limit + 1}`;
const items = rows.slice(0, limit).map(toChat);
const last = items[items.length - 1];
return {
items: items.map(toChatResponse),
next_cursor:
rows.length > limit ? encodeChatCursor({ id: last.id }) : null,
};
}
}
function toChat(row: ChatRow): Chat {
return {
id: row.id,
title: row.title,
memberIds: row.memberIds,
createdAt: row.createdAt.toISOString(),
};
}
+22
View File
@@ -0,0 +1,22 @@
import { Chat } from '../../common/domain';
export interface ChatResponse {
id: string;
title: string;
member_ids: string[];
created_at: string;
}
export interface ChatPageResponse {
items: ChatResponse[];
next_cursor: string | null;
}
export function toChatResponse(chat: Chat): ChatResponse {
return {
id: chat.id,
title: chat.title,
member_ids: chat.memberIds,
created_at: chat.createdAt,
};
}
+22
View File
@@ -0,0 +1,22 @@
import {
ArrayMaxSize,
ArrayMinSize,
ArrayUnique,
IsArray,
IsString,
} from 'class-validator';
import { CodePointLength } from '../../common/dto/code-point-length.validator';
import { NotBlank } from '../../common/dto/not-blank.validator';
export class CreateChatDto {
@NotBlank()
@CodePointLength(1, 100)
title!: string;
@IsArray()
@ArrayMinSize(1)
@ArrayMaxSize(100)
@ArrayUnique()
@IsString({ each: true })
member_ids!: string[];
}
+49
View File
@@ -0,0 +1,49 @@
import { validate as isUuid } from './uuid';
export interface ChatCursor {
id: string;
}
export interface MessageCursor {
chatId: string;
id: string;
}
export function encodeChatCursor(cursor: ChatCursor): string {
return encode(cursor);
}
export function decodeChatCursor(raw: string): ChatCursor | null {
const decoded = decode(raw);
return decoded && isUuidField(decoded.id) ? { id: decoded.id } : null;
}
export function encodeMessageCursor(cursor: MessageCursor): string {
return encode(cursor);
}
export function decodeMessageCursor(raw: string): MessageCursor | null {
const decoded = decode(raw);
return decoded && isUuidField(decoded.chatId) && isUuidField(decoded.id)
? { chatId: decoded.chatId, id: decoded.id }
: null;
}
function isUuidField(value: unknown): value is string {
return typeof value === 'string' && isUuid(value);
}
function encode(value: object): string {
return Buffer.from(JSON.stringify(value), 'utf8').toString('base64url');
}
function decode(raw: string): Record<string, unknown> | null {
try {
const parsed = JSON.parse(
Buffer.from(raw, 'base64url').toString('utf8'),
);
return typeof parsed === 'object' && parsed !== null ? parsed : null;
} catch {
return null;
}
}
+22
View File
@@ -0,0 +1,22 @@
export interface User {
id: string;
username: string;
displayName: string;
createdAt: string;
}
export interface Chat {
id: string;
title: string;
memberIds: string[];
createdAt: string;
}
export interface Message {
id: string;
chatId: string;
senderId: string;
text: string;
createdAt: string;
attachmentIds: string[];
}
@@ -0,0 +1,33 @@
import { registerDecorator, ValidationOptions } from 'class-validator';
export function codePointLength(value: string): number {
return [...value].length;
}
export function CodePointLength(
min: number,
max: number,
options?: ValidationOptions,
) {
return function (object: object, propertyName: string) {
registerDecorator({
name: 'codePointLength',
target: object.constructor,
propertyName,
constraints: [min, max],
options,
validator: {
validate(value: unknown): boolean {
if (typeof value !== 'string') {
return false;
}
const length = codePointLength(value);
return length >= min && length <= max;
},
defaultMessage(): string {
return `${propertyName} must be between ${min} and ${max} Unicode code points`;
},
},
});
};
}
+20
View File
@@ -0,0 +1,20 @@
import { registerDecorator, ValidationOptions } from 'class-validator';
export function NotBlank(options?: ValidationOptions) {
return function (object: object, propertyName: string) {
registerDecorator({
name: 'notBlank',
target: object.constructor,
propertyName,
options,
validator: {
validate(value: unknown): boolean {
return typeof value === 'string' && value.trim().length > 0;
},
defaultMessage(): string {
return `${propertyName} must not be empty or whitespace-only`;
},
},
});
};
}
+35
View File
@@ -0,0 +1,35 @@
import { ApiError } from '../filters/api-error';
const DEFAULT_LIMIT = 50;
const MIN_LIMIT = 1;
const MAX_LIMIT = 100;
export function parseLimit(raw: unknown): number {
if (raw === undefined) {
return DEFAULT_LIMIT;
}
if (typeof raw !== 'string' || !/^\d+$/.test(raw)) {
throw new ApiError('invalid_request', 'limit must be an integer');
}
const limit = Number(raw);
if (limit < MIN_LIMIT || limit > MAX_LIMIT) {
throw new ApiError(
'invalid_request',
`limit must be between ${MIN_LIMIT} and ${MAX_LIMIT}`,
);
}
return limit;
}
export function rawCursorParam(raw: unknown): string | null {
if (raw === undefined) {
return null;
}
if (typeof raw !== 'string' || raw.length === 0) {
throw new ApiError(
'invalid_request',
'cursor must be a non-empty string',
);
}
return raw;
}
+34
View File
@@ -0,0 +1,34 @@
import { HttpException, HttpStatus } from '@nestjs/common';
export type ErrorCode =
| 'invalid_request'
| 'unauthorized'
| 'not_found'
| 'conflict'
| 'payload_too_large'
| 'unsupported_media_type'
| 'rate_limited'
| 'unavailable'
| 'internal_error';
const STATUS_BY_CODE: Record<ErrorCode, number> = {
invalid_request: HttpStatus.BAD_REQUEST,
unauthorized: HttpStatus.UNAUTHORIZED,
not_found: HttpStatus.NOT_FOUND,
conflict: HttpStatus.CONFLICT,
payload_too_large: HttpStatus.PAYLOAD_TOO_LARGE,
unsupported_media_type: HttpStatus.UNSUPPORTED_MEDIA_TYPE,
rate_limited: HttpStatus.TOO_MANY_REQUESTS,
unavailable: HttpStatus.SERVICE_UNAVAILABLE,
internal_error: HttpStatus.INTERNAL_SERVER_ERROR,
};
export class ApiError extends HttpException {
constructor(
public readonly code: ErrorCode,
message: string,
public readonly headers: Record<string, string> = {},
) {
super(message, STATUS_BY_CODE[code]);
}
}
+121
View File
@@ -0,0 +1,121 @@
import {
ArgumentsHost,
Catch,
ExceptionFilter,
HttpException,
HttpStatus,
Logger,
} from '@nestjs/common';
import { FastifyReply, FastifyRequest } from 'fastify';
import { isDatabaseUnavailable } from '../../db/prisma.service';
import { routeTemplate } from '../../telemetry/http';
import { ApiError, ErrorCode } from './api-error';
const CODE_BY_STATUS: Partial<Record<number, ErrorCode>> = {
[HttpStatus.BAD_REQUEST]: 'invalid_request',
[HttpStatus.UNAUTHORIZED]: 'unauthorized',
[HttpStatus.NOT_FOUND]: 'not_found',
[HttpStatus.CONFLICT]: 'conflict',
[HttpStatus.PAYLOAD_TOO_LARGE]: 'payload_too_large',
[HttpStatus.UNSUPPORTED_MEDIA_TYPE]: 'unsupported_media_type',
[HttpStatus.TOO_MANY_REQUESTS]: 'rate_limited',
[HttpStatus.SERVICE_UNAVAILABLE]: 'unavailable',
};
interface Resolved {
status: number;
code: ErrorCode;
message: string;
}
@Catch()
export class HttpExceptionFilter implements ExceptionFilter {
private readonly logger = new Logger('HttpExceptionFilter');
catch(exception: unknown, host: ArgumentsHost): void {
const ctx = host.switchToHttp();
const reply = ctx.getResponse<FastifyReply>();
const request = ctx.getRequest<FastifyRequest>();
const requestId = request.requestId;
const { status, code, message } = this.resolve(exception);
if (status >= 500) {
this.logger.error({
event: 'http.error',
message,
method: request.method,
route: routeTemplate(request),
status,
request_id: requestId,
});
}
if (exception instanceof ApiError) {
reply.headers(exception.headers);
}
reply
.code(status)
.header('x-request-id', requestId)
.send({ error: { code, message, request_id: requestId } });
}
private resolve(exception: unknown): Resolved {
if (exception instanceof ApiError) {
return {
status: exception.getStatus(),
code: exception.code,
message: exception.message,
};
}
if (exception instanceof HttpException) {
const status = exception.getStatus();
const code = CODE_BY_STATUS[status] ?? 'internal_error';
const message =
extractMessage(exception.getResponse()) ?? exception.message;
return { status, code, message };
}
const fastifyStatus = (exception as { statusCode?: unknown })
?.statusCode;
if (typeof fastifyStatus === 'number' && CODE_BY_STATUS[fastifyStatus]) {
const message =
exception instanceof Error ? exception.message : 'Invalid request';
return {
status: fastifyStatus,
code: CODE_BY_STATUS[fastifyStatus]!,
message,
};
}
if (isDatabaseUnavailable(exception)) {
return {
status: 503,
code: 'unavailable',
message: 'database unavailable',
};
}
return {
status: 500,
code: 'internal_error',
message: 'Internal server error',
};
}
}
function extractMessage(response: unknown): string | null {
if (typeof response === 'string') {
return response;
}
if (
typeof response === 'object' &&
response !== null &&
'message' in response
) {
const value = (response as { message: unknown }).message;
if (typeof value === 'string') {
return value;
}
if (Array.isArray(value) && value.every((v) => typeof v === 'string')) {
return value.join('; ');
}
}
return null;
}
+23
View File
@@ -0,0 +1,23 @@
import { randomUUID } from 'crypto';
import { FastifyInstance, FastifyRequest } from 'fastify';
import { hostname } from 'os';
const REQUEST_ID_HEADER = 'x-request-id';
const INSTANCE_ID_HEADER = 'x-instance-id';
export const INSTANCE_ID = process.env.INSTANCE_ID ?? hostname();
declare module 'fastify' {
interface FastifyRequest {
requestId: string;
}
}
export function registerRequestId(instance: FastifyInstance): void {
instance.addHook('onRequest', (request: FastifyRequest, reply, done) => {
request.requestId = randomUUID();
reply.header(REQUEST_ID_HEADER, request.requestId);
reply.header(INSTANCE_ID_HEADER, INSTANCE_ID);
done();
});
}
+6
View File
@@ -0,0 +1,6 @@
const UUID_PATTERN =
/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
export function validate(value: string): boolean {
return UUID_PATTERN.test(value);
}
+21
View File
@@ -0,0 +1,21 @@
import { ValidationPipe } from '@nestjs/common';
import { NestFastifyApplication } from '@nestjs/platform-fastify';
import { registerUploadBody } from './attachments/upload-body';
import { HttpExceptionFilter } from './common/filters/http-exception.filter';
import { registerRequestId } from './common/interceptors/request-id';
export function configureApp(app: NestFastifyApplication): void {
const instance = app.getHttpAdapter().getInstance();
registerRequestId(instance);
registerUploadBody(instance);
app.useGlobalPipes(
new ValidationPipe({
whitelist: true,
forbidNonWhitelisted: true,
transform: true,
forbidUnknownValues: true,
}),
);
app.useGlobalFilters(new HttpExceptionFilter());
}
+9
View File
@@ -0,0 +1,9 @@
import { Global, Module } from '@nestjs/common';
import { PrismaService } from './prisma.service';
@Global()
@Module({
providers: [PrismaService],
exports: [PrismaService],
})
export class DbModule {}
+52
View File
@@ -0,0 +1,52 @@
import { Injectable, OnModuleDestroy } from '@nestjs/common';
import { Prisma, PrismaClient } from '@prisma/client';
import { readFileSync } from 'fs';
import { registerDbPool } from '../telemetry/db-pool';
const UNAVAILABLE_CODES = new Set(['P1001', 'P1002', 'P1008', 'P1017', 'P2024']);
@Injectable()
export class PrismaService extends PrismaClient implements OnModuleDestroy {
constructor() {
const url = readDatabaseUrl();
super({ datasourceUrl: url });
registerDbPool(this, url);
}
async onModuleDestroy(): Promise<void> {
await this.$disconnect();
}
}
function readDatabaseUrl(): string {
const file = process.env.DATABASE_URL_FILE;
const url = file ? readFileSync(file, 'utf8').trim() : process.env.DATABASE_URL;
if (!url) {
throw new Error('DATABASE_URL_FILE or DATABASE_URL must be set');
}
return url;
}
export function isUniqueViolation(error: unknown): boolean {
return (
error instanceof Prisma.PrismaClientKnownRequestError &&
error.code === 'P2002'
);
}
export function isForeignKeyViolation(error: unknown): boolean {
return (
error instanceof Prisma.PrismaClientKnownRequestError &&
error.code === 'P2003'
);
}
export function isDatabaseUnavailable(error: unknown): boolean {
if (error instanceof Prisma.PrismaClientInitializationError) {
return true;
}
return (
error instanceof Prisma.PrismaClientKnownRequestError &&
UNAVAILABLE_CODES.has(error.code)
);
}
+351
View File
@@ -0,0 +1,351 @@
const ALICE_ID = '11111111-1111-4111-8111-111111111111';
const BOB_ID = '22222222-2222-4222-8222-222222222222';
const CHAT_ID = '33333333-3333-4333-8333-333333333333';
const MESSAGE_ID = '44444444-4444-4444-8444-444444444444';
const CURSOR =
'eyJjaGF0SWQiOiIzMzMzMzMzMy0zMzMzLTQzMzMtODMzMy0zMzMzMzMzMzMzMzMiLCJpZCI6IjQ0NDQ0NDQ0LTQ0NDQtNDQ0NC04NDQ0LTQ0NDQ0NDQ0NDQ0NCJ9';
const ALICE = {
id: ALICE_ID,
username: 'alice',
display_name: 'Алиса',
created_at: '2026-09-01T12:00:00.000Z',
};
const CHAT = {
id: CHAT_ID,
title: 'Проект по ВНП',
member_ids: [ALICE_ID, BOB_ID],
created_at: '2026-09-01T12:01:00.000Z',
};
const MESSAGE = {
id: MESSAGE_ID,
chat_id: CHAT_ID,
sender_id: ALICE_ID,
text: 'Привет, Боб! 👋',
created_at: '2026-09-01T12:02:00.000Z',
attachment_ids: [],
};
export interface Example {
summary: string;
value: unknown;
}
export interface OperationExamples {
request?: Record<string, Example>;
responses: Record<string, Record<string, Example>>;
}
function example(summary: string, value: unknown): Example {
return { summary, value };
}
function error(code: string, message: string): unknown {
return {
error: {
code,
message,
request_id: '9b2c6f1e-5d4a-4c3b-8a2f-1e0d9c8b7a6f',
},
};
}
const UNAUTHORIZED = {
missing: example(
'Нет/неверный/истёкший/отозванный Bearer-токен',
error('unauthorized', 'missing or invalid session'),
),
};
const UNAVAILABLE = {
database: example(
'PostgreSQL недоступен',
error('unavailable', 'database unavailable'),
),
};
const CHAT_NOT_FOUND = {
notFound: example(
'Чата нет или вызывающий не участник (не 403)',
error('not_found', 'chat not found'),
),
};
const INVALID_CHAT_ID = example(
'chat_id не UUID',
error('invalid_request', 'chatId must be a valid UUID'),
);
export const GUIDE = `
Учебный мессенджер, лабораторная 6: PostgreSQL, сессии, Redis-кэш сессий, вложения в S3 и их фоновая обработка.
Документ — обязательный контракт \`contracts/openapi.json\`, дополненный примерами запросов и ответов.
**Как пройти сценарий в Swagger UI**
1. \`POST /api/v1/users\` — зарегистрируйте пользователей (примеры alice, bob, eve в списке Examples). Повторная регистрация того же username — 409, меняйте username.
2. **Authorize → basicAuth**: username и password из шага 1 → \`POST /api/v1/auth/sessions\` → скопируйте \`token\`.
3. **Authorize → bearerAuth**: вставьте token (без слова Bearer). Basic отправляется только на вход, остальные запросы идут с Bearer.
4. \`GET /api/v1/users/me\` — свой id; id второго участника — из ответа его регистрации.
5. \`POST /api/v1/chats\` с \`member_ids\` = [свой id, id собеседника] → \`chat_id\`.
6. \`POST /api/v1/chats/{chat_id}/messages\` с заголовком \`Idempotency-Key\` (любая строка из \`A-Za-z0-9._:-\`, до 128 символов). Повтор с тем же ключом и телом вернёт то же сообщение, с другим телом — 409. Затем \`GET …/messages?limit=1\` и следующая страница с \`cursor\` = \`next_cursor\`.
7. \`POST /api/v1/chats/{chat_id}/attachments?filename=photo.png\` с телом PNG/JPEG → 202 \`queued\`. \`GET /api/v1/attachments/{id}\` повторять, пока \`status\` не станет \`ready\` (до этого \`/content\` отвечает 409), затем \`/content?variant=thumbnail\`, \`crop\` или \`optimized\`.
8. Чтобы действовать от другого пользователя: Logout в Authorize, вход им и новый token.
9. \`DELETE /api/v1/auth/sessions/current\` — logout; тот же token дальше даёт 401.
Сессия живёт \`SESSION_TTL_SECONDS\` (по умолчанию 3600 с). Пароль — 12–128 символов Unicode, двоеточие допустимо.
`;
export const OPERATION_EXAMPLES: Record<string, OperationExamples> = {
'GET /health/live': {
responses: {
'200': { ok: example('Процесс жив', { status: 'ok' }) },
},
},
'GET /health/ready': {
responses: {
'200': {
ok: example('PostgreSQL доступен (Redis не критичен)', {
status: 'ok',
}),
},
'503': UNAVAILABLE,
},
},
'POST /api/v1/users': {
request: {
alice: example('Алиса', {
username: 'alice',
display_name: 'Алиса',
password: 'Alice-password-123',
}),
bob: example('Боб', {
username: 'bob',
display_name: 'Боб',
password: 'Bob-password-123',
}),
eve: example('Ева (посторонняя)', {
username: 'eve',
display_name: 'Ева',
password: 'Eve-password-123',
}),
colon: example('Пароль с двоеточием и эмодзи', {
username: 'carol',
display_name: 'Кэрол 🌸',
password: 'пароль:с:двоеточием🔑',
}),
},
responses: {
'201': { created: example('Пользователь создан', ALICE) },
'400': {
username: example(
'username не соответствует [a-z0-9_]{3,32}',
error('invalid_request', 'username must match [a-z0-9_]{3,32}'),
),
password: example(
'password короче 12 символов',
error(
'invalid_request',
'password must be between 12 and 128 Unicode code points',
),
),
extra: example(
'Лишнее поле в JSON',
error('invalid_request', 'property extra should not exist'),
),
},
'409': {
taken: example(
'username уже занят',
error('conflict', 'username is already taken'),
),
},
'503': UNAVAILABLE,
},
},
'POST /api/v1/auth/sessions': {
responses: {
'201': {
created: example('Сессия создана', {
token: 'q8Hn3vXl0ZbJ7mWcY2sRkT5uPaE9dFgVhN1oLxI4w6A',
token_type: 'Bearer',
expires_at: '2026-09-01T13:00:00.000Z',
user: ALICE,
}),
},
'401': {
invalid: example(
'Неверный пароль, неизвестный пользователь или нет Basic — ответ одинаковый',
error('unauthorized', 'invalid credentials'),
),
},
'503': UNAVAILABLE,
},
},
'DELETE /api/v1/auth/sessions/current': {
responses: {
'401': UNAUTHORIZED,
'503': {
...UNAVAILABLE,
cache: example(
'Redis недоступен — отзыв не выполнен, повторите позже',
error('unavailable', 'session cache unavailable'),
),
},
},
},
'GET /api/v1/users/me': {
responses: {
'200': { me: example('Текущий пользователь из сессии', ALICE) },
'401': UNAUTHORIZED,
'503': UNAVAILABLE,
},
},
'POST /api/v1/chats': {
request: {
pair: example('Чат Алисы и Боба', {
title: CHAT.title,
member_ids: CHAT.member_ids,
}),
solo: example('Чат только с собой', {
title: 'Заметки',
member_ids: [ALICE_ID],
}),
},
responses: {
'201': { created: example('Чат создан атомарно', CHAT) },
'400': {
noCaller: example(
'Вызывающий не в member_ids',
error('invalid_request', 'member_ids must include the caller'),
),
unknown: example(
'Неизвестный участник — транзакция откатывается',
error('invalid_request', 'member_ids contain an unknown user'),
),
duplicate: example(
'Дубликаты в member_ids',
error(
'invalid_request',
"All member_ids's elements must be unique",
),
),
},
'401': UNAUTHORIZED,
'503': UNAVAILABLE,
},
},
'GET /api/v1/chats': {
responses: {
'200': {
page: example('Страница чатов с продолжением', {
items: [CHAT],
next_cursor: 'eyJpZCI6IjMzMzMzMzMzLTMzMzMtNDMzMy04MzMzLTMzMzMzMzMzMzMzMyJ9',
}),
last: example('Последняя страница', {
items: [CHAT],
next_cursor: null,
}),
},
'400': {
limit: example(
'limit вне 1–100',
error('invalid_request', 'limit must be between 1 and 100'),
),
cursor: example(
'Испорченный или чужой cursor',
error('invalid_request', 'invalid cursor'),
),
},
'401': UNAUTHORIZED,
'503': UNAVAILABLE,
},
},
'GET /api/v1/chats/{chat_id}': {
responses: {
'200': { chat: example('Чат, где вызывающий — участник', CHAT) },
'400': { chatId: INVALID_CHAT_ID },
'401': UNAUTHORIZED,
'404': CHAT_NOT_FOUND,
'503': UNAVAILABLE,
},
},
'POST /api/v1/chats/{chat_id}/messages': {
request: {
text: example('Обычное сообщение', { text: MESSAGE.text }),
long: example('Максимум — 4000 символов Unicode', {
text: 'Длинный текст…',
}),
},
responses: {
'201': {
created: example('Отправитель берётся из сессии', MESSAGE),
},
'400': {
blank: example(
'Пустой или пробельный текст',
error(
'invalid_request',
'text must not be empty or whitespace-only',
),
),
sender: example(
'sender_id в теле не принимается',
error('invalid_request', 'property sender_id should not exist'),
),
chatId: INVALID_CHAT_ID,
key: example(
'Нет Idempotency-Key или неверный формат',
error(
'invalid_request',
'Idempotency-Key header must match [A-Za-z0-9._:-]{1,128}',
),
),
},
'401': UNAUTHORIZED,
'404': CHAT_NOT_FOUND,
'409': {
keyReused: example(
'Тот же Idempotency-Key с другим телом',
error(
'conflict',
'Idempotency-Key was already used with a different request',
),
),
},
'503': UNAVAILABLE,
},
},
'GET /api/v1/chats/{chat_id}/messages': {
responses: {
'200': {
page: example('Страница истории (limit=1) с cursor', {
items: [MESSAGE],
next_cursor: CURSOR,
}),
last: example('Последняя страница', {
items: [MESSAGE],
next_cursor: null,
}),
},
'400': {
otherChat: example(
'cursor от другого чата',
error('invalid_request', 'cursor belongs to a different chat'),
),
cursor: example(
'Испорченный cursor или cursor несуществующего сообщения',
error('invalid_request', 'invalid cursor'),
),
limit: example(
'limit вне 1–100',
error('invalid_request', 'limit must be between 1 and 100'),
),
},
'401': UNAUTHORIZED,
'404': CHAT_NOT_FOUND,
'503': UNAVAILABLE,
},
},
};
+43
View File
@@ -0,0 +1,43 @@
import { NestFastifyApplication } from '@nestjs/platform-fastify';
import { OpenAPIObject, SwaggerModule } from '@nestjs/swagger';
import { readFileSync } from 'fs';
import { join } from 'path';
import { GUIDE, OPERATION_EXAMPLES } from './examples';
const CONTRACT_PATH = join(process.cwd(), 'contracts', 'openapi.json');
export function setupSwagger(app: NestFastifyApplication): void {
SwaggerModule.setup('docs', app, loadDocument(), {
customSiteTitle: 'Messenger lab6 API',
swaggerOptions: {
persistAuthorization: true,
displayRequestDuration: true,
tryItOutEnabled: true,
},
});
}
export function loadDocument(): OpenAPIObject {
const document = JSON.parse(readFileSync(CONTRACT_PATH, 'utf8'));
document.info.description = GUIDE;
document.servers = [{ url: '/', description: 'Этот экземпляр API' }];
for (const [key, examples] of Object.entries(OPERATION_EXAMPLES)) {
const [method, path] = key.split(' ');
const operation = document.paths[path]?.[method.toLowerCase()];
if (!operation) {
throw new Error(`examples for unknown operation ${key}`);
}
if (examples.request) {
operation.requestBody.content['application/json'].examples =
examples.request;
}
for (const [status, named] of Object.entries(examples.responses)) {
const content = operation.responses[status]?.content;
if (!content) {
throw new Error(`examples for unknown response ${key} ${status}`);
}
content['application/json'].examples = named;
}
}
return document;
}
+44
View File
@@ -0,0 +1,44 @@
import { Controller, Get } from '@nestjs/common';
import { ApiError } from '../common/filters/api-error';
import { PrismaService } from '../db/prisma.service';
import { S3Service } from '../storage/s3.service';
import { isDraining } from './shutdown';
@Controller('health')
export class HealthController {
constructor(
private readonly prisma: PrismaService,
private readonly s3: S3Service,
) {}
@Get('live')
live(): { status: 'ok' } {
return { status: 'ok' };
}
@Get('core')
async core(): Promise<{ status: 'ok' }> {
await this.assertCoreReady();
return { status: 'ok' };
}
@Get('ready')
async ready(): Promise<{ status: 'ok' }> {
await this.assertCoreReady();
if (!(await this.s3.isAvailable())) {
throw new ApiError('unavailable', 'object storage unavailable');
}
return { status: 'ok' };
}
private async assertCoreReady(): Promise<void> {
if (isDraining()) {
throw new ApiError('unavailable', 'shutting down');
}
try {
await this.prisma.$queryRaw`SELECT 1`;
} catch {
throw new ApiError('unavailable', 'database unavailable');
}
}
}
+7
View File
@@ -0,0 +1,7 @@
import { Module } from '@nestjs/common';
import { HealthController } from './health.controller';
@Module({
controllers: [HealthController],
})
export class HealthModule {}
+33
View File
@@ -0,0 +1,33 @@
import { INestApplication, Logger } from '@nestjs/common';
import { shutdownTelemetry } from '../telemetry/sdk';
let draining = false;
export function isDraining(): boolean {
return draining;
}
export function enableGracefulShutdown(app: INestApplication): void {
const delayMs = Number(process.env.SHUTDOWN_DELAY_MS ?? 3000);
const shutdown = (signal: string): void => {
if (draining) {
return;
}
draining = true;
Logger.log(`${signal}: readiness → 503, closing in ${delayMs} ms`, 'Shutdown');
setTimeout(() => {
app
.close()
.then(
() => 0,
() => 1,
)
.then(async (code) => {
await shutdownTelemetry();
process.exit(code);
});
}, delayMs);
};
process.once('SIGTERM', () => shutdown('SIGTERM'));
process.once('SIGINT', () => shutdown('SIGINT'));
}
+98
View File
@@ -0,0 +1,98 @@
import { context, propagation } from '@opentelemetry/api';
import { validate as isUuid } from '../common/uuid';
export const PIPELINE_VERSION = 1;
export const JOB_TYPE = 'attachment.process.v1';
const TRACEPARENT = /^00-[0-9a-f]{32}-[0-9a-f]{16}-[0-9a-f]{2}$/;
const TRACESTATE_MAX = 512;
export interface TraceCarrier {
traceparent?: string;
tracestate?: string;
}
export interface JobMessage extends TraceCarrier {
schema_version: 1;
job_id: string;
type: typeof JOB_TYPE;
attachment_id: string;
pipeline_version: number;
attempt: number;
created_at: string;
}
const FIELDS = [
'schema_version',
'job_id',
'type',
'attachment_id',
'pipeline_version',
'attempt',
'created_at',
'traceparent',
'tracestate',
];
export function jobMessage(
job: { id: string; attachmentId: string; pipelineVersion: number; attempt: number },
createdAt: Date,
): JobMessage {
return {
schema_version: 1,
job_id: job.id,
type: JOB_TYPE,
attachment_id: job.attachmentId,
pipeline_version: job.pipelineVersion,
attempt: job.attempt,
created_at: createdAt.toISOString(),
...currentTraceCarrier(),
};
}
export function currentTraceCarrier(): TraceCarrier {
const carrier: Record<string, string> = {};
propagation.inject(context.active(), carrier);
const result: TraceCarrier = {};
if (TRACEPARENT.test(carrier.traceparent ?? '')) {
result.traceparent = carrier.traceparent;
if (carrier.tracestate && carrier.tracestate.length <= TRACESTATE_MAX) {
result.tracestate = carrier.tracestate;
}
}
return result;
}
export function parseJobMessage(raw: Buffer): JobMessage | null {
let value: any;
try {
value = JSON.parse(raw.toString('utf8'));
} catch {
return null;
}
if (
typeof value !== 'object' ||
value === null ||
Array.isArray(value) ||
Object.keys(value).some((key) => !FIELDS.includes(key)) ||
value.schema_version !== 1 ||
value.type !== JOB_TYPE ||
typeof value.job_id !== 'string' ||
!isUuid(value.job_id) ||
typeof value.attachment_id !== 'string' ||
!isUuid(value.attachment_id) ||
!Number.isInteger(value.pipeline_version) ||
value.pipeline_version < 1 ||
!Number.isInteger(value.attempt) ||
value.attempt < 1 ||
typeof value.created_at !== 'string' ||
Number.isNaN(Date.parse(value.created_at)) ||
(value.traceparent !== undefined &&
(typeof value.traceparent !== 'string' || !TRACEPARENT.test(value.traceparent))) ||
(value.tracestate !== undefined &&
(typeof value.tracestate !== 'string' || value.tracestate.length > TRACESTATE_MAX))
) {
return null;
}
return value as JobMessage;
}
+257
View File
@@ -0,0 +1,257 @@
import { AttachmentJob, Prisma, PrismaClient } from '@prisma/client';
import { randomUUID } from 'crypto';
import { jobMessage, PIPELINE_VERSION } from './job-message';
type Tx = Prisma.TransactionClient;
export const RETRY_EXHAUSTED = 'retry_exhausted';
export function maxAttempts(): number {
return Number(process.env.JOB_MAX_ATTEMPTS ?? 5);
}
export function backoffMs(attempt: number, random = Math.random): number {
const base = Number(process.env.JOB_RETRY_BASE_MS ?? 1000);
const cap = Number(process.env.JOB_RETRY_MAX_MS ?? 30000);
const delay = Math.min(cap, base * 2 ** (attempt - 1));
return Math.round(delay / 2 + random() * (delay / 2));
}
export async function enqueueJob(
tx: Tx,
attachmentId: string,
now: Date,
): Promise<AttachmentJob> {
const job = await tx.attachmentJob.create({
data: {
id: randomUUID(),
attachmentId,
pipelineVersion: PIPELINE_VERSION,
status: 'queued',
attempt: 1,
maxAttempts: maxAttempts(),
nextAttemptAt: now,
createdAt: now,
updatedAt: now,
},
});
await tx.jobEvent.create({
data: { jobId: job.id, attempt: 1, event: 'enqueued', createdAt: now },
});
await publishLater(tx, job, now);
return job;
}
export async function scheduleRetry(
tx: Tx,
job: AttachmentJob,
reason: string,
now: Date,
event = 'retry_scheduled',
): Promise<AttachmentJob> {
const delay = backoffMs(job.attempt);
const next = await tx.attachmentJob.update({
where: { id: job.id },
data: {
status: 'queued',
attempt: job.attempt + 1,
nextAttemptAt: new Date(now.getTime() + delay),
leaseOwner: null,
leaseUntil: null,
lastError: reason,
},
});
await tx.attachment.update({
where: { id: job.attachmentId },
data: { status: 'queued' },
});
await tx.jobEvent.create({
data: {
jobId: job.id,
attempt: job.attempt,
event,
detail: `${reason}; next attempt ${next.attempt} in ${delay} ms`,
createdAt: now,
},
});
await publishLater(tx, next, next.nextAttemptAt);
return next;
}
export async function markFailed(
tx: Tx,
job: AttachmentJob,
errorCode: string,
reason: string,
now: Date,
): Promise<void> {
await tx.attachmentJob.update({
where: { id: job.id },
data: {
status: 'failed',
errorCode,
lastError: reason,
leaseOwner: null,
leaseUntil: null,
finishedAt: now,
},
});
await tx.attachment.update({
where: { id: job.attachmentId },
data: { status: 'failed', errorCode },
});
await tx.jobEvent.create({
data: {
jobId: job.id,
attempt: job.attempt,
event: 'failed',
detail: `${errorCode}: ${reason}`,
createdAt: now,
},
});
}
export async function retryOrFail(
tx: Tx,
job: AttachmentJob,
reason: string,
now: Date,
event?: string,
): Promise<'retry' | 'failed'> {
if (job.attempt >= job.maxAttempts) {
await markFailed(tx, job, RETRY_EXHAUSTED, reason, now);
return 'failed';
}
await scheduleRetry(tx, job, reason, now, event);
return 'retry';
}
export async function recoverExpiredLeases(
prisma: PrismaClient,
now = new Date(),
limit = 100,
): Promise<number> {
const expired = await prisma.attachmentJob.findMany({
where: { status: 'processing', leaseUntil: { lt: now } },
orderBy: { leaseUntil: 'asc' },
take: limit,
});
let recovered = 0;
for (const candidate of expired) {
await prisma.$transaction(async (tx) => {
const taken = await tx.attachmentJob.updateMany({
where: {
id: candidate.id,
status: 'processing',
leaseOwner: candidate.leaseOwner,
leaseUntil: { lt: now },
},
data: { leaseOwner: null, leaseUntil: null, status: 'queued' },
});
if (taken.count === 0) {
return;
}
await retryOrFail(tx, candidate, 'lease expired', now, 'lease_expired');
recovered++;
});
}
return recovered;
}
export async function republishStale(
prisma: PrismaClient,
staleMs: number,
now = new Date(),
limit = 100,
): Promise<number> {
const cutoff = new Date(now.getTime() - staleMs);
const stale = await prisma.$queryRaw<AttachmentJob[]>`
SELECT j.id, j.attachment_id AS "attachmentId", j.pipeline_version AS "pipelineVersion", j.attempt
FROM attachment_jobs j
WHERE j.status = 'queued'
AND j.next_attempt_at < ${cutoff}
AND NOT EXISTS (
SELECT 1 FROM outbox o
WHERE o.job_id = j.id AND (o.published_at IS NULL OR o.published_at > ${cutoff})
)
ORDER BY j.next_attempt_at
LIMIT ${limit}`;
for (const job of stale) {
await prisma.$transaction(async (tx) => {
await tx.jobEvent.create({
data: { jobId: job.id, attempt: job.attempt, event: 'republished', createdAt: now },
});
await publishLater(tx, job, now);
});
}
return stale.length;
}
export type ReplayResult =
| 'replayed'
| 'already_ready'
| 'not_failed'
| 'source_missing'
| 'not_found';
export async function replayJob(
prisma: PrismaClient,
jobId: string,
actor: string,
now = new Date(),
): Promise<ReplayResult> {
return prisma.$transaction(async (tx) => {
const job = await tx.attachmentJob.findUnique({ where: { id: jobId } });
if (!job) {
return 'not_found';
}
if (job.status === 'ready') {
return 'already_ready';
}
if (job.status !== 'failed') {
return 'not_failed';
}
const source = await tx.attachment.updateMany({
where: { id: job.attachmentId, status: 'failed', stagingKey: { not: null } },
data: { status: 'queued', errorCode: null },
});
if (source.count === 0) {
return 'source_missing';
}
const taken = await tx.attachmentJob.updateMany({
where: { id: jobId, status: 'failed' },
data: {
status: 'queued',
attempt: 1,
errorCode: null,
nextAttemptAt: now,
finishedAt: null,
},
});
if (taken.count === 0) {
throw new Error('job changed during replay');
}
await tx.jobEvent.create({
data: {
jobId,
attempt: 1,
event: 'replayed',
detail: `by ${actor}; previous ${job.errorCode} after attempt ${job.attempt}`,
createdAt: now,
},
});
await publishLater(tx, { ...job, attempt: 1 }, now);
return 'replayed';
});
}
async function publishLater(tx: Tx, job: AttachmentJob, availableAt: Date): Promise<void> {
await tx.outboxEntry.create({
data: {
jobId: job.id,
payload: jobMessage(job, availableAt) as unknown as Prisma.InputJsonObject,
availableAt,
createdAt: new Date(),
},
});
}
+298
View File
@@ -0,0 +1,298 @@
import { Logger } from '@nestjs/common';
import { metrics, SpanStatusCode, trace } from '@opentelemetry/api';
import { Attachment, AttachmentJob, PrismaClient } from '@prisma/client';
import { createHash, randomUUID } from 'crypto';
import {
ObjectMissingError,
ObjectTooLargeError,
S3Service,
} from '../storage/s3.service';
import { JOB_TYPE, JobMessage, PIPELINE_VERSION } from './job-message';
import { markFailed, retryOrFail } from './job-store';
import {
InvalidImageError,
RenderedVariant,
RenderTimeoutError,
renderVariants,
} from './variants';
export type Outcome = 'ready' | 'retry' | 'failed' | 'skipped';
export interface ProcessorOptions {
timeoutMs: number;
leaseMs: number;
maxBytes: number;
maxPixels: number;
}
export function processorOptions(): ProcessorOptions {
const timeoutMs = Number(process.env.JOB_TIMEOUT_SECONDS ?? 60) * 1000;
return {
timeoutMs,
leaseMs: Number(process.env.JOB_LEASE_SECONDS ?? timeoutMs / 1000 + 30) * 1000,
maxBytes: Number(process.env.MAX_UPLOAD_BYTES ?? 10485760),
maxPixels: Number(process.env.MAX_IMAGE_PIXELS ?? 40000000),
};
}
export interface ProcessorHooks {
afterStore?: () => Promise<void>;
afterCommit?: () => Promise<void>;
}
class PermanentError extends Error {
constructor(
readonly code: string,
reason: string,
) {
super(reason);
}
}
class TimeoutError extends Error {}
const tracer = trace.getTracer('messenger-worker');
const timeToReady = metrics.getMeter('messenger-worker').createHistogram('attachments.time_to_ready', {
description: 'From durable upload acceptance to the committed ready state',
unit: 's',
advice: {
explicitBucketBoundaries: [
0.1, 0.25, 0.5, 1, 2, 3, 5, 10, 20, 30, 45, 60, 90, 120, 150, 180, 240, 300, 450, 600,
],
},
});
export class JobProcessor {
private readonly logger = new Logger('JobProcessor');
constructor(
private readonly prisma: PrismaClient,
private readonly s3: S3Service,
private readonly options: ProcessorOptions = processorOptions(),
private readonly hooks: ProcessorHooks = {},
) {}
async handle(message: JobMessage): Promise<Outcome> {
const job = await this.prisma.attachmentJob.findUnique({
where: { id: message.job_id },
include: { attachment: true },
});
if (
!job ||
job.attachmentId !== message.attachment_id ||
job.pipelineVersion !== message.pipeline_version ||
job.pipelineVersion !== PIPELINE_VERSION
) {
this.logger.warn(`job ${message.job_id}: unknown or mismatched, dropped`);
return 'skipped';
}
const lease = await this.claim(job);
if (!lease) {
if (job.status === 'ready') {
await this.removeStaging(job.attachment);
}
this.logger.log(`job ${job.id}: ${job.status}, not due or leased elsewhere, skipped`);
return 'skipped';
}
let variants: RenderedVariant[];
try {
variants = await this.withTimeout(this.produce(job.attachment));
} catch (error) {
return this.recordFailure(lease, error);
}
await this.hooks.afterStore?.();
return this.commit(lease, job.attachment, variants);
}
private async claim(job: AttachmentJob): Promise<AttachmentJob | null> {
const now = new Date();
const owner = randomUUID();
return this.prisma.$transaction(async (tx) => {
const taken = await tx.attachmentJob.updateMany({
where: { id: job.id, status: 'queued', nextAttemptAt: { lte: now } },
data: {
status: 'processing',
leaseOwner: owner,
leaseUntil: new Date(now.getTime() + this.options.leaseMs),
},
});
if (taken.count === 0) {
return null;
}
await tx.attachment.update({
where: { id: job.attachmentId },
data: { status: 'processing' },
});
const leased = await tx.attachmentJob.findUniqueOrThrow({ where: { id: job.id } });
await tx.jobEvent.create({
data: { jobId: job.id, attempt: leased.attempt, event: 'processing', createdAt: now },
});
return leased;
});
}
private async produce(attachment: Attachment): Promise<RenderedVariant[]> {
if (!attachment.stagingKey) {
throw new PermanentError('staging_missing', 'staging object was already removed');
}
let original: Buffer;
try {
original = await this.s3.getBuffer(attachment.stagingKey, this.options.maxBytes);
} catch (error) {
if (error instanceof ObjectMissingError) {
throw new PermanentError('staging_missing', 'staging object not found');
}
if (error instanceof ObjectTooLargeError) {
throw new PermanentError('invalid_image', 'staging object exceeds size limit');
}
throw error;
}
const sha256 = createHash('sha256').update(original).digest('hex');
if (original.length !== attachment.sizeBytes || sha256 !== attachment.sha256) {
throw new PermanentError('checksum_mismatch', 'staging object differs from accepted upload');
}
let variants: RenderedVariant[];
try {
variants = await tracer.startActiveSpan('attachment.render_variants', async (span) => {
span.setAttribute('messenger.image.bytes', original.length);
try {
return await renderVariants(
original,
this.options.maxPixels,
Math.ceil(this.options.timeoutMs / 1000),
);
} catch (error) {
span.setStatus({ code: SpanStatusCode.ERROR, message: error instanceof Error ? error.name : 'error' });
throw error;
} finally {
span.end();
}
});
} catch (error) {
if (error instanceof InvalidImageError) {
throw new PermanentError('invalid_image', 'image cannot be decoded');
}
throw error;
}
await this.s3.putBuffer(attachment.objectKey, original, attachment.contentType, sha256);
for (const variant of variants) {
await this.s3.putBuffer(
variantKey(attachment.id, variant),
variant.body,
variant.contentType,
variant.sha256,
);
}
return variants;
}
private async commit(
lease: AttachmentJob,
attachment: Attachment,
variants: RenderedVariant[],
): Promise<Outcome> {
const now = new Date();
const committed = await this.prisma.$transaction(async (tx) => {
const owned = await tx.attachmentJob.updateMany({
where: { id: lease.id, status: 'processing', leaseOwner: lease.leaseOwner },
data: {
status: 'ready',
leaseOwner: null,
leaseUntil: null,
lastError: null,
finishedAt: now,
},
});
if (owned.count === 0) {
return false;
}
await tx.attachmentVariant.createMany({
data: variants.map((variant) => ({
attachmentId: attachment.id,
pipelineVersion: PIPELINE_VERSION,
name: variant.name,
contentType: variant.contentType,
objectKey: variantKey(attachment.id, variant),
sizeBytes: variant.body.length,
width: variant.width,
height: variant.height,
sha256: variant.sha256,
createdAt: now,
})),
skipDuplicates: true,
});
await tx.attachment.update({
where: { id: attachment.id },
data: { status: 'ready', errorCode: null },
});
await tx.jobEvent.create({
data: { jobId: lease.id, attempt: lease.attempt, event: 'ready', createdAt: now },
});
return true;
});
if (!committed) {
this.logger.warn(`job ${lease.id}: lease lost before commit, result left to owner`);
return 'skipped';
}
timeToReady.record((now.getTime() - attachment.createdAt.getTime()) / 1000, {
'job.type': JOB_TYPE,
});
await this.hooks.afterCommit?.();
await this.removeStaging(attachment);
return 'ready';
}
private async recordFailure(lease: AttachmentJob, error: unknown): Promise<Outcome> {
const now = new Date();
const reason =
error instanceof PermanentError
? error.message
: error instanceof TimeoutError || error instanceof RenderTimeoutError
? 'processing timed out'
: `temporary failure: ${error instanceof Error ? error.name : 'unknown'}`;
this.logger.warn(`job ${lease.id} attempt ${lease.attempt}: ${reason}`);
return this.prisma.$transaction(async (tx) => {
const owned = await tx.attachmentJob.count({
where: { id: lease.id, status: 'processing', leaseOwner: lease.leaseOwner },
});
if (owned === 0) {
return 'skipped' as const;
}
if (error instanceof PermanentError) {
await markFailed(tx, lease, error.code, reason, now);
return 'failed' as const;
}
return retryOrFail(tx, lease, reason, now);
});
}
private async removeStaging(attachment: Attachment): Promise<void> {
if (!attachment.stagingKey) {
return;
}
try {
await this.s3.delete(attachment.stagingKey);
await this.prisma.attachment.update({
where: { id: attachment.id },
data: { stagingKey: null },
});
} catch {
this.logger.warn(`staging cleanup postponed for ${attachment.id}`);
}
}
private withTimeout<T>(work: Promise<T>): Promise<T> {
let timer: NodeJS.Timeout;
const deadline = new Promise<never>((_, reject) => {
timer = setTimeout(() => reject(new TimeoutError()), this.options.timeoutMs);
});
return Promise.race([work, deadline]).finally(() => clearTimeout(timer));
}
}
export function variantKey(
attachmentId: string,
variant: Pick<RenderedVariant, 'name' | 'extension'>,
): string {
return `attachments/${attachmentId}/v${PIPELINE_VERSION}/${variant.name}.${variant.extension}`;
}
+111
View File
@@ -0,0 +1,111 @@
import { createHash } from 'crypto';
import sharp, { Sharp } from 'sharp';
sharp.cache(false);
sharp.concurrency(1);
const WHITE = { r: 255, g: 255, b: 255 };
const JPEG_QUALITY = 82;
const WEBP_QUALITY = 80;
export interface RenderedVariant {
name: 'thumbnail' | 'crop' | 'optimized';
contentType: string;
extension: string;
body: Buffer;
width: number;
height: number;
sha256: string;
}
export class InvalidImageError extends Error {
constructor(reason: string) {
super(reason);
this.name = 'InvalidImageError';
}
}
export class RenderTimeoutError extends Error {
constructor() {
super('image processing exceeded its time limit');
this.name = 'RenderTimeoutError';
}
}
export function fitInside(
width: number,
height: number,
box: number,
): { width: number; height: number } {
if (width <= box && height <= box) {
return { width, height };
}
if (width >= height) {
return { width: box, height: Math.max(1, Math.floor((height * box) / width)) };
}
return { width: Math.max(1, Math.floor((width * box) / height)), height: box };
}
export async function renderVariants(
original: Buffer,
maxPixels: number,
timeoutSeconds = 0,
): Promise<RenderedVariant[]> {
const open = (): Sharp =>
sharp(original, { failOn: 'error', limitInputPixels: maxPixels })
.timeout({ seconds: timeoutSeconds })
.rotate();
try {
const meta = await sharp(original, { limitInputPixels: maxPixels }).metadata();
if (!meta.width || !meta.height) {
throw new InvalidImageError('image has no dimensions');
}
const swapped = (meta.orientation ?? 1) >= 5;
const width = swapped ? meta.height : meta.width;
const height = swapped ? meta.width : meta.height;
const thumb = fitInside(width, height, 256);
const large = fitInside(width, height, 1280);
const thumbnail = open()
.resize(thumb.width, thumb.height, { fit: 'fill' })
.flatten({ background: WHITE })
.jpeg({ quality: JPEG_QUALITY });
const crop = open()
.resize(128, 128, { fit: 'cover', position: 'centre' })
.flatten({ background: WHITE })
.jpeg({ quality: JPEG_QUALITY });
const optimized = open()
.resize(large.width, large.height, { fit: 'fill' })
.webp({ quality: WEBP_QUALITY, effort: 4 });
return [
await finish('thumbnail', 'image/jpeg', 'jpg', thumbnail),
await finish('crop', 'image/jpeg', 'jpg', crop),
await finish('optimized', 'image/webp', 'webp', optimized),
];
} catch (error) {
if (error instanceof InvalidImageError) {
throw error;
}
if (error instanceof Error && /timeout/i.test(error.message)) {
throw new RenderTimeoutError();
}
throw new InvalidImageError(error instanceof Error ? error.message : 'decode failed');
}
}
async function finish(
name: RenderedVariant['name'],
contentType: string,
extension: string,
pipeline: Sharp,
): Promise<RenderedVariant> {
const { data, info } = await pipeline.toBuffer({ resolveWithObject: true });
return {
name,
contentType,
extension,
body: data,
width: info.width,
height: info.height,
sha256: createHash('sha256').update(data).digest('hex'),
};
}
+39
View File
@@ -0,0 +1,39 @@
import './telemetry/register';
import { Logger } from '@nestjs/common';
import { NestFactory } from '@nestjs/core';
import {
FastifyAdapter,
NestFastifyApplication,
} from '@nestjs/platform-fastify';
import { AppModule } from './app.module';
import { configureApp } from './configure-app';
import { setupSwagger } from './docs/swagger';
import { enableGracefulShutdown } from './health/shutdown';
import { registerHttpTelemetry } from './telemetry/http';
import { JsonLogger } from './telemetry/json-logger';
async function bootstrap(): Promise<void> {
const app = await NestFactory.create<NestFastifyApplication>(
AppModule,
new FastifyAdapter(),
{
bufferLogs: true,
logger: new JsonLogger(),
},
);
configureApp(app);
registerHttpTelemetry(app.getHttpAdapter().getInstance());
setupSwagger(app);
enableGracefulShutdown(app);
const host = process.env.APP_HOST ?? '0.0.0.0';
const port = Number(process.env.APP_PORT ?? 8080);
await app.listen(port, host);
Logger.log(`Listening on http://${host}:${port}`, 'Bootstrap');
}
bootstrap().catch((error) => {
Logger.error('Fatal error during bootstrap', error);
process.exit(1);
});
+41
View File
@@ -0,0 +1,41 @@
import {
ArrayMaxSize,
IsArray,
IsString,
Validate,
ValidateIf,
ValidatorConstraint,
ValidatorConstraintInterface,
} from 'class-validator';
import { CodePointLength } from '../../common/dto/code-point-length.validator';
import { validate as isUuid } from '../../common/uuid';
@ValidatorConstraint({ name: 'uniqueUuidList' })
class UniqueUuidList implements ValidatorConstraintInterface {
validate(value: unknown): boolean {
if (
!Array.isArray(value) ||
!value.every((id) => typeof id === 'string' && isUuid(id))
) {
return false;
}
const normalized = (value as string[]).map((id) => id.toLowerCase());
return new Set(normalized).size === normalized.length;
}
defaultMessage(): string {
return 'attachment_ids must contain unique UUIDs';
}
}
export class CreateMessageDto {
@IsString()
@CodePointLength(0, 4000)
text!: string;
@ValidateIf((dto: CreateMessageDto) => dto.attachment_ids !== undefined)
@IsArray()
@ArrayMaxSize(10)
@Validate(UniqueUuidList)
attachment_ids?: string[];
}
+26
View File
@@ -0,0 +1,26 @@
import { Message } from '../../common/domain';
export interface MessageResponse {
id: string;
chat_id: string;
sender_id: string;
text: string;
created_at: string;
attachment_ids: string[];
}
export interface MessagePageResponse {
items: MessageResponse[];
next_cursor: string | null;
}
export function toMessageResponse(message: Message): MessageResponse {
return {
id: message.id,
chat_id: message.chatId,
sender_id: message.senderId,
text: message.text,
created_at: message.createdAt,
attachment_ids: message.attachmentIds,
};
}

Some files were not shown because too many files have changed in this diff Show More