forked from mtusi2026/mtusi2026lab5
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
0790599d2f | ||
|
|
2638e1c2c4 | ||
|
|
855ee27ea2 | ||
|
|
551cf70989 | ||
|
|
d19824f546 | ||
|
|
1982bdb690 |
+101
-9
@@ -1,25 +1,117 @@
|
||||
# Copy to .env for your LOCAL synthetic test environment; never commit .env.
|
||||
# These are logical interface suggestions; map actual framework names in REPORT.md.
|
||||
# Полный список переменных, которые читают приложение и стенд; значения-образцы
|
||||
# не являются секретами. Соответствие «переменная -> фактическая конфигурация»
|
||||
# описано в REPORT.md.
|
||||
#
|
||||
# Приложение (проверяется zod-схемой app/src/config/app-config.schema.ts):
|
||||
APP_HOST=0.0.0.0
|
||||
APP_PORT=8080
|
||||
APP_ENV=lab
|
||||
SESSION_TTL_SECONDS=3600
|
||||
# Choose credentials locally. File paths contain no secret values.
|
||||
DATABASE_URL_FILE=/run/secrets/database_url
|
||||
# Optional on 3, required use on 4 in lab2; Redis OR Valkey:
|
||||
SESSION_STORE=postgres
|
||||
CACHE_URL_FILE=/run/secrets/cache_url
|
||||
API_REPLICAS=2
|
||||
DATABASE_URL=postgresql://messenger:change-me@postgres:5432/messenger
|
||||
REDIS_URL=redis://redis:6379
|
||||
DATABASE_POOL_MAX=10
|
||||
DATABASE_CONNECTION_TIMEOUT_MS=5000
|
||||
DATABASE_STATEMENT_TIMEOUT_MS=5000
|
||||
DATABASE_IDLE_TX_TIMEOUT_MS=30000
|
||||
CACHE_TTL_SECONDS=30
|
||||
CACHE_MODE=protected
|
||||
CACHE_LOCK_TTL_MS=1000
|
||||
CACHE_MISS_DELAY_MS=0
|
||||
# Бюджет проверки зависимостей в /health/ready: LB пробует ту же ручку
|
||||
# (timeout check 500 ms), поэтому проба обязана уложиться заметно раньше.
|
||||
HEALTH_TIMEOUT_MS=400
|
||||
# Токен доступа к /internal/metrics; сгенерируйте свой.
|
||||
METRICS_TOKEN=change-me
|
||||
# Дрейн перед выходом по SIGTERM: реплика перестаёт проходить пробы LB,
|
||||
# ждёт DRAIN_DELAY_MS (длиннее обнаружения отказа ~1 s) и только потом закрывается.
|
||||
DRAIN_DELAY_MS=3000
|
||||
# Переопределяет X-Instance-Id; по умолчанию это hostname контейнера
|
||||
# (= короткий ID контейнера, по нему работает scripts/stop-one.sh).
|
||||
# INSTANCE_ID=api-1
|
||||
IDEMPOTENCY_TTL_SECONDS=86400
|
||||
|
||||
# Стенд (compose.yaml и scripts/):
|
||||
API_REPLICAS=2
|
||||
API_CPUS_PER_REPLICA=1.5
|
||||
API_MEMORY_PER_REPLICA=1024m
|
||||
PG_CPUS=1.5
|
||||
PG_MEMORY=512m
|
||||
REDIS_CPUS=0.25
|
||||
REDIS_MEMORY=128m
|
||||
LB_CPUS=0.25
|
||||
LB_MEMORY=128m
|
||||
# Тег образа API; BUILD_ID отличает сборки при rolling update.
|
||||
API_IMAGE=mtusi-messenger:lab5
|
||||
BUILD_ID=local
|
||||
APP_PUBLIC_PORT=8080
|
||||
LB_STATS_PORT=8404
|
||||
# Loopback-порты PostgreSQL/Redis нужны только операторским проверкам с хоста
|
||||
# (make test-e2e, psql). API-реплики портов не публикуют.
|
||||
POSTGRES_PUBLISHED_PORT=5432
|
||||
REDIS_PUBLISHED_PORT=6379
|
||||
|
||||
# PostgreSQL (пароль подставьте свой; тот же, что в DATABASE_URL):
|
||||
POSTGRES_DB=messenger
|
||||
POSTGRES_USER=messenger
|
||||
POSTGRES_PASSWORD=change-me
|
||||
|
||||
# Объектное хранилище MinIO. Credentials — файлы ./secrets/* (scripts/init-env.sh), не здесь.
|
||||
S3_ENDPOINT=http://s3:9000
|
||||
S3_PUBLIC_ENDPOINT=http://localhost:9000
|
||||
S3_REGION=us-east-1
|
||||
S3_BUCKET=messenger-attachments
|
||||
S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id
|
||||
S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key
|
||||
S3_PUBLIC_BIND=127.0.0.1
|
||||
S3_PUBLIC_PORT=9000
|
||||
S3_CPUS=1.0
|
||||
S3_MEMORY=512m
|
||||
S3_CONNECT_TIMEOUT_MS=1000
|
||||
S3_REQUEST_TIMEOUT_MS=15000
|
||||
MAX_UPLOAD_BYTES=10485760
|
||||
MAX_IMAGE_PIXELS=40000000
|
||||
# Каждая загрузка держит копию файла (≤ MAX_UPLOAD_BYTES) до полного декода;
|
||||
# худший декод (40 Мп RGBA16 interlaced PNG) ≈ 340 MiB. Граница памяти реплики:
|
||||
# базовый RSS (~250 MiB) + UPLOAD_MAX_IN_FLIGHT × 10 MiB + IMAGE_DECODE_MAX_CONCURRENT × 340 MiB
|
||||
# = 250 + 160 + 340 ≈ 750 MiB при API_MEMORY_PER_REPLICA=1024m.
|
||||
UPLOAD_MAX_IN_FLIGHT=16
|
||||
# Не больше стольких одновременных загрузок одного пользователя на реплику.
|
||||
UPLOAD_MAX_IN_FLIGHT_PER_USER=4
|
||||
IMAGE_DECODE_MAX_CONCURRENT=1
|
||||
DIRECT_UPLOAD_MAX_PENDING_PER_USER=10
|
||||
# UPLOAD_TIMEOUT_MS, DIRECT_UPLOAD_URL_TTL_SECONDS, ATTACHMENT_SWEEP_GRACE_SECONDS,
|
||||
# ATTACHMENT_SWEEP_INTERVAL_MS задаются в compose.yaml (environment) с умолчаниями
|
||||
# и переопределяются из shell только для экспериментов.
|
||||
|
||||
# lab5: очередь и асинхронная обработка изображений. URL брокера берётся из
|
||||
# файла секрета (scripts/init-env.sh), байты и credentials в сообщения не
|
||||
# попадают: только job_id/attachment_id/pipeline_version/attempt.
|
||||
BROKER_URL_FILE=/run/secrets/broker_url
|
||||
BROKER_DELIVERY_LIMIT=10
|
||||
BROKER_PUBLISH_TIMEOUT_MS=5000
|
||||
# Сколько заданий worker обрабатывает параллельно; prefetch равен этому числу.
|
||||
WORKER_CONCURRENCY=2
|
||||
JOB_MAX_ATTEMPTS=5
|
||||
JOB_TIMEOUT_SECONDS=60
|
||||
JOB_LEASE_SECONDS=30
|
||||
JOB_RETRY_BASE_MS=1000
|
||||
JOB_RETRY_MAX_MS=30000
|
||||
PIPELINE_VERSION=1
|
||||
OUTBOX_POLL_INTERVAL_MS=500
|
||||
LEASE_REAPER_INTERVAL_MS=2000
|
||||
# 0 выключает фоновую уборку staging (ручной запуск остаётся).
|
||||
STAGING_CLEANUP_INTERVAL_MS=60000
|
||||
# Сколько staging-объект неуспешной загрузки живёт до удаления.
|
||||
STAGING_RETENTION_SECONDS=86400
|
||||
WORKER_HEALTH_PORT=8081
|
||||
QUEUED_RECONCILE_GRACE_SECONDS=15
|
||||
OUTBOX_RETENTION_SECONDS=86400
|
||||
JOB_EVENTS_RETENTION_SECONDS=2592000
|
||||
# 1 включает чтение fault:<point> из Redis; в обычном стенде — 0.
|
||||
FAULT_INJECTION_ENABLED=0
|
||||
WORKER_REPLICAS=1
|
||||
WORKER_CPUS=1.5
|
||||
# Бюджет worker: база ≈250 MiB + WORKER_CONCURRENCY × (10 MiB исходник +
|
||||
# ≈340 MiB худший декод 40 Мп) ≈ 950 MiB при concurrency 2; лимит 1280m.
|
||||
WORKER_MEMORY=1280m
|
||||
BROKER_CPUS=0.5
|
||||
BROKER_MEMORY=512m
|
||||
|
||||
@@ -1,18 +1,35 @@
|
||||
## Лабораторная 5
|
||||
|
||||
Автор / группа: TODO
|
||||
Целевая оценка (3/4/5): TODO
|
||||
Трек 5, если заявлен: TODO
|
||||
Автор / группа: Илья Тюнин, БВТ2303
|
||||
Целевая оценка (3/4/5): 5
|
||||
Трек 5, если заявлен: трек 2 «Доказанная идемпотентность pipeline»
|
||||
|
||||
Что реализовано и что перенесено: TODO
|
||||
Команды запуска из чистого clone: TODO
|
||||
Результаты проверок и ссылка на REPORT.md: TODO
|
||||
Ограничения / известные ошибки: TODO
|
||||
Источники и использование ИИ: TODO
|
||||
Что реализовано и что перенесено: upload переведён на 202 `queued`; байты
|
||||
принимаются в S3 `staging/*`, задание и outbox коммитятся в PostgreSQL одной
|
||||
транзакцией. Добавлены RabbitMQ 4.1 (quorum queue, DLX/DLQ), отдельный worker
|
||||
с lease/heartbeat, retry с equal jitter, идемпотентными вариантами и DLQ,
|
||||
операторский CLI (`status`, `failed`, `replay`, `reprocess`, `queues`,
|
||||
`publish-raw`), версии pipeline v1/v2 и fault injection через Redis. Из lab4
|
||||
сохранены API, auth, чаты, сообщения, direct upload, кэш, drain, sweeper и
|
||||
resilience hooks.
|
||||
|
||||
- [ ] Весь обязательный минимум текущей и предыдущих работ сохранён с объявленными переходами.
|
||||
- [ ] Контракт и публичные тесты не ослаблены.
|
||||
- [ ] make check и make test выполнены; приложен реальный вывод.
|
||||
- [ ] Сценарии отказов/безопасности своего уровня воспроизведены.
|
||||
- [ ] Все секреты и приватные ключи находятся вне Git; отчёт не содержит tokens.
|
||||
- [ ] README задания сохранён, REPORT.md содержит инструкцию именно моего решения.
|
||||
Команды запуска из чистого clone: `make init-env`, `make stand-up`,
|
||||
`make wait-ready`, `make test`, `make test-e2e`, `make test-dependencies`,
|
||||
`make lab5-demo`; отказы трека 2 — стенд с
|
||||
`FAULT_INJECTION_ENABLED=1 JOB_TIMEOUT_SECONDS=10 JOB_LEASE_SECONDS=8 JOB_RETRY_BASE_MS=500 JOB_RETRY_MAX_MS=4000`
|
||||
и `make lab5-faults` / `make lab5-track2`.
|
||||
|
||||
Результаты проверок и ссылка на REPORT.md: `REPORT.md`,
|
||||
`evidence/lab5/summary.md`; smoke, e2e, dependency, persistence/failover,
|
||||
worker-down demo и fault-сценарии.
|
||||
|
||||
Ограничения / известные ошибки: single-node PostgreSQL/RabbitMQ/MinIO;
|
||||
fault points только внутри процесса; S3-запись может повториться, но остаётся
|
||||
побайтно детерминированной; track 1 не заявлен.
|
||||
|
||||
- [x] Весь обязательный минимум текущей и предыдущих работ сохранён с объявленными переходами.
|
||||
- [x] Контракт и публичные тесты не ослаблены.
|
||||
- [x] make check и make test выполнены; приложен реальный вывод.
|
||||
- [x] Сценарии отказов/безопасности своего уровня воспроизведены.
|
||||
- [x] Все секреты и приватные ключи находятся вне Git; отчёт не содержит tokens.
|
||||
- [x] README задания сохранён, REPORT.md содержит инструкцию именно моего решения.
|
||||
|
||||
+38
@@ -0,0 +1,38 @@
|
||||
FROM node:25.7.0-slim AS build
|
||||
|
||||
WORKDIR /usr/src/app
|
||||
|
||||
RUN apt-get update \
|
||||
&& apt-get install -y --no-install-recommends python3 make g++ \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
COPY app/package.json app/package-lock.json ./
|
||||
RUN npm ci
|
||||
|
||||
COPY app/ ./
|
||||
RUN npm run build && npm prune --omit=dev
|
||||
|
||||
FROM node:25.7.0-slim AS runtime
|
||||
|
||||
WORKDIR /usr/src/app
|
||||
|
||||
# BUILD_ID доказывается через GET /internal/metrics и меняется при rolling
|
||||
# update: он показывает, какая именно сборка отвечает на запрос.
|
||||
ARG BUILD_ID=unknown
|
||||
ENV BUILD_ID=$BUILD_ID
|
||||
|
||||
ENV NODE_ENV=production
|
||||
ENV APP_PORT=8080
|
||||
ENV DRAIN_DELAY_MS=5000
|
||||
COPY --from=build /usr/src/app/package.json /usr/src/app/package-lock.json ./
|
||||
COPY --from=build /usr/src/app/node_modules ./node_modules
|
||||
COPY --from=build /usr/src/app/dist ./dist
|
||||
|
||||
USER node
|
||||
|
||||
EXPOSE 8080
|
||||
|
||||
HEALTHCHECK --interval=10s --timeout=3s --start-period=10s --retries=3 \
|
||||
CMD node -e "const port=process.env.APP_PORT||'8080'; fetch('http://127.0.0.1:'+port+'/health/ready').then((response) => { if (!response.ok) process.exit(1); }).catch(() => process.exit(1))"
|
||||
|
||||
CMD ["node", "dist/main.js"]
|
||||
@@ -4,10 +4,10 @@ CA_FILE ?=
|
||||
ACTION_SCRIPT ?=
|
||||
TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",)
|
||||
|
||||
.PHONY: help check validate test load test-persistence test-failover
|
||||
.PHONY: help check validate test load test-persistence test-failover test-dependencies stand-up stand-down stand-status wait-ready init-env test-e2e s3-security consistency bench lab5-demo lab5-faults lab5-track2 jobs
|
||||
|
||||
help:
|
||||
@echo "check: offline structure; validate: full OpenAPI (requirements-dev.txt); test: running API; load: synthetic writes"
|
||||
@echo "check: offline structure; validate: full OpenAPI (requirements-dev.txt); test: running API; load: synthetic writes; init-env: .env + secrets; test-e2e: app e2e; s3-security; consistency; bench; lab5-demo; lab5-faults; lab5-track2; jobs ARGS=..."
|
||||
|
||||
check:
|
||||
$(PYTHON) scripts/check_contract.py
|
||||
@@ -28,3 +28,54 @@ test-persistence:
|
||||
test-failover:
|
||||
@test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable stop-one hook"; exit 2)
|
||||
$(PYTHON) tests/resilience.py --mode failover --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS)
|
||||
|
||||
COMPOSE_ENV_FILE ?= .env
|
||||
COMPOSE = docker compose --env-file $(COMPOSE_ENV_FILE)
|
||||
|
||||
test-dependencies:
|
||||
cd app && E2E_COMPOSE_DIR="$(CURDIR)" E2E_COMPOSE_ENV_FILE="$(COMPOSE_ENV_FILE)" npm run test:dependencies
|
||||
|
||||
stand-up:
|
||||
$(COMPOSE) up -d --build --wait
|
||||
|
||||
stand-down:
|
||||
$(COMPOSE) down
|
||||
|
||||
stand-status:
|
||||
$(COMPOSE) ps
|
||||
|
||||
wait-ready:
|
||||
@for attempt in $$(seq 1 60); do \
|
||||
if curl -fsS "$(BASE_URL)/health/ready" >/dev/null 2>&1; then \
|
||||
echo "stand is ready ($(BASE_URL))"; exit 0; \
|
||||
fi; \
|
||||
sleep 1; \
|
||||
done; \
|
||||
echo "stand did not become ready within 60s" >&2; exit 1
|
||||
|
||||
init-env:
|
||||
./scripts/init-env.sh
|
||||
|
||||
test-e2e:
|
||||
scripts/app-e2e.sh
|
||||
|
||||
s3-security:
|
||||
scripts/lab4/check_s3_security.sh
|
||||
|
||||
consistency:
|
||||
scripts/lab4/consistency_check.sh
|
||||
|
||||
bench:
|
||||
scripts/lab4/run_matrix.sh
|
||||
|
||||
lab5-demo:
|
||||
$(PYTHON) scripts/lab5/worker_down_demo.py --base-url "$(BASE_URL)" $(TLS_ARGS)
|
||||
|
||||
lab5-faults:
|
||||
$(PYTHON) scripts/lab5/fault_suite.py --base-url "$(BASE_URL)" --scenario level4 --output evidence/lab5/faults-level4.json $(TLS_ARGS)
|
||||
|
||||
lab5-track2:
|
||||
$(PYTHON) scripts/lab5/fault_suite.py --base-url "$(BASE_URL)" --scenario track2 --repeat 3 --output evidence/lab5/track2.json $(TLS_ARGS)
|
||||
|
||||
jobs:
|
||||
docker compose --env-file $(COMPOSE_ENV_FILE) exec -T api node dist/jobs/jobs.cli.js $(ARGS)
|
||||
|
||||
@@ -1,37 +1,384 @@
|
||||
# Отчёт — лабораторная 5
|
||||
|
||||
- Автор, группа: TODO
|
||||
- Целевая оценка; выбранный трек 5, если нужен: TODO
|
||||
- Commit/PR: TODO после публикации
|
||||
- Использованные источники, библиотеки и помощь ИИ: TODO
|
||||
- Автор, группа: Илья Тюнин, БВТ2303
|
||||
- Целевая оценка: 5, трек 2 «Доказанная идемпотентность pipeline»
|
||||
- Commit/PR: ветка `lab5`, коммиты `lab5: ...`; PR — после публикации
|
||||
- Источники: [RabbitMQ reliability](https://www.rabbitmq.com/docs/reliability), [RabbitMQ confirms](https://www.rabbitmq.com/docs/confirms), `contracts/jobs.schema.json`, `contracts/attachment-states.md`, amqplib 2.2.1, sharp 0.35.5. ИИ использован для черновика плана, ревью кода и текста отчёта; решения и проверки выполнены и воспроизведены локально.
|
||||
|
||||
## Запуск из чистого clone
|
||||
|
||||
TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных.
|
||||
Требования: Docker 29.x с Compose v2, Python 3.9+, Node 25.7.0 для локальных
|
||||
юнит/e2e-прогонов. Образы приложения собираются из этого репозитория.
|
||||
|
||||
```sh
|
||||
make init-env # .env (если нет) + secrets/*; значения не печатаются
|
||||
make check # offline-структура и OpenAPI
|
||||
make stand-up # build + PostgreSQL, Redis, MinIO, RabbitMQ, API x2, worker, HAProxy
|
||||
make wait-ready # /health/ready
|
||||
make test # публичный smoke: 202 -> ready -> thumbnail 256x160
|
||||
make test-e2e # e2e приложения против стенда
|
||||
make test-dependencies # отказы Redis/PostgreSQL/S3, пауза S3, saturation
|
||||
make test-persistence ACTION_SCRIPT=scripts/restart.sh
|
||||
scripts/start-replica.sh
|
||||
make test-failover ACTION_SCRIPT=scripts/stop-one.sh
|
||||
make lab5-demo # worker остановлен -> upload 202 -> рестарт broker/API -> ready
|
||||
```
|
||||
|
||||
Проверка трека 2 требует включённого fault injection и укороченных таймаутов:
|
||||
|
||||
```sh
|
||||
FAULT_INJECTION_ENABLED=1 JOB_TIMEOUT_SECONDS=10 JOB_LEASE_SECONDS=8 \
|
||||
JOB_RETRY_BASE_MS=500 JOB_RETRY_MAX_MS=4000 make stand-up
|
||||
make lab5-faults # level4 -> evidence/lab5/faults-level4.json
|
||||
make lab5-track2 # track2 x3 -> evidence/lab5/track2.json
|
||||
make stand-up # вернуть штатные значения
|
||||
```
|
||||
|
||||
Остановка: `make stand-down`. Намеренное удаление учебных данных (необратимо):
|
||||
`docker compose --env-file .env down -v` — удаляет PostgreSQL, Redis, MinIO и
|
||||
RabbitMQ volumes. Секреты лежат в `secrets/` и в Git не попадают.
|
||||
|
||||
## Архитектура и решения
|
||||
|
||||
TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab4, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации.
|
||||
```mermaid
|
||||
flowchart LR
|
||||
C[client] --> H[HAProxy]
|
||||
H --> A1[API 1]
|
||||
H --> A2[API 2]
|
||||
A1 --> PG[(PostgreSQL)]
|
||||
A2 --> PG
|
||||
A1 --> S3S[(S3 staging/*)]
|
||||
A1 --> O[(job_outbox)]
|
||||
O --> R[Outbox relay]
|
||||
R --> MQ[(RabbitMQ quorum queue)]
|
||||
MQ --> W[worker]
|
||||
W --> S3F[(S3 attachments/*)]
|
||||
W --> PG
|
||||
A1 --> RDS[(Redis: sessions, cache, faults)]
|
||||
```
|
||||
|
||||
Источник истины — PostgreSQL. Брокер только доставляет сообщение; S3 хранит
|
||||
байты. API и worker — разные процессы, worker не импортирует HTTP-модули.
|
||||
|
||||
### Состояния
|
||||
|
||||
| `attachments.status` | Значения |
|
||||
| --- | --- |
|
||||
| публичные | `queued`, `processing`, `ready`, `failed` |
|
||||
| внутренние | `pending` (идёт приём), `deleting`, `rejected` (lab4) |
|
||||
|
||||
| `attachment_jobs.status` | Значения |
|
||||
| --- | --- |
|
||||
| | `queued`, `processing`, `succeeded`, `failed`, `superseded` |
|
||||
|
||||
Правила переходов выполняются в транзакции, меняющей job:
|
||||
|
||||
- claim: `queued -> processing` только при `attachments.status='queued'`;
|
||||
- success: `ready`, если `coalesce(ready_pipeline_version,0) <= pipeline_version`,
|
||||
иначе job становится `superseded`, а строка вложения не меняется;
|
||||
- retry: `processing -> queued` только при `attachments.status='processing'`;
|
||||
- final failure: `failed` + `error_code`, только если вложение не `ready`;
|
||||
- replay: `failed -> queued`;
|
||||
- `ready` никогда не откатывается; старая job не перезаписывает новый результат.
|
||||
|
||||
### Что уже сохранено в момент 202
|
||||
|
||||
До ответа 202 выполняются и фиксируются одним COMMIT:
|
||||
|
||||
1. все байты приняты в `staging/<id>/original`; размер и SHA-256 сверены,
|
||||
`ImageInspector` и полный decode проверили контейнер и пиксели;
|
||||
2. `attachments` — строка `queued` с `size_bytes`, `sha256`, `width`, `height`;
|
||||
3. `attachment_jobs` — логическая работа `(attachment_id, pipeline_version)`,
|
||||
`attempt=1`, `max_attempts`;
|
||||
4. `job_outbox` — намерение опубликовать сообщение.
|
||||
|
||||
Только после COMMIT вызывается fault point `api.after_commit_before_publish` и
|
||||
relay. Брокер не входит в `/health/ready`: outbox позволяет принимать загрузки
|
||||
при недоступном RabbitMQ, а relay опубликует задание после восстановления.
|
||||
|
||||
### Окна отказа
|
||||
|
||||
| Окно из contract | Механизм | Сценарий | Доказательство |
|
||||
| --- | --- | --- | --- |
|
||||
| после staging до записи задания | строка `pending` + `expires_at`; sweeper удаляет staging и строку | lab4 sweep + `make test-dependencies` | `make-test-dependencies.txt` |
|
||||
| после COMMIT до publish | transactional outbox, relay повторяет непомеченные строки | `crash_after_commit_before_publish` | `faults-level4.json` |
|
||||
| после publish до confirm | ConfirmChannel; без confirm транзакция relay откатывается | `relay_crash_after_publish` | `track2.json` |
|
||||
| после записи S3 до COMMIT результата | lease истекает, reaper возвращает job в `queued`; повторный рендер детерминирован | `crash_after_side_effect` | `track2.json` |
|
||||
| после COMMIT результата до ack | redelivery видит `succeeded` и отвечает `duplicate_skipped` | `crash_after_commit_before_ack` | `track2.json` |
|
||||
|
||||
### Publisher confirm и consumer ack
|
||||
|
||||
- **Publisher confirm** — брокер принял persistent-сообщение. Outbox помечает
|
||||
строку опубликованной только после confirm.
|
||||
- **Consumer ack** — worker устойчиво зафиксировал эффект в PostgreSQL и S3.
|
||||
Ack после commit; падение между commit и ack даёт redelivery и no-op.
|
||||
|
||||
Это at-least-once доставка. Ключ эффекта — `(attachment_id, pipeline_version,
|
||||
variant)` (PK `attachment_variants`); логическая работа уникальна по
|
||||
`(attachment_id, pipeline_version)`. Повторная доставка не создаёт второй набор
|
||||
вариантов. `attempt` увеличивается только управляемым retry/replay/истечением
|
||||
lease, но не redelivery брокера.
|
||||
|
||||
### Зависший worker и долгая работа
|
||||
|
||||
Claim ставит `lease_owner`, `lease_token`, `lease_until`. Heartbeat продлевает
|
||||
lease каждые `JOB_LEASE_SECONDS/3`; 0 обновлённых строк означает потерю lease и
|
||||
аборт job. `JOB_TIMEOUT_SECONDS` — жёсткий дедлайн: `AbortController` плюс
|
||||
`sharp(...).timeout({seconds})` дают transient `processing_timeout`. Reaper
|
||||
возвращает job с истёкшим lease в очередь, поэтому убитый worker не оставляет
|
||||
вечный `processing`.
|
||||
|
||||
### DLQ, failed registry, replay
|
||||
|
||||
- **DLQ** (`attachments.process.dlq`): невалидный JSON/схема и сообщение,
|
||||
не совпадающее с job (`mismatch`). Это не очередь повторных попыток.
|
||||
- **Failed registry**: `cli failed` — jobs в `failed` с `last_error_code`.
|
||||
- **Replay** (`cli replay`) разрешён только для failed job, при отсутствии
|
||||
активной job, и отказывает `already_ready`, `no_failed_job`, `job_active`,
|
||||
`source_expired`. Новая попытка защищена тем же ключом эффекта; успешный
|
||||
результат не исполняется повторно.
|
||||
|
||||
### Staging retention
|
||||
|
||||
Worker удаляет `staging/<id>/original` после успешного commit. Остатки убирает
|
||||
`StagingCleaner`: для `ready` — через минуту, для `failed` — через
|
||||
`STAGING_RETENTION_SECONDS` (24 ч). Pending/rejected загрузки удаляет lab4
|
||||
sweeper по `expires_at + grace`. Текущую работу cleaner не трогает.
|
||||
|
||||
### Качество вариантов и метаданные
|
||||
|
||||
| Вариант | v1 | v2 |
|
||||
| --- | --- | --- |
|
||||
| thumbnail | JPEG q80, fit 256×256, без апскейла, alpha на белом | JPEG q90 progressive mozjpeg |
|
||||
| crop | JPEG q80, 128×128 cover centre (апскейл разрешён) | JPEG q90 progressive mozjpeg |
|
||||
| optimized | WebP q75 effort 4, fit 1280×1280 | WebP q82 effort 5 smartSubsample |
|
||||
|
||||
EXIF orientation применяется `.rotate()`; размеры берутся из
|
||||
`metadata().autoOrient`. sharp по умолчанию удаляет EXIF/ICC (нет
|
||||
`withMetadata`). `fitInside` использует целочисленную арифметику: 320×200 →
|
||||
256×160, 1000×335 → 256×85, 10000×1 → 256×1.
|
||||
|
||||
### Память worker
|
||||
|
||||
База ≈250 MiB + `WORKER_CONCURRENCY × (10 MiB исходник + ≈340 MiB худший декод
|
||||
40 Мп)` ≈ 950 MiB при concurrency 2. Лимит `WORKER_MEMORY=1280m`.
|
||||
`big_image` загружает два `worst_decode_png` (8000×5000) параллельно: измеренный
|
||||
пик — **992.2 MiB / 1.25 GiB** (в предыдущих прогонах 934–966 MiB), `OOMKilled=false`, оба вложения дошли до ready.
|
||||
Запас ~300 MiB; при росте concurrency лимит нужно поднимать вместе с бюджетом.
|
||||
|
||||
### Конфигурация
|
||||
|
||||
| Переменная | Фактическое значение / роль |
|
||||
| --- | --- |
|
||||
| `BROKER_URL_FILE` | `/run/secrets/broker_url`, amqp URL с паролем из secret |
|
||||
| `BROKER_DELIVERY_LIMIT` | 10, `x-delivery-limit` quorum-очереди |
|
||||
| `WORKER_CONCURRENCY` | 2, prefetch и число параллельных job |
|
||||
| `JOB_MAX_ATTEMPTS` | 5, ограниченные retry |
|
||||
| `JOB_TIMEOUT_SECONDS` | 60, жёсткий дедлайн рендера |
|
||||
| `JOB_LEASE_SECONDS` | 30, lease + heartbeat/3 |
|
||||
| `JOB_RETRY_BASE_MS` / `JOB_RETRY_MAX_MS` | 1000 / 30000, equal jitter |
|
||||
| `PIPELINE_VERSION` | 1, версия для новых upload |
|
||||
| `OUTBOX_POLL_INTERVAL_MS` | 500, период relay |
|
||||
| `LEASE_REAPER_INTERVAL_MS` | 2000 |
|
||||
| `STAGING_CLEANUP_INTERVAL_MS` | 60000, 0 отключает |
|
||||
| `STAGING_RETENTION_SECONDS` | 86400 |
|
||||
| `WORKER_HEALTH_PORT` | 8081 |
|
||||
| `FAULT_INJECTION_ENABLED` | 0 в штатном стенде, 1 для suite |
|
||||
| `WORKER_REPLICAS` / `WORKER_CPUS` / `WORKER_MEMORY` | 1 / 1.5 / 1280m |
|
||||
| `BROKER_CPUS` / `BROKER_MEMORY` | 0.5 / 512m |
|
||||
|
||||
Остальные переменные lab4 (`DATABASE_URL`, `REDIS_URL`, S3, лимиты загрузки,
|
||||
таймауты) сохранены; `S3_ACCESS_KEY_ID_FILE`/`S3_SECRET_ACCESS_KEY_FILE`
|
||||
подменяются в worker на отдельную identity.
|
||||
|
||||
### Роли инфраструктуры
|
||||
|
||||
| Роль | Service | Образ | Готовность | Volume | Credentials |
|
||||
| --- | --- | --- | --- | --- | --- |
|
||||
| API | `api` | сборка `mtusi-messenger:lab5` | `/health/ready` (PG/Redis/S3) | — | `s3_api_*`, `broker_url` |
|
||||
| Worker | `worker` | та же сборка, `dist/worker.main.js` | `127.0.0.1:8081/health` | — | `s3_worker_*`, `broker_url` |
|
||||
| Broker | `broker` | `rabbitmq:4.1.8-alpine@sha256:153322e3974e6365c50e523e5eadd5a7fae231f0e8c58f67c03bb3e02056e34e` | `rabbitmq-diagnostics check_port_connectivity` | `broker-data` | `rabbitmq.conf` |
|
||||
| PostgreSQL | `postgres` | `postgres:16.4-alpine` | `pg_isready` | `postgres-data` | `.env` / `DATABASE_URL` |
|
||||
| Redis | `redis` | `redis:7.4.2-alpine` | `redis-cli ping` | `redis-data` | только compose-сеть |
|
||||
| S3 | `s3` | `minio/minio:RELEASE.2025-09-07T16-13-09Z@sha256:14cea493d9a34af32f524e538b8346cf79f3321eff8e708c1e2960462bd8936e` | `mc ready local` | `s3-data` | root secrets |
|
||||
| S3 bootstrap | `s3-init` | `minio/mc:...@sha256:a7fe349ef4bd8521fb8497f55c6042871b2ae640607cf99d9bede5e9bdf11727` | exit 0 | — | root + API/worker |
|
||||
| LB | `haproxy` | `haproxy:3.2-alpine` | depends on api healthy | — | — |
|
||||
|
||||
### Изменения относительно lab4
|
||||
|
||||
- upload 201 → 202 `queued`; metadata отдаёт `queued/processing/ready/failed`,
|
||||
`error_code` и `variants`;
|
||||
- байты пишутся в `staging/*`, worker переносит оригинал в `attachments/*`;
|
||||
- добавлены `attachment_jobs`, `attachment_variants`, `job_outbox`, `job_events`;
|
||||
- добавлены broker, worker, отдельные S3-identity и политики;
|
||||
- `openContent` отдаёт `original/thumbnail/crop/optimized` и 409 до ready;
|
||||
- сообщения можно привязывать к `queued/processing` вложениям, failed даёт 409;
|
||||
- сохранены lab4-механизмы: idempotency, cache, drain, sweeper, direct upload,
|
||||
failover/persistence hooks.
|
||||
|
||||
## Доказательства критериев
|
||||
|
||||
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
|
||||
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл |
|
||||
| --- | --- | --- | --- |
|
||||
| Публичный smoke | make test | TODO | TODO |
|
||||
| Остальные пункты уровня 3 | TODO | TODO | TODO |
|
||||
| Все пункты уровня 4, если заявлен | TODO | TODO | TODO |
|
||||
| Выбранный трек 5, если заявлен | TODO | TODO | TODO |
|
||||
|
||||
TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены.
|
||||
| Сборка и lint | `npm run build && npm run lint` | без ошибок; `dist/worker.main.js` и `dist/jobs/jobs.cli.js` | `build-lint.txt` |
|
||||
| Публичный smoke | `make test` | `PASS: lab5 public API scenario`; 202 → ready → thumbnail 256×160 JPEG, sha совпал | `evidence/lab5/make-test.txt` |
|
||||
| «3»: API 202 только после устойчивого приёма | `make test`, `make lab5-demo` | staging + attachment + job + outbox в одном COMMIT; worker остановлен, upload принят | `make-test.txt`, `worker-down-demo.txt` |
|
||||
| «3»: отдельные broker и worker | `make lab5-demo`, `docker compose ps` | broker/worker healthy; worker не в API-процессе | `compose-ps.txt`, `worker-down-demo.txt` |
|
||||
| «3»: состояния и thumbnail | `make test`, `make test-e2e` | metadata `queued/processing/ready`; thumbnail 256×160; original неизменен | `make-test.txt`, `make-test-e2e.txt` |
|
||||
| «3»: ограниченный retry и failed | `retry_and_replay` | 4 `retry_scheduled`, затем `failed`/`retry_exhausted`; replay → ready | `faults-level4.json` |
|
||||
| «3»: принятое задание переживает рестарт broker | `make lab5-demo` | quorum-очередь сохранила сообщение после restart broker+API | `worker-down-demo.txt`, `queues.txt` |
|
||||
| «4»: crop/optimized, качество, EXIF | `make test`, unit `pipelines.spec.ts` | 128×128 JPEG, WebP 320×200, EXIF 6 → 170×256, прозрачность на белом | `unit-tests.txt`, `make-test.txt` |
|
||||
| «4»: outbox, crash COMMIT→publish | `crash_after_commit_before_publish` | attachment готов без ручного создания job; один `created` | `faults-level4.json` |
|
||||
| «4»: backoff/jitter, permanent/transient, DLQ/failed/replay | `retry_and_replay`, `poison` | transient exhausted, DLQ +2, replay с audit | `faults-level4.json` |
|
||||
| «4»: crash после S3 до ack | `crash_after_side_effect` | attempt 2, lease_expired, варианты байт-идентичны baseline | `faults-level4.json`, `track2.json` |
|
||||
| «4»: timeout/lease, concurrency, очистка staging | `hung_worker`, `make lab5-demo`, CLI status | `processing_timeout` retry; `staging_present=false` после ready | `faults-level4.json`, `queues.txt` |
|
||||
| «4»: память worker на 40 Мп | `big_image` | 2 × 40 Мп параллельно, пик 934–992 MiB, OOMKilled=false | `big-image-memory.txt`, `faults-level4.json` |
|
||||
| «5» трек 2: версионирование, дубль, reorder, crash, stale | `make lab5-track2` | все инварианты I1–I6 истинны на 3 повторах | `track2.json` |
|
||||
| «5» трек 2: отсутствие потери и перезаписи | `stale_version` | ready_pipeline_version=2, v1 `superseded`, v2 etag не изменился | `track2.json` |
|
||||
| «5» трек 2: не exactly-once доставка | отчёт, `duplicate` | 5 duplicate publish дали 5 `duplicate_skipped`, S3 etag не изменился | `track2.json` |
|
||||
| Регрессия ревью: потерянное сообщение | purge очереди после publish | reconciler вернул job, ready за 5.1 s, событие `reconciled` | `reconciler.txt` |
|
||||
| Регрессия ревью: рестарт broker | restart broker при живом worker | consumer вернулся, следующий upload ready | `broker-resubscribe.txt` |
|
||||
| Разделение RabbitMQ identities | `broker-init` + `rabbitmqctl list_permissions` | API не читает `attachments.process`; worker — отдельный пользователь | `broker-identities.txt` |
|
||||
|
||||
## Эксперименты и отказы
|
||||
|
||||
TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения.
|
||||
### Гипотеза трека 2
|
||||
|
||||
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
|
||||
| --- | --- | --- | --- |
|
||||
| TODO | TODO | TODO | TODO |
|
||||
At-least-once delivery + DB-fenced claim + versioned effect keys дают
|
||||
**exactly-once effects** для `(attachment_id, pipeline_version, variant)` при
|
||||
дубле, reorder, crash до/после side effect и устаревшей job после новой версии.
|
||||
Доставка при этом остаётся at-least-once: дубли и повторные публикации
|
||||
наблюдаются и поглощаются, а не исчезают.
|
||||
|
||||
### Методика
|
||||
|
||||
- Стенд: 8 vCPU (i7-11370H), 15 GiB RAM; worker 1 реплика, concurrency 2,
|
||||
лимит 1280 MiB; broker 0.5 CPU / 512 MiB; JOB_TIMEOUT_SECONDS=10,
|
||||
JOB_LEASE_SECONDS=8, retry 500–4000 ms; `track2` повторён 3 раза.
|
||||
- Перед сценариями загружается reference и запоминаются SHA-256 вариантов v1.
|
||||
- Каждый сценарий начинается и заканчивается очисткой `fault:*`; timeline
|
||||
опрашивается каждые 200 мс.
|
||||
- Инварианты:
|
||||
- **I1** — ready (или ожидаемый failed) не позже 60 с;
|
||||
- **I2** — ровно 3 варианта ready-версии и ровно одно `succeeded` на job;
|
||||
- **I3** — ready не откатывается, `ready_pipeline_version` не убывает;
|
||||
- **I4** — подтверждённая работа не потеряна (есть успешный эффект);
|
||||
- **I5** — нет дублей `(pipeline_version, variant)`;
|
||||
- **I6** — SHA-256 вариантов совпадает с baseline для той же версии.
|
||||
|
||||
### Результаты
|
||||
|
||||
Таблица заполняется из `evidence/lab5/track2.json` и
|
||||
`evidence/lab5/faults-level4.json`; фактические значения и пик памяти worker
|
||||
приведены в `evidence/lab5/summary.md`.
|
||||
|
||||
| Сценарий | Группа | Наблюдение (3 повтора для track2) |
|
||||
| --- | --- | --- |
|
||||
| `duplicate` | track2 | 5 повторных publish → 5 `duplicate_skipped` (`already_succeeded`); S3 etag/last_modified не изменились; attempt 1 |
|
||||
| `reorder` | track2 | attempt 99 и дубль attempt 1 поглощены; ровно 2 skip, 1 `succeeded`; attempt 1 |
|
||||
| `crash_before_side_effect` | track2 | worker убит до S3; попытка 2, `lease_expired`, процесс worker перезапущен |
|
||||
| `crash_after_side_effect` | track2 | попытка 2, `lease_expired`, SHA-256 вариантов совпал с baseline (I6) |
|
||||
| `crash_after_commit_before_ack` | track2 | попытка 1, один `succeeded`, redelivery → `duplicate_skipped` (`already_succeeded`) |
|
||||
| `relay_crash_after_publish` | track2 | API убит между publish и mark; relay опубликовал снова, дубль поглощён; все 3 повтора |
|
||||
| `stale_version` | track2 | `ready_pipeline_version=2`, v1 `superseded`, остались только варианты v2, v2 etag не изменились, served thumbnail SHA ≠ v1 |
|
||||
| `crash_after_commit_before_publish` | level4 | API убит после COMMIT; outbox relay опубликовал после рестарта; ровно один `created`, ready |
|
||||
| `retry_and_replay` | level4 | 4 `retry_scheduled`, затем `failed`/`retry_exhausted`; replay → attempt 6 → ready; `replayed` с operator |
|
||||
| `hung_worker` | level4 | abort по `JOB_TIMEOUT_SECONDS`, `retry_scheduled` с `processing_timeout`, затем ready |
|
||||
| `poison` | level4 | два невалидных сообщения ушли в DLQ, jobs не созданы |
|
||||
| `big_image` | level4 | оба 40 Мп вложения ready, `OOMKilled=false`; пик 934–992 MiB при лимите 1280 MiB |
|
||||
|
||||
Итог: `level4` — **7/7**, `track2` — **21/21** (7 сценариев × 3), все
|
||||
инварианты I1–I6 истинны; см. `evidence/lab5/faults-level4.json`,
|
||||
`evidence/lab5/track2.json`.
|
||||
|
||||
### Ограничения эксперимента
|
||||
|
||||
Это не exactly-once **доставка**: redelivery и повторная публикация наблюдаются.
|
||||
S3-запись может повториться (объект перезаписывается тем же детерминированным
|
||||
байтом), PostgreSQL — одна нода, а fault points находятся внутри процесса.
|
||||
Сценарии `relay_crash_after_publish` и `crash_after_commit_before_publish`
|
||||
зависят от гонки relay и поэтому фиксируют наблюдение, если дубль не успел
|
||||
возникнуть. Внешние эффекты за пределами `(attachment, version, variant)` не
|
||||
защищены.
|
||||
|
||||
## Исправления по независимому ревью
|
||||
|
||||
После первого зелёного прогона ревью выявило латентные отказы; внесены правки:
|
||||
|
||||
- **Потеря queued job после DLQ/исчезнувшего сообщения:** `LeaseReaper` теперь
|
||||
периодически вызывает `reconcileQueuedJobs`; job без unpublished outbox
|
||||
получает новую outbox-строку. `claim` при ошибке БД повторяется с задержкой,
|
||||
nack отложен, поэтому delivery limit не сжигается в tight loop.
|
||||
- **Потеря consumer после рестарта broker:** `BrokerClient` отслеживает
|
||||
`subscribing`, повторяет `basic.consume` при закрытии канала, `basic.cancel`
|
||||
и ошибке подписки; частично установленное соединение закрывается.
|
||||
- **Publish без подтверждения/маршрута:** добавлены timeout подтверждения,
|
||||
`mandatory: true` и обработка `basic.return`; outbox не помечает строку
|
||||
published при возврате.
|
||||
- **Clock skew «not_due»:** due-check выполняется в SQL (`next_attempt_at <= now()`).
|
||||
- **Порядок блокировок:** claim/reap/replay/cleaner блокируют job, затем
|
||||
attachment; это исключает 40P01 между claim и commit.
|
||||
- **Staging cleaner vs replay:** удаление объекта выполняется под row-lock
|
||||
вместе с `clearStaging`; replay ждёт и видит `queued`.
|
||||
- **Relay в worker:** retry и lease-expiry публикуются даже при недоступном API;
|
||||
при ошибке relay использует backoff.
|
||||
- **Варианты и lease:** ключ варианта содержит attempt
|
||||
(`.../<variant>-a<attempt>.<ext>`); потерявший lease worker не перезапишет
|
||||
объект, на который смотрит БД.
|
||||
- **Долгий рендер:** общий дедлайн распределяется по операциям sharp, abort
|
||||
проверяется между ними.
|
||||
- **Терминальное событие reaper:** при `retry_exhausted` пишется `failed`.
|
||||
- **Direct upload cap:** проверка лимита и insert сериализованы advisory-lock.
|
||||
- **Retention:** опубликованный outbox и старые события чистятся по TTL,
|
||||
добавлен индекс `job_outbox(job_id)`.
|
||||
- **Legacy lab4 rows:** миграция 008 ставит им v1 job в outbox, чтобы появились
|
||||
варианты, а не пустой `variants: []`.
|
||||
- **Least privilege:** API теряет `DeleteObject` на варианты (только
|
||||
`attachments/*/original`), S3 policies ограничены префиксами; RabbitMQ
|
||||
разделён на `messenger` (публикация, без чтения очереди) и
|
||||
`messenger-worker` (consumption), `broker-init` приводит существующий volume
|
||||
к этой схеме.
|
||||
|
||||
Осталось осознанным упрощением: финальный оригинал permanently failed job
|
||||
сохраняется для операторского replay, staging удаляется по retention.
|
||||
|
||||
## Повторное ревью и вторые правки
|
||||
|
||||
Второй независимый проход не нашёл P0/P1, но выявил последствия первых правок;
|
||||
они устранены:
|
||||
|
||||
- `withTransaction` снимает `error`-события checked-out pg-клиента и
|
||||
`release(error)`, поэтому `idle_in_transaction_session_timeout` не роняет
|
||||
процесс; relay дополнительно ограничен бюджетом `< idle_timeout/2`.
|
||||
- Reconciler срабатывает только при пустой очереди и отсчитывает grace от
|
||||
`job_outbox.published_at`, поэтому нормальный backlog не размножается.
|
||||
- Миграция 009 добавляет partial index по `queued` jobs, индексы по
|
||||
`job_events(created_at)` и `job_outbox(published_at)`; pruning стал пакетным.
|
||||
- Reconciliation пишется отдельным событием `reconciled`, а не
|
||||
`retry_scheduled`, чтобы не искажать счётчик retry.
|
||||
- После успеха/исчерпания попыток worker удаляет варианты старых attempt
|
||||
(`-aN`), для чего worker получил `DeleteObject` только на `attachments/*/v*/*`.
|
||||
- `replay` передаёт ожидаемый `staging_key` и под блокировкой отказывает
|
||||
`source_changed`, если cleaner успел изменить источник.
|
||||
- Publisher снимает метрику ровно один раз (`settled`), confirm-channel
|
||||
`close`/`error` обрабатываются, а сбой consumer-подписки больше не рвёт
|
||||
рабочее publish-соединение.
|
||||
- RabbitMQ разделён на `messenger` (API: публикация и topology, без чтения
|
||||
очереди) и `messenger-worker` (consumption); одноразовый `broker-init`
|
||||
создаёт второго пользователя и сужает права первого.
|
||||
|
||||
## Ограничения и следующие шаги
|
||||
|
||||
TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза.
|
||||
- Реализованы уровни 3–4 и трек 2; трек 1 (сравнение concurrency/prefetch) не
|
||||
заявлен.
|
||||
- Финальный original permanently failed job не удаляется автоматически: он
|
||||
нужен для безопасного replay, пока staging ещё не истёк; ручная очистка
|
||||
описана в отчёте.
|
||||
- Брокер и S3 — single-node; production-варианты требуют кворумов и ретраев
|
||||
между зонами.
|
||||
- `job_outbox` публикуется одним relay-процессом на реплику; при росте нагрузки
|
||||
нужен отдельный relay или партиционирование.
|
||||
- Ошибки классифицируются по сообщению sharp/`SERVER_FAULT`; для новых
|
||||
форматов список стоит расширять по мере появления.
|
||||
- Следующий шаг: метрики очереди lag и p95 ready, отдельный DLQ-replay UI,
|
||||
сравнение concurrency/prefetch (трек 1).
|
||||
|
||||
@@ -0,0 +1,59 @@
|
||||
# compiled output
|
||||
/dist
|
||||
/node_modules
|
||||
/build
|
||||
|
||||
# TypeScript build cache
|
||||
*.tsbuildinfo
|
||||
|
||||
# Logs
|
||||
logs
|
||||
*.log
|
||||
npm-debug.log*
|
||||
pnpm-debug.log*
|
||||
yarn-debug.log*
|
||||
yarn-error.log*
|
||||
lerna-debug.log*
|
||||
|
||||
# OS
|
||||
.DS_Store
|
||||
|
||||
# Tests
|
||||
/coverage
|
||||
/.nyc_output
|
||||
|
||||
# IDEs and editors
|
||||
/.idea
|
||||
.project
|
||||
.classpath
|
||||
.c9/
|
||||
*.launch
|
||||
.settings/
|
||||
*.sublime-workspace
|
||||
|
||||
# IDE - VSCode
|
||||
.vscode/*
|
||||
!.vscode/settings.json
|
||||
!.vscode/tasks.json
|
||||
!.vscode/launch.json
|
||||
!.vscode/extensions.json
|
||||
|
||||
# dotenv environment variable files
|
||||
.env
|
||||
.env.development.local
|
||||
.env.test.local
|
||||
.env.production.local
|
||||
.env.local
|
||||
|
||||
# temp directory
|
||||
.temp
|
||||
.tmp
|
||||
|
||||
# Runtime data
|
||||
pids
|
||||
*.pid
|
||||
*.seed
|
||||
*.pid.lock
|
||||
|
||||
# Diagnostic reports (https://nodejs.org/api/report.html)
|
||||
report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json
|
||||
@@ -0,0 +1,6 @@
|
||||
{
|
||||
"singleQuote": true,
|
||||
"trailingComma": "all",
|
||||
"tabWidth": 4,
|
||||
"useTabs": true
|
||||
}
|
||||
+314
@@ -0,0 +1,314 @@
|
||||
# app — API мессенджера
|
||||
|
||||
NestJS 12 + Fastify, ESM TypeScript. PostgreSQL — источник истины бизнес-данных,
|
||||
Redis — серверные Bearer-сессии и производный кэш истории сообщений.
|
||||
|
||||
Общая документация (в корне репозитория):
|
||||
|
||||
| Документ | Что содержит |
|
||||
|---|---|
|
||||
| [`../REPORT.md`](../REPORT.md) | архитектура, схема данных, транзакции, политика кэша, команды, результаты проверок и трека 1 |
|
||||
| [`../evidence/README.md`](../evidence/README.md) | индекс доказательств: что подтверждено и как это воспроизвести |
|
||||
| [`../contracts/openapi.json`](../contracts/openapi.json) | нормативный контракт API |
|
||||
| [`../tests/README.md`](../tests/README.md) | публичные проверки репозитория (`make check`, `make test`, persistence) |
|
||||
|
||||
## Что внутри
|
||||
|
||||
```text
|
||||
src/
|
||||
main.ts bootstrap: Fastify, ValidationPipe, shutdown hooks
|
||||
app.module.ts сборка модулей приложения
|
||||
config/ Zod-схема env + глобальный ConfigModule
|
||||
database/ pg.Pool, транзакции, миграции (001..004)
|
||||
redis/ клиент Redis (AOF-совместимый, без offline queue)
|
||||
auth/ Basic-вход, Redis-сессии, logout
|
||||
users/ регистрация (Argon2id), профиль
|
||||
chats/ создание чата в транзакции, список и чтение
|
||||
messages/ отправка сообщения, keyset-история
|
||||
cache/ версионированный кэш истории + lock на страницу
|
||||
idempotency/ журнал Idempotency-Key (claim/read/store/forget/GC)
|
||||
metrics/ /internal/metrics и /internal/metrics/reset
|
||||
health/ liveness/readiness с проверкой зависимостей (readiness — проба LB)
|
||||
common/
|
||||
http/ cursor (формат+календарь), page (конверт {items,next_cursor}), request-id
|
||||
errors/ exception filter и тип тела ошибки ApiErrorBody
|
||||
validation/ декоратор CodePointLength (длины в Unicode code points)
|
||||
auth/ guard и декоратор текущего пользователя
|
||||
test/
|
||||
app.e2e-spec.ts black-box набор против запущенного API (PG + Redis)
|
||||
scripts/
|
||||
graph-modules.ts генератор графа зависимостей (nestjs-spelunker)
|
||||
```
|
||||
|
||||
Принципы: raw `pg.Pool` без ORM, параметризованные запросы, никаких in-memory
|
||||
хранилищ в боевых путях; кэш — производное состояние и всегда допускает
|
||||
восстановление из PostgreSQL; отказ зависимости даёт ограниченную ошибку
|
||||
(`503`), а не аварийное завершение процесса; длины строк проверяются в Unicode
|
||||
code points (свой декоратор `@CodePointLength`), как в JSON Schema, а не в
|
||||
UTF-16-единицах; состояние, от которого зависит корректность (сессии, журнал
|
||||
идемпотентности), живёт только в общих PostgreSQL/Redis — локальных сессий и
|
||||
файлов на диске реплики нет, поэтому `N` взаимозаменяемых реплик за одним LB
|
||||
дают один и тот же результат.
|
||||
|
||||
## Быстрый старт
|
||||
|
||||
Рекомендуемый путь — весь стенд из корня репозитория (PostgreSQL, Redis, миграции, API):
|
||||
|
||||
```sh
|
||||
cd ..
|
||||
cp .env.example .env
|
||||
password=$(openssl rand -hex 32)
|
||||
sed -i "s/^POSTGRES_PASSWORD=.*/POSTGRES_PASSWORD=$password/" .env
|
||||
sed -i "s|^DATABASE_URL=.*|DATABASE_URL=postgresql://messenger:$password@postgres:5432/messenger|" .env
|
||||
sed -i "s/^METRICS_TOKEN=.*/METRICS_TOKEN=$(openssl rand -hex 16)/" .env
|
||||
make stand-up # сборка + up -d --wait: migrate -> api x API_REPLICAS -> haproxy
|
||||
make wait-ready
|
||||
curl -fsS http://localhost:8080/health/ready
|
||||
for i in 1 2 3 4; do curl -sD- -o /dev/null localhost:8080/health/live | grep -i x-instance-id; done
|
||||
```
|
||||
|
||||
Публичной точкой входа служит HAProxy (`compose.yaml`): реплики API портов не
|
||||
публикуют. `API_REPLICAS` задаёт их число, `API_TOTAL_CPUS` / `API_TOTAL_MEMORY_MB`
|
||||
делятся между ними — это бюджет серии трека 1 (`../REPORT.md`).
|
||||
|
||||
Локальный запуск без Docker (нужны доступные PostgreSQL и Redis):
|
||||
|
||||
```sh
|
||||
cd app
|
||||
npm ci
|
||||
npm run build
|
||||
|
||||
export DATABASE_URL=postgresql://messenger:<password>@127.0.0.1:5432/messenger
|
||||
export REDIS_URL=redis://127.0.0.1:6379
|
||||
export APP_PORT=8080
|
||||
node dist/database/migration.runner.js # применяет миграции из dist/database/migrations
|
||||
|
||||
npm run start:dev # или npm run start:prod после build
|
||||
```
|
||||
|
||||
## Переменные окружения
|
||||
|
||||
Читаются только через конфигурационный слой (`src/config`), типы и диапазоны
|
||||
проверяет Zod-схема на старте — при некорректном значении приложение падает
|
||||
с понятной ошибкой.
|
||||
|
||||
| Переменная | По умолчанию | Назначение |
|
||||
|---|---|---|
|
||||
| `APP_HOST` / `APP_PORT` | `0.0.0.0` / `8080` | адрес прослушивания |
|
||||
| `APP_ENV` | `lab` | метка окружения |
|
||||
| `DATABASE_URL` | — (обязательна) | строка подключения PostgreSQL |
|
||||
| `REDIS_URL` | — (обязательна) | строка подключения Redis |
|
||||
| `DATABASE_POOL_MAX` | `10` | размер пула соединений |
|
||||
| `DATABASE_CONNECTION_TIMEOUT_MS` | `5000` | таймаут установки соединения |
|
||||
| `DATABASE_STATEMENT_TIMEOUT_MS` | `5000` | таймаут SQL-запроса |
|
||||
| `SESSION_TTL_SECONDS` | `3600` | TTL сессии (минимум 2) |
|
||||
| `HEALTH_TIMEOUT_MS` | `400` | бюджет проверки PostgreSQL/Redis внутри `/health/ready`; меньше `timeout check` HAProxy (500 ms) и `DATABASE_STATEMENT_TIMEOUT_MS` |
|
||||
| `CACHE_TTL_SECONDS` | `30` | TTL страницы истории в Redis |
|
||||
| `CACHE_MODE` | `protected` | `protected` (lock на страницу) или `naive` (только для эксперимента) |
|
||||
| `CACHE_LOCK_TTL_MS` | `1000` | TTL lock-а страницы |
|
||||
| `CACHE_MISS_DELAY_MS` | `0` | искусственная пауза на промахе; влияет только на `naive` |
|
||||
| `METRICS_TOKEN` | — (необязательна) | токен доступа к `/internal/metrics*`; без него endpoint закрыт |
|
||||
| `DRAIN_DELAY_MS` | `5000` (в стенде `3000`) | окно дрейна при `SIGTERM`: реплика сразу перестаёт проходить пробу LB, ждёт этот интервал и только затем закрывает сервер и пулы |
|
||||
| `INSTANCE_ID` | hostname контейнера | значение `X-Instance-Id`; в Docker hostname = короткий ID контейнера, поэтому `scripts/stop-one.sh` находит реплику по нему |
|
||||
| `IDEMPOTENCY_TTL_SECONDS` | `86400` | срок хранения журнала Idempotency-Key (минимум 60) |
|
||||
|
||||
Поддерживаются также `DATABASE_URL_FILE` / `REDIS_URL_FILE`: если заданы, значение
|
||||
читается из файла (docker secrets / Swarm). Подробности и компромиссы —
|
||||
[`../REPORT.md`](../REPORT.md), раздел «Конфигурация и секреты».
|
||||
|
||||
## Эндпоинты
|
||||
|
||||
Публичный контракт — [`../contracts/openapi.json`](../contracts/openapi.json).
|
||||
|
||||
| Метод и путь | Аутентификация | Назначение |
|
||||
|---|---|---|
|
||||
| `GET /health/live` | нет | процесс жив; не обращается к зависимостям, поэтому отвечает `200` и при их отказе |
|
||||
| `GET /health/ready` | нет | готовность реплики: флаг дреина + PostgreSQL и Redis (бюджет `HEALTH_TIMEOUT_MS`), иначе `503`; эту же ручку пробует HAProxy |
|
||||
| `POST /api/v1/users` | нет | регистрация с паролем (Argon2id) |
|
||||
| `POST /api/v1/auth/sessions` | `Basic` | вход, выдаёт opaque Bearer-токен |
|
||||
| `DELETE /api/v1/auth/sessions/current` | `Bearer` | отзыв текущей сессии, `204` |
|
||||
| `GET /api/v1/users/me` | `Bearer` | текущий пользователь |
|
||||
| `POST /api/v1/chats` | `Bearer` | создание чата (транзакция) |
|
||||
| `GET /api/v1/chats` | `Bearer` | список доступных чатов |
|
||||
| `GET /api/v1/chats/:chatId` | `Bearer` | чат по id (чужой — `404`) |
|
||||
| `POST /api/v1/chats/:chatId/messages` | `Bearer` + обязательный `Idempotency-Key` | отправка сообщения (см. ниже) |
|
||||
| `GET /api/v1/chats/:chatId/messages` | `Bearer` | история, keyset-пагинация `(created_at, id)` |
|
||||
| `GET /internal/metrics` | `Bearer METRICS_TOKEN` | счётчики и агрегаты латентности |
|
||||
| `POST /internal/metrics/reset` | `Bearer METRICS_TOKEN` | сброс счётчиков (`200`) |
|
||||
|
||||
Правила: `Basic` принимается только на создании сессии, бизнес-запросы — только
|
||||
`Bearer`; заголовок `X-User-Id` не аутентифицирует и не подменяет владельца
|
||||
сессии; истёкший, отозванный или неизвестный токен — `401` с соответствующим
|
||||
`WWW-Authenticate`.
|
||||
|
||||
## npm-скрипты
|
||||
|
||||
| Команда | Что делает |
|
||||
|---|---|
|
||||
| `npm run build` | сборка в `dist/` |
|
||||
| `npm run start:dev` | watch-режим |
|
||||
| `npm run start:prod` | запуск собранного приложения (`node dist/main`) |
|
||||
| `npm test` / `npm run test:unit` | unit-тесты (vitest) |
|
||||
| `npm run test:e2e` | black-box набор против запущенного API (см. ниже) |
|
||||
| `npm run test:dependencies` | автоматическая проверка отказов Redis/PostgreSQL: останавливает и поднимает контейнеры Compose-стенда |
|
||||
| `npm run idempotency:gc` | ручной прогон удаления просроченных записей Idempotency-Key |
|
||||
| `npm run lint` | oxlint по `src/` и `test/` |
|
||||
| `npm run format` | prettier |
|
||||
| `npm run graph` | граф модулей и провайдеров через nestjs-spelunker (пишет `../docs/architecture/`) |
|
||||
|
||||
## Тесты
|
||||
|
||||
Unit-тесты не требуют инфраструктуры:
|
||||
|
||||
```sh
|
||||
npm test
|
||||
```
|
||||
|
||||
Интеграционный набор работает по HTTP против запущенного стенда и дополнительно
|
||||
подключается к PostgreSQL (очистка тестовых данных) и Redis:
|
||||
|
||||
```sh
|
||||
export API_BASE_URL=http://localhost:8080
|
||||
export DATABASE_URL=postgresql://messenger:<password>@<pg-host>:5432/messenger
|
||||
export REDIS_URL=redis://<redis-host>:6379
|
||||
npm run test:e2e
|
||||
```
|
||||
|
||||
Название `API_BASE_URL` (а не `BASE_URL`) выбрано потому, что Vitest сам
|
||||
устанавливает `process.env.BASE_URL` в значение Vite base. Кейс истечения сессии
|
||||
включается переменной `SESSION_TTL_SECONDS=2` и у запущенного API, и у набора.
|
||||
|
||||
## Отказы зависимостей
|
||||
|
||||
Отдельный набор останавливает и поднимает контейнеры Compose-стенда и проверяет
|
||||
ограниченные ошибки вместо 500:
|
||||
|
||||
```sh
|
||||
cd .. # из корня репозитория, стенд поднят через docker compose
|
||||
make test-dependencies
|
||||
```
|
||||
|
||||
Он включён по флагу `E2E_DEPENDENCY_FAULTS=1` (им же управляет цель Makefile),
|
||||
потому что требует прав на управление Docker Compose-проектом; остальные
|
||||
переменные — `E2E_COMPOSE_DIR`, `E2E_COMPOSE_ENV_FILE`, `E2E_REDIS_SERVICE`,
|
||||
`E2E_POSTGRES_SERVICE`, `E2E_API_SERVICE`, `E2E_LB_STATS_URL`,
|
||||
`E2E_POSTGRES_USER`, `E2E_POSTGRES_DB`, `E2E_PG_HBA_PATH`.
|
||||
|
||||
Проверяются три топологии отказа:
|
||||
|
||||
1. **Отказ Redis** (общая зависимость): `/health/live` реплики остаётся `200`,
|
||||
`/health/ready` и бизнес-запросы — `503 unavailable` с `request_id` из
|
||||
`X-Request-Id` и без деталей драйвера, запрос без сессии — по-прежнему `401`.
|
||||
Неготовыми становятся все реплики, HAProxy исключает их из ротации и
|
||||
отвечает собственным `503`; после старта Redis сессия и подтверждённое
|
||||
сообщение живы.
|
||||
2. **Отказ PostgreSQL** (общая зависимость): то же плюс проверка, что после
|
||||
возврата зависимости данные на месте и новая запись проходит.
|
||||
3. **Отказ PostgreSQL у одной реплики** (локальный путь к БД): для адреса
|
||||
выбранной реплики временно ставится `reject`-строка в `pg_hba.conf` (первое
|
||||
совпадение) и обрываются её соединения. Выбранная реплика отвечает `503` и её
|
||||
слот уходит в `DOWN`, здоровая продолжает `200`/`UP` и обслуживает все чтения
|
||||
через LB; `pg_hba.conf` восстанавливается в `finally`, после чего реплика
|
||||
возвращается в ротацию и снова получает запросы.
|
||||
|
||||
Реплики API портов не публикуют, поэтому отдельная реплика опрашивается изнутри
|
||||
своего контейнера (loopback), а состояние слотов — из `stats;csv` HAProxy
|
||||
(`127.0.0.1:8404`). Каждый запрос к остановленной БД ждёт
|
||||
`DATABASE_CONNECTION_TIMEOUT_MS`, поэтому набор идёт около минуты.
|
||||
|
||||
## Реплики, балансировка и остановка реплики
|
||||
|
||||
Все реплики запускаются из одного образа и одного `compose.yaml`; API портов не
|
||||
публикует, трафик идёт только через HAProxy. Состояние, от которого зависит
|
||||
корректность, лежит в общих PostgreSQL и Redis, поэтому клиент с одним
|
||||
Bearer-токеном обслуживается любой репликой:
|
||||
|
||||
```sh
|
||||
make test BASE_URL=http://localhost:8080 # сквозной сценарий через LB
|
||||
python3 scripts/lab3/lab3.py check spread # распределение X-Instance-Id
|
||||
make test-failover ACTION_SCRIPT=scripts/stop-one.sh # отказ одной реплики
|
||||
scripts/start-replica.sh # вернуть остановленную реплику
|
||||
```
|
||||
|
||||
- `X-Instance-Id` = `INSTANCE_ID` или hostname процесса, стабилен для процесса;
|
||||
в Docker hostname контейнера — короткий ID, поэтому `scripts/stop-one.sh`
|
||||
останавливает ровно ту реплику, которую обслужил запрос теста. Тот же заголовок
|
||||
добавляется к ответам об ошибках, которые формируются фильтром до интерсептора.
|
||||
- **Остановка (`SIGTERM`)** идёт в порядке «сначала дрейн, потом разбор ресурсов»:
|
||||
реплика помечается not-ready (проба LB и `/health/ready` начинают отвечать
|
||||
`503`), `DRAIN_DELAY_MS` ждёт, пока LB её исключит (обнаружение ≈1 s), и лишь
|
||||
затем закрываются HTTP-сервер, пул PostgreSQL и соединение Redis. Если
|
||||
полагаться на стандартный порядок Nest (`onModuleDestroy` вызывает закрытие
|
||||
пулов раньше `beforeApplicationShutdown`), реплика какое-то время остаётся в
|
||||
ротации уже без ресурсов — это и было причиной явной последовательности в
|
||||
`src/main.ts`.
|
||||
- **Проба LB — тот же `/health/ready`**, что и healthcheck контейнера: реплика,
|
||||
потерявшая обязательную зависимость (свой путь к PostgreSQL или Redis),
|
||||
выходит из ротации, и трафик уходит на здоровые реплики. Проба с зависимостями
|
||||
ограничена общим дедлайном `HEALTH_TIMEOUT_MS=400` на реплику; проверка
|
||||
PostgreSQL (`SELECT 1`) идёт по отдельному соединению, а не из общего пула
|
||||
бизнес-запросов, а для `PING` Redis стоит `AbortSignal.timeout`, поэтому проба
|
||||
укладывается в `timeout check 500ms` HAProxy и не реагирует на перегрузку.
|
||||
Когда недоступна общая для всех зависимость,
|
||||
неготовыми становятся все реплики, и клиент получает `503` от самого LB: это
|
||||
честный ответ «готовых узлов нет» вместо маскировки отказа.
|
||||
- `/health/live` зависимостей не касается: он отвечает `200`, пока жив процесс,
|
||||
и по нему проверяют, что реплика не упала (в том числе при отказе PG/Redis).
|
||||
|
||||
## Идемпотентность отправки сообщения
|
||||
|
||||
`POST /api/v1/chats/{chatId}/messages` требует заголовок `Idempotency-Key`
|
||||
(ASCII `[A-Za-z0-9._:-]{1,128}`). Область уникальности — пользователь + метод +
|
||||
путь чата + ключ, поэтому одинаковый ключ у разных пользователей или в разных
|
||||
чатах не конфликтует. Журнал — таблица `idempotency_keys` (миграция 003), то
|
||||
есть общая для всех реплик.
|
||||
|
||||
| Ситуация | Ответ |
|
||||
|---|---|
|
||||
| Заголовок отсутствует, пуст или не подходит под шаблон | `400 invalid_request` (до обращения к БД) |
|
||||
| Первый запрос с ключом | `201` и сообщение; заявка, вставка сообщения, инкремент версии чата и тело ответа записываются **одним оператором SQL** (атомарность обеспечивает сам оператор) |
|
||||
| Повтор того же тела с тем же ключом | тот же `201` и тот же JSON (включая `id` и `created_at`), новое сообщение не создаётся |
|
||||
| Другое тело с тем же ключом | `409 conflict` |
|
||||
| Повтор после `404` (нет доступа к чату) | `404`, ключ не занят: оператор вставляет заявку только при наличии членства, иначе не создаётся ничего |
|
||||
| Ключ старше `IDEMPOTENCY_TTL_SECONDS` | запись удаляется, ключ считается свободным |
|
||||
| Одновременные повторы через разные реплики | ровно одно сообщение: заявка берётся `INSERT ... ON CONFLICT DO NOTHING`, повтор читает сохранённый ответ |
|
||||
|
||||
Путь записи — один round trip: `messages.createIdempotent` проверяет членство,
|
||||
занимает ключ (`INSERT ... ON CONFLICT DO NOTHING`), вставляет сообщение, поднимает
|
||||
`chats.message_version` и сохраняет тело ответа, а повтор читает сохранённый ответ
|
||||
вместе с повторной проверкой доступа (`idempotency.read`). До оптимизации это были
|
||||
8 обращений к БД в явной транзакции (см. REPORT.md, раздел «A/B улучшения»).
|
||||
|
||||
Отпечаток тела считается по валидированному DTO с фиксированным порядком полей,
|
||||
поэтому порядок ключей в JSON не влияет на результат. Транзиентные сбои
|
||||
PostgreSQL (`57014`, `40001`, `40P01`) и обрывы соединения отдаются как
|
||||
`503 unavailable` и не фиксируются как окончательный результат: клиент может
|
||||
безопасно повторить запрос с тем же ключом. Просроченные записи удаляются
|
||||
фоновым проходом (`idempotency.purged` в метриках) под
|
||||
`pg_try_advisory_xact_lock`, а также вручную: `npm run idempotency:gc`.
|
||||
|
||||
## Харнесс экспериментов
|
||||
|
||||
Уровни 3/4 и трек 1 обслуживает `../scripts/lab3/lab3.py` (только stdlib) и
|
||||
профили k6 в `../scripts/load/`: `run`, `sweep`, `track1-extras`, `failover`,
|
||||
`rolling-update`, `ab`, `analyze`, `watch`, `check`, `quiesce`, `track1`.
|
||||
Сырые результаты прогонов, манифесты условий измерения и графики складываются в
|
||||
`../evidence/track1/`. Подробности — в `../REPORT.md` и
|
||||
`../evidence/README.md`.
|
||||
|
||||
## Метрики
|
||||
|
||||
`GET /internal/metrics` возвращает `{counters, latency_ms, instance_id, build_id}`: именованные SQL-метки,
|
||||
операции Redis, `cache_hit`/`cache_miss`/`cache_lock_wait`, `session_get/set/del`,
|
||||
`request_count` и процентили латентности (буфер ограничен). Деградации
|
||||
наблюдаемы отдельными счётчиками: `cache_fallback`, `cache_write_error`,
|
||||
`cache_unlock_error`, `cache_lock_timeout`, `cache_corrupt_entry`,
|
||||
`database.pool_error`, `database.rollback_error`, `redis.error`, а также
|
||||
идемпотентность: `idempotency.claim`, `idempotency.replay`, `idempotency.conflict`,
|
||||
`idempotency.claim_lost`, `idempotency.released`, `idempotency.purged`,
|
||||
`idempotency.purge_error`, `messages.write`.
|
||||
|
||||
Счётчики локальны процессу: при нескольких репликах их нельзя суммировать как
|
||||
распределённый снимок, а рестарт API обнуляет их (данные PostgreSQL и Redis при
|
||||
этом не затрагиваются). `instance_id` и `build_id` позволяют сопоставить ответ
|
||||
конкретной реплике и сборке — этим пользуются rolling update и эксперименты.
|
||||
@@ -0,0 +1,15 @@
|
||||
{
|
||||
"$schema": "https://json.schemastore.org/nest-cli",
|
||||
"collection": "@nestjs/schematics",
|
||||
"sourceRoot": "src",
|
||||
"compilerOptions": {
|
||||
"deleteOutDir": true,
|
||||
"assets": [
|
||||
{
|
||||
"include": "database/migrations/*.sql",
|
||||
"outDir": "dist"
|
||||
}
|
||||
],
|
||||
"watchAssets": true
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,10 @@
|
||||
{
|
||||
"$schema": "https://raw.githubusercontent.com/oxc-project/oxc/main/crates/oxc_linter/src/rules.rs",
|
||||
"rules": {
|
||||
"@typescript-eslint/no-explicit-any": "off",
|
||||
"@typescript-eslint/no-floating-promises": "warn"
|
||||
},
|
||||
"env": {
|
||||
"node": true
|
||||
}
|
||||
}
|
||||
Generated
+8385
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,73 @@
|
||||
{
|
||||
"name": "app",
|
||||
"version": "0.0.1",
|
||||
"description": "",
|
||||
"author": "",
|
||||
"private": true,
|
||||
"license": "UNLICENSED",
|
||||
"type": "module",
|
||||
"scripts": {
|
||||
"build": "nest build",
|
||||
"deploy": "nest deploy",
|
||||
"format": "prettier --write \"src/**/*.ts\" \"test/**/*.ts\"",
|
||||
"start": "nest start",
|
||||
"start:dev": "nest start --watch",
|
||||
"start:debug": "nest start --debug --watch",
|
||||
"start:prod": "node dist/main",
|
||||
"lint": "oxlint src/ test/",
|
||||
"test": "vitest run",
|
||||
"test:unit": "vitest run src/",
|
||||
"test:watch": "vitest",
|
||||
"test:cov": "vitest run --coverage",
|
||||
"test:debug": "vitest --inspect-brk --no-file-parallelism",
|
||||
"graph": "npm run build && DATABASE_URL=postgresql://graph-only.invalid/messenger REDIS_URL=redis://graph-only.invalid:6379 tsx scripts/graph-modules.ts",
|
||||
"test:e2e": "vitest run --config ./vitest.config.e2e.ts",
|
||||
"test:dependencies": "E2E_DEPENDENCY_FAULTS=1 vitest run --config ./vitest.config.e2e.ts dependency-failures",
|
||||
"attachments:sweep": "tsx src/attachments/attachments-sweep.cli.ts",
|
||||
"idempotency:gc": "tsx src/messages/idempotency/idempotency-gc.cli.ts",
|
||||
"jobs": "node dist/jobs/jobs.cli.js"
|
||||
},
|
||||
"dependencies": {
|
||||
"@aws-sdk/client-s3": "^3.1156.0",
|
||||
"@aws-sdk/s3-request-presigner": "^3.1156.0",
|
||||
"@nestjs/common": "^12.0.1",
|
||||
"@nestjs/config": "^12.0.0",
|
||||
"@nestjs/core": "^12.0.1",
|
||||
"@nestjs/platform-express": "^12.0.1",
|
||||
"@nestjs/platform-fastify": "^12.0.3",
|
||||
"@smithy/node-http-handler": "^4.12.1",
|
||||
"amqplib": "^2.2.1",
|
||||
"argon2": "^0.45.1",
|
||||
"class-transformer": "^0.5.1",
|
||||
"class-validator": "^0.15.1",
|
||||
"date-fns": "^4.4.0",
|
||||
"fastify": "^5.12.5",
|
||||
"pg": "^8.23.1",
|
||||
"redis": "^6.3.0",
|
||||
"reflect-metadata": "^0.2.2",
|
||||
"rxjs": "^7.8.1",
|
||||
"sharp": "^0.35.5",
|
||||
"zod": "^4.6.5"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@nestjs/cli": "^12.0.0",
|
||||
"@nestjs/mau": "^0.2.6",
|
||||
"@nestjs/schematics": "^12.0.0",
|
||||
"@nestjs/testing": "^12.0.1",
|
||||
"@types/express": "^5.0.0",
|
||||
"@types/node": "^24.0.0",
|
||||
"@types/pg": "^8.23.1",
|
||||
"@types/supertest": "^7.0.0",
|
||||
"@vitest/coverage-v8": "^4.1.2",
|
||||
"dotenv": "^18.0.5",
|
||||
"nestjs-spelunker": "^1.3.2",
|
||||
"oxlint": "^1.58.0",
|
||||
"prettier": "^3.4.2",
|
||||
"source-map-support": "^0.5.21",
|
||||
"supertest": "^7.2.2",
|
||||
"tsx": "^4.23.15",
|
||||
"typescript": "^6.0.3",
|
||||
"vite-tsconfig-paths": "^5.1.4",
|
||||
"vitest": "^4.1.2"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,123 @@
|
||||
import { mkdir, writeFile } from 'node:fs/promises';
|
||||
import 'reflect-metadata';
|
||||
import { SpelunkerModule, type DebuggedTree } from 'nestjs-spelunker';
|
||||
import { AppModule } from '../dist/app.module.js';
|
||||
|
||||
function mermaidId(name: string): string {
|
||||
return name.replace(/[^a-zA-Z0-9_]/gu, '_');
|
||||
}
|
||||
|
||||
function findDependencyModule(
|
||||
tree: DebuggedTree[],
|
||||
dependency: string,
|
||||
): DebuggedTree | undefined {
|
||||
return tree.find(
|
||||
(candidate) =>
|
||||
candidate.providers.some((item) => item.name === dependency) ||
|
||||
candidate.controllers.some((item) => item.name === dependency),
|
||||
);
|
||||
}
|
||||
|
||||
function renderMermaid(tree: DebuggedTree[]): string {
|
||||
const lines = [
|
||||
'flowchart LR',
|
||||
' %% Generated by scripts/graph-modules.ts',
|
||||
' %% Solid arrows: module imports; dotted arrows: constructor dependencies',
|
||||
];
|
||||
const moduleNames = tree.map((module) => module.name);
|
||||
|
||||
for (const module of tree) {
|
||||
const moduleId = mermaidId(module.name);
|
||||
lines.push(` subgraph ${moduleId}["${module.name}"]`);
|
||||
for (const provider of module.providers) {
|
||||
lines.push(
|
||||
` ${moduleId}_${mermaidId(provider.name)}["provider: ${provider.name}"]`,
|
||||
);
|
||||
}
|
||||
for (const controller of module.controllers) {
|
||||
lines.push(
|
||||
` ${moduleId}_${mermaidId(controller.name)}["controller: ${controller.name}"]`,
|
||||
);
|
||||
}
|
||||
lines.push(' end');
|
||||
}
|
||||
|
||||
for (const module of tree) {
|
||||
for (const imported of module.imports) {
|
||||
if (moduleNames.includes(imported)) {
|
||||
lines.push(
|
||||
` ${mermaidId(module.name)} -->|imports| ${mermaidId(imported)}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
for (const item of [...module.providers, ...module.controllers]) {
|
||||
for (const dependency of item.dependencies) {
|
||||
const targetModule = findDependencyModule(tree, dependency);
|
||||
if (targetModule) {
|
||||
const source = `${mermaidId(module.name)}_${mermaidId(item.name)}`;
|
||||
const target = `${mermaidId(targetModule.name)}_${mermaidId(dependency)}`;
|
||||
lines.push(` ${source} -.->|injects| ${target}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return `${lines.join('\n')}\n`;
|
||||
}
|
||||
|
||||
function renderMarkdown(tree: DebuggedTree[], mermaid: string): string {
|
||||
const lines = [
|
||||
'# NestJS dependency graph',
|
||||
'',
|
||||
'Generated from `AppModule` with `nestjs-spelunker` static module debugging.',
|
||||
'',
|
||||
'- Solid arrows: module imports.',
|
||||
'- Dotted arrows: constructor dependencies.',
|
||||
'- The generator does not boot PostgreSQL or Redis.',
|
||||
'',
|
||||
'```mermaid',
|
||||
mermaid.trim(),
|
||||
'```',
|
||||
'',
|
||||
'## Module details',
|
||||
'',
|
||||
];
|
||||
|
||||
for (const module of tree) {
|
||||
lines.push(
|
||||
`### ${module.name}`,
|
||||
'',
|
||||
`- Imports: ${module.imports.join(', ') || 'none'}`,
|
||||
);
|
||||
lines.push(
|
||||
`- Exports: ${module.exports.map((item) => `${item.name} (${item.type})`).join(', ') || 'none'}`,
|
||||
);
|
||||
lines.push('', '| Kind | Name | Dependencies |', '| --- | --- | --- |');
|
||||
for (const provider of module.providers) {
|
||||
lines.push(
|
||||
`| provider | ${provider.name} | ${provider.dependencies.join(', ') || 'none'} |`,
|
||||
);
|
||||
}
|
||||
for (const controller of module.controllers) {
|
||||
lines.push(
|
||||
`| controller | ${controller.name} | ${controller.dependencies.join(', ') || 'none'} |`,
|
||||
);
|
||||
}
|
||||
lines.push('');
|
||||
}
|
||||
|
||||
return lines.join('\n');
|
||||
}
|
||||
|
||||
const tree = await SpelunkerModule.debug(AppModule);
|
||||
const mermaid = renderMermaid(tree);
|
||||
await mkdir('../docs/architecture', { recursive: true });
|
||||
await writeFile(
|
||||
'../docs/architecture/nest-module-details.json',
|
||||
JSON.stringify(tree, null, 2),
|
||||
);
|
||||
await writeFile('../docs/architecture/nest-module-graph.mmd', mermaid);
|
||||
await writeFile(
|
||||
'../docs/architecture/nest-dependency-graph.md',
|
||||
renderMarkdown(tree, mermaid),
|
||||
);
|
||||
@@ -0,0 +1,35 @@
|
||||
import { Module } from '@nestjs/common';
|
||||
import { AppConfigModule } from './config/app-config.module.js';
|
||||
import { DatabaseModule } from './database/database.module.js';
|
||||
import { RedisModule } from './redis/redis.module.js';
|
||||
import { AuthModule } from './auth/auth.module.js';
|
||||
import { ChatsModule } from './chats/chats.module.js';
|
||||
import { HealthModule } from './health/health.module.js';
|
||||
import { AttachmentsModule } from './attachments/attachments.module.js';
|
||||
import { MessagesModule } from './messages/messages.module.js';
|
||||
import { MetricsModule } from './metrics/metrics.module.js';
|
||||
import { UsersModule } from './users/users.module.js';
|
||||
import { APP_FILTER } from '@nestjs/core';
|
||||
import { ApiExceptionFilter } from './common/errors/api-exception.filter.js';
|
||||
|
||||
@Module({
|
||||
imports: [
|
||||
AppConfigModule,
|
||||
DatabaseModule,
|
||||
RedisModule,
|
||||
UsersModule,
|
||||
AuthModule,
|
||||
HealthModule,
|
||||
ChatsModule,
|
||||
MessagesModule,
|
||||
AttachmentsModule,
|
||||
MetricsModule,
|
||||
],
|
||||
providers: [
|
||||
{
|
||||
provide: APP_FILTER,
|
||||
useClass: ApiExceptionFilter,
|
||||
},
|
||||
],
|
||||
})
|
||||
export class AppModule {}
|
||||
@@ -0,0 +1,80 @@
|
||||
import {
|
||||
Injectable,
|
||||
Logger,
|
||||
OnModuleDestroy,
|
||||
OnModuleInit,
|
||||
} from '@nestjs/common';
|
||||
import { ConfigService } from '@nestjs/config';
|
||||
import type { AppConfig } from '../config/app-config.schema.js';
|
||||
import { MetricsService } from '../metrics/metrics.service.js';
|
||||
import { ObjectStorage } from '../storage/object-storage.js';
|
||||
import { AttachmentsRepository } from './attachments.repository.js';
|
||||
|
||||
const SWEEP_BATCH = 100;
|
||||
|
||||
/**
|
||||
* Removes unfinished uploads (pending past expiry + grace) and rows whose
|
||||
* deletion was claimed but never finished. A live upload is never touched:
|
||||
* its expiry is a hard request deadline and the grace period is far longer.
|
||||
*/
|
||||
@Injectable()
|
||||
export class AttachmentSweeper implements OnModuleInit, OnModuleDestroy {
|
||||
private readonly logger = new Logger(AttachmentSweeper.name);
|
||||
private timer: NodeJS.Timeout | undefined;
|
||||
private running = false;
|
||||
|
||||
constructor(
|
||||
private readonly config: ConfigService<AppConfig>,
|
||||
private readonly repository: AttachmentsRepository,
|
||||
private readonly storage: ObjectStorage,
|
||||
private readonly metrics: MetricsService,
|
||||
) {}
|
||||
|
||||
onModuleInit(): void {
|
||||
const interval = this.config.getOrThrow('ATTACHMENT_SWEEP_INTERVAL_MS');
|
||||
if (interval <= 0) return;
|
||||
this.timer = setInterval(() => {
|
||||
if (this.running) return;
|
||||
this.running = true;
|
||||
this.sweepOnce()
|
||||
.catch((error: unknown) => {
|
||||
this.logger.warn(
|
||||
`Sweep failed: ${error instanceof Error ? error.message : 'unknown error'}`,
|
||||
);
|
||||
})
|
||||
.finally(() => {
|
||||
this.running = false;
|
||||
});
|
||||
}, interval);
|
||||
this.timer.unref();
|
||||
}
|
||||
|
||||
onModuleDestroy(): void {
|
||||
if (this.timer) clearInterval(this.timer);
|
||||
this.timer = undefined;
|
||||
}
|
||||
|
||||
async sweepOnce(): Promise<{ claimed: number; deleted: number }> {
|
||||
const claimed = await this.repository.claimExpired(
|
||||
this.config.getOrThrow('ATTACHMENT_SWEEP_GRACE_SECONDS'),
|
||||
SWEEP_BATCH,
|
||||
);
|
||||
let deleted = 0;
|
||||
for (const item of claimed) {
|
||||
try {
|
||||
// New uploads live in staging; lab4 rows have no staging key and
|
||||
// their final object is the only copy.
|
||||
await this.storage.delete(item.staging_key ?? item.object_key);
|
||||
await this.repository.deleteClaimed(item.id);
|
||||
deleted += 1;
|
||||
this.metrics.increment('attachments.sweep.deleted');
|
||||
} catch (error) {
|
||||
this.metrics.increment('attachments.sweep.error');
|
||||
this.logger.warn(
|
||||
`Sweep of attachment ${item.id} failed: ${error instanceof Error ? error.message : 'unknown error'}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
return { claimed: claimed.length, deleted };
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,109 @@
|
||||
export type AttachmentStatus = 'queued' | 'processing' | 'ready' | 'failed';
|
||||
export type VariantName = 'thumbnail' | 'crop' | 'optimized';
|
||||
|
||||
export const VARIANT_ORDER: VariantName[] = ['thumbnail', 'crop', 'optimized'];
|
||||
|
||||
export type Variant = {
|
||||
name: VariantName;
|
||||
content_type: 'image/jpeg' | 'image/png' | 'image/webp';
|
||||
size_bytes: number;
|
||||
width: number;
|
||||
height: number;
|
||||
sha256: string;
|
||||
};
|
||||
|
||||
export type Attachment = {
|
||||
id: string;
|
||||
chat_id: string;
|
||||
owner_id: string;
|
||||
filename: string;
|
||||
content_type: 'image/png' | 'image/jpeg';
|
||||
size_bytes: number;
|
||||
sha256: string;
|
||||
status: AttachmentStatus;
|
||||
variants: Variant[];
|
||||
error_code: string | null;
|
||||
created_at: string;
|
||||
updated_at: string;
|
||||
};
|
||||
|
||||
export type AttachmentRow = {
|
||||
id: string;
|
||||
chat_id: string;
|
||||
owner_id: string;
|
||||
filename: string;
|
||||
content_type: 'image/png' | 'image/jpeg';
|
||||
upload_mode: 'proxy' | 'direct';
|
||||
status:
|
||||
| 'pending'
|
||||
| 'queued'
|
||||
| 'processing'
|
||||
| 'ready'
|
||||
| 'failed'
|
||||
| 'deleting'
|
||||
| 'rejected';
|
||||
object_key: string;
|
||||
staging_key: string | null;
|
||||
size_bytes: string | number | null;
|
||||
sha256: string | null;
|
||||
width: number | null;
|
||||
height: number | null;
|
||||
error_code: string | null;
|
||||
ready_pipeline_version: number | null;
|
||||
expires_at: Date | null;
|
||||
created_at: Date;
|
||||
updated_at: Date;
|
||||
};
|
||||
|
||||
export type VariantRow = {
|
||||
attachment_id: string;
|
||||
pipeline_version: number;
|
||||
variant: VariantName;
|
||||
object_key: string;
|
||||
content_type: 'image/jpeg' | 'image/png' | 'image/webp';
|
||||
size_bytes: string | number;
|
||||
width: number;
|
||||
height: number;
|
||||
sha256: string;
|
||||
job_id: string;
|
||||
created_at: Date;
|
||||
};
|
||||
|
||||
function toVariant(row: VariantRow): Variant {
|
||||
return {
|
||||
name: row.variant,
|
||||
content_type: row.content_type,
|
||||
size_bytes: Number(row.size_bytes),
|
||||
width: row.width,
|
||||
height: row.height,
|
||||
sha256: row.sha256,
|
||||
};
|
||||
}
|
||||
|
||||
export function toAttachment(
|
||||
row: AttachmentRow,
|
||||
variants: VariantRow[] = [],
|
||||
): Attachment {
|
||||
const ordered =
|
||||
row.status === 'ready'
|
||||
? [...variants].sort(
|
||||
(left, right) =>
|
||||
VARIANT_ORDER.indexOf(left.variant) -
|
||||
VARIANT_ORDER.indexOf(right.variant),
|
||||
)
|
||||
: [];
|
||||
return {
|
||||
id: row.id,
|
||||
chat_id: row.chat_id,
|
||||
owner_id: row.owner_id,
|
||||
filename: row.filename,
|
||||
content_type: row.content_type,
|
||||
size_bytes: Number(row.size_bytes),
|
||||
sha256: row.sha256 as string,
|
||||
status: row.status as AttachmentStatus,
|
||||
variants: ordered.map(toVariant),
|
||||
error_code: row.status === 'failed' ? row.error_code : null,
|
||||
created_at: row.created_at.toISOString(),
|
||||
updated_at: row.updated_at.toISOString(),
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,42 @@
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import { parseAppConfig } from '../config/app-config.schema.js';
|
||||
import { DatabaseClient } from '../database/database.client.js';
|
||||
import { MetricsService } from '../metrics/metrics.service.js';
|
||||
import { ObjectStorage } from '../storage/object-storage.js';
|
||||
import { AttachmentSweeper } from './attachment-sweeper.service.js';
|
||||
import { AttachmentsRepository } from './attachments.repository.js';
|
||||
|
||||
// Manual operator pass over unfinished uploads (the API also sweeps in the
|
||||
// background). Never touches rows before expiry + ATTACHMENT_SWEEP_GRACE_SECONDS.
|
||||
// in the stand: docker compose exec api node dist/attachments/attachments-sweep.cli.js
|
||||
// on the host: npm run attachments:sweep (needs reachable DATABASE_URL and S3)
|
||||
async function main(): Promise<void> {
|
||||
const config = parseAppConfig(process.env as Record<string, unknown>);
|
||||
const configService = {
|
||||
getOrThrow: (key: keyof typeof config) => config[key],
|
||||
} as never;
|
||||
const metrics = new MetricsService();
|
||||
const database = new DatabaseClient(configService, metrics);
|
||||
try {
|
||||
const sweeper = new AttachmentSweeper(
|
||||
configService,
|
||||
new AttachmentsRepository(database),
|
||||
new ObjectStorage(configService),
|
||||
metrics,
|
||||
);
|
||||
let total = 0;
|
||||
for (;;) {
|
||||
const { claimed, deleted } = await sweeper.sweepOnce();
|
||||
total += deleted;
|
||||
// A pass that claimed rows but could not delete any would loop forever.
|
||||
if (claimed === 0 || deleted === 0) break;
|
||||
}
|
||||
process.stdout.write(
|
||||
`Swept ${total} attachment(s) older than ${config.ATTACHMENT_SWEEP_GRACE_SECONDS} seconds past expiry\n`,
|
||||
);
|
||||
} finally {
|
||||
await database.onModuleDestroy();
|
||||
}
|
||||
}
|
||||
|
||||
if (process.argv[1] === fileURLToPath(import.meta.url)) await main();
|
||||
@@ -0,0 +1,109 @@
|
||||
import {
|
||||
Body,
|
||||
Controller,
|
||||
Get,
|
||||
Headers,
|
||||
HttpCode,
|
||||
Param,
|
||||
ParseUUIDPipe,
|
||||
Post,
|
||||
Query,
|
||||
Req,
|
||||
Res,
|
||||
StreamableFile,
|
||||
UseGuards,
|
||||
} from '@nestjs/common';
|
||||
import { ConfigService } from '@nestjs/config';
|
||||
import type { FastifyReply, FastifyRequest } from 'fastify';
|
||||
import type { Readable } from 'node:stream';
|
||||
import { CurrentUser } from '../common/auth/current-user.decorator.js';
|
||||
import { CurrentUserGuard } from '../common/auth/current-user.guard.js';
|
||||
import type { AppConfig } from '../config/app-config.schema.js';
|
||||
import type { User } from '../users/user.js';
|
||||
import { AttachmentsService } from './attachments.service.js';
|
||||
import { CreateDirectUploadDto } from './dto/create-direct-upload.dto.js';
|
||||
import { DirectUploadsService } from './direct-uploads.service.js';
|
||||
import type { UploadRequest } from './upload-request.js';
|
||||
import { contentDisposition } from './filename.js';
|
||||
|
||||
@Controller('api/v1')
|
||||
@UseGuards(CurrentUserGuard)
|
||||
export class AttachmentsController {
|
||||
private readonly maxUploadBytes: number;
|
||||
|
||||
constructor(
|
||||
private readonly attachments: AttachmentsService,
|
||||
private readonly directUploads: DirectUploadsService,
|
||||
config: ConfigService<AppConfig>,
|
||||
) {
|
||||
this.maxUploadBytes = config.getOrThrow('MAX_UPLOAD_BYTES');
|
||||
}
|
||||
|
||||
@Post('chats/:chatId/attachments')
|
||||
@HttpCode(202)
|
||||
upload(
|
||||
@Param('chatId', new ParseUUIDPipe()) chatId: string,
|
||||
@CurrentUser() user: User,
|
||||
@Query('filename') filename: unknown,
|
||||
@Headers('content-length') contentLength: string | undefined,
|
||||
@Req() request: FastifyRequest & UploadRequest,
|
||||
@Res({ passthrough: true }) reply: FastifyReply,
|
||||
) {
|
||||
// A gigantic body is not worth draining: close the connection after the answer.
|
||||
if (Number(contentLength) > 2 * this.maxUploadBytes)
|
||||
reply.header('Connection', 'close');
|
||||
return this.attachments.uploadProxy(
|
||||
chatId,
|
||||
user,
|
||||
filename,
|
||||
request.uploadContentType,
|
||||
contentLength,
|
||||
request.body as Readable | undefined,
|
||||
);
|
||||
}
|
||||
|
||||
@Post('chats/:chatId/attachments/direct-uploads')
|
||||
@HttpCode(201)
|
||||
initDirectUpload(
|
||||
@Param('chatId', new ParseUUIDPipe()) chatId: string,
|
||||
@CurrentUser() user: User,
|
||||
@Body() body: CreateDirectUploadDto,
|
||||
) {
|
||||
return this.directUploads.init(chatId, user, body);
|
||||
}
|
||||
|
||||
@Post('attachments/:attachmentId/complete')
|
||||
@HttpCode(200)
|
||||
completeDirectUpload(
|
||||
@Param('attachmentId', new ParseUUIDPipe()) attachmentId: string,
|
||||
@CurrentUser() user: User,
|
||||
) {
|
||||
return this.directUploads.complete(attachmentId, user);
|
||||
}
|
||||
|
||||
@Get('attachments/:attachmentId')
|
||||
metadata(
|
||||
@Param('attachmentId', new ParseUUIDPipe()) attachmentId: string,
|
||||
@CurrentUser() user: User,
|
||||
) {
|
||||
return this.attachments.getMetadata(attachmentId, user);
|
||||
}
|
||||
|
||||
@Get('attachments/:attachmentId/content')
|
||||
async content(
|
||||
@Param('attachmentId', new ParseUUIDPipe()) attachmentId: string,
|
||||
@CurrentUser() user: User,
|
||||
@Res({ passthrough: true }) reply: FastifyReply,
|
||||
@Query('variant') variant?: string,
|
||||
) {
|
||||
const { row, contentType, sizeBytes, stream } =
|
||||
await this.attachments.openContent(attachmentId, user, variant);
|
||||
reply.header('X-Content-Type-Options', 'nosniff');
|
||||
reply.header('Cache-Control', 'private, no-store');
|
||||
return new StreamableFile(stream, {
|
||||
type: contentType,
|
||||
length: sizeBytes,
|
||||
disposition: contentDisposition(row.filename),
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,42 @@
|
||||
import { Module } from '@nestjs/common';
|
||||
import { DatabaseModule } from '../database/database.module.js';
|
||||
import { ChatsModule } from '../chats/chats.module.js';
|
||||
import { UsersModule } from '../users/users.module.js';
|
||||
import { RedisModule } from '../redis/redis.module.js';
|
||||
import { MetricsModule } from '../metrics/metrics.module.js';
|
||||
import { StorageModule } from '../storage/storage.module.js';
|
||||
import { JobsModule } from '../jobs/jobs.module.js';
|
||||
import { OutboxRelay } from '../jobs/outbox-relay.service.js';
|
||||
import { AttachmentSweeper } from './attachment-sweeper.service.js';
|
||||
import { AttachmentsController } from './attachments.controller.js';
|
||||
import { AttachmentsRepository } from './attachments.repository.js';
|
||||
import { ImageDecoder } from './image/image-decoder.js';
|
||||
import { UploadSlots } from './upload-slots.js';
|
||||
import { DirectUploadsService } from './direct-uploads.service.js';
|
||||
import { AttachmentsService } from './attachments.service.js';
|
||||
|
||||
@Module({
|
||||
imports: [
|
||||
DatabaseModule,
|
||||
ChatsModule,
|
||||
UsersModule,
|
||||
RedisModule,
|
||||
MetricsModule,
|
||||
StorageModule,
|
||||
JobsModule,
|
||||
],
|
||||
controllers: [AttachmentsController],
|
||||
providers: [
|
||||
AttachmentsRepository,
|
||||
AttachmentsService,
|
||||
DirectUploadsService,
|
||||
ImageDecoder,
|
||||
UploadSlots,
|
||||
AttachmentSweeper,
|
||||
// The relay runs only in the API process; the worker never imports
|
||||
// AttachmentsModule.
|
||||
OutboxRelay,
|
||||
],
|
||||
exports: [AttachmentsRepository],
|
||||
})
|
||||
export class AttachmentsModule {}
|
||||
@@ -0,0 +1,216 @@
|
||||
import { Injectable } from '@nestjs/common';
|
||||
import { DatabaseClient } from '../database/database.client.js';
|
||||
import type { AttachmentRow } from './attachment.js';
|
||||
|
||||
export type InsertPending = {
|
||||
id: string;
|
||||
chatId: string;
|
||||
ownerId: string;
|
||||
filename: string;
|
||||
contentType: string;
|
||||
uploadMode: 'proxy' | 'direct';
|
||||
objectKey: string;
|
||||
stagingKey: string | null;
|
||||
sizeBytes: number | null;
|
||||
sha256: string | null;
|
||||
ttlMs: number;
|
||||
};
|
||||
|
||||
const VISIBLE_MEMBER = `JOIN chat_members m ON m.chat_id = a.chat_id AND m.user_id = $2`;
|
||||
|
||||
const INSERT_PENDING_SQL = `INSERT INTO attachments
|
||||
(id, chat_id, owner_id, filename, content_type, upload_mode, status,
|
||||
object_key, staging_key, size_bytes, sha256, expires_at)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, 'pending', $7, $8, $9, $10,
|
||||
now() + make_interval(secs => $11::double precision / 1000.0))`;
|
||||
|
||||
@Injectable()
|
||||
export class AttachmentsRepository {
|
||||
constructor(private readonly database: DatabaseClient) {}
|
||||
|
||||
private values(input: InsertPending): unknown[] {
|
||||
return [
|
||||
input.id,
|
||||
input.chatId,
|
||||
input.ownerId,
|
||||
input.filename,
|
||||
input.contentType,
|
||||
input.uploadMode,
|
||||
input.objectKey,
|
||||
input.stagingKey,
|
||||
input.sizeBytes,
|
||||
input.sha256,
|
||||
input.ttlMs,
|
||||
];
|
||||
}
|
||||
|
||||
async insertPending(input: InsertPending): Promise<void> {
|
||||
await this.database.query(
|
||||
'attachments.insert_pending',
|
||||
INSERT_PENDING_SQL,
|
||||
this.values(input),
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Serializes the per-owner pending check with the insert. Two parallel
|
||||
* `init` calls cannot both pass the limit.
|
||||
*/
|
||||
async insertPendingLimited(
|
||||
input: InsertPending,
|
||||
maxPending: number,
|
||||
): Promise<boolean> {
|
||||
return this.database.withTransaction(async (client) => {
|
||||
await client.query(
|
||||
`SELECT pg_advisory_xact_lock(hashtext($1)::bigint)`,
|
||||
[input.ownerId],
|
||||
);
|
||||
const count = await client.query<{ count: string }>(
|
||||
`SELECT count(*) AS count FROM attachments
|
||||
WHERE owner_id = $1 AND status IN ('pending', 'rejected')
|
||||
AND upload_mode = 'direct' AND expires_at > now()`,
|
||||
[input.ownerId],
|
||||
);
|
||||
if (Number(count.rows[0].count) >= maxPending) return false;
|
||||
await client.query(INSERT_PENDING_SQL, this.values(input));
|
||||
return true;
|
||||
});
|
||||
}
|
||||
|
||||
async deletePending(id: string): Promise<void> {
|
||||
await this.database.query(
|
||||
'attachments.delete_pending',
|
||||
`DELETE FROM attachments WHERE id = $1 AND status = 'pending'`,
|
||||
[id],
|
||||
);
|
||||
}
|
||||
|
||||
async rejectPending(id: string): Promise<boolean> {
|
||||
const result = await this.database.query(
|
||||
'attachments.reject_pending',
|
||||
`UPDATE attachments SET status = 'rejected', updated_at = now()
|
||||
WHERE id = $1 AND status = 'pending'`,
|
||||
[id],
|
||||
);
|
||||
return result.rowCount === 1;
|
||||
}
|
||||
|
||||
/**
|
||||
* Takes a pending row out of play before its object is deleted. The object
|
||||
* is deleted only when this returns true: a row that is already `ready`
|
||||
* (its commit succeeded even if the reply was lost) keeps its object.
|
||||
*/
|
||||
async claimForCompensation(id: string): Promise<boolean> {
|
||||
const result = await this.database.query(
|
||||
'attachments.claim_for_compensation',
|
||||
`UPDATE attachments SET status = 'deleting', updated_at = now()
|
||||
WHERE id = $1 AND status = 'pending'`,
|
||||
[id],
|
||||
);
|
||||
return result.rowCount === 1;
|
||||
}
|
||||
|
||||
async findById(id: string): Promise<AttachmentRow | undefined> {
|
||||
const result = await this.database.query<AttachmentRow>(
|
||||
'attachments.find_by_id',
|
||||
'SELECT * FROM attachments WHERE id = $1',
|
||||
[id],
|
||||
);
|
||||
return result.rows[0];
|
||||
}
|
||||
|
||||
async findVisible(
|
||||
id: string,
|
||||
userId: string,
|
||||
): Promise<AttachmentRow | undefined> {
|
||||
const result = await this.database.query<AttachmentRow>(
|
||||
'attachments.find_visible',
|
||||
`SELECT a.* FROM attachments a ${VISIBLE_MEMBER}
|
||||
WHERE a.id = $1 AND a.status IN ('queued','processing','ready','failed')`,
|
||||
[id, userId],
|
||||
);
|
||||
return result.rows[0];
|
||||
}
|
||||
|
||||
async findForComplete(
|
||||
id: string,
|
||||
userId: string,
|
||||
): Promise<AttachmentRow | undefined> {
|
||||
const result = await this.database.query<AttachmentRow>(
|
||||
'attachments.find_for_complete',
|
||||
`SELECT a.* FROM attachments a ${VISIBLE_MEMBER}
|
||||
WHERE a.id = $1 AND a.upload_mode = 'direct' AND a.owner_id = $2
|
||||
AND a.status IN ('pending','queued','processing','ready','failed')`,
|
||||
[id, userId],
|
||||
);
|
||||
return result.rows[0];
|
||||
}
|
||||
|
||||
async countPendingDirect(ownerId: string): Promise<number> {
|
||||
const result = await this.database.query<{ count: string }>(
|
||||
'attachments.count_pending_direct',
|
||||
`SELECT count(*) AS count FROM attachments
|
||||
WHERE owner_id = $1 AND status IN ('pending', 'rejected')
|
||||
AND upload_mode = 'direct' AND expires_at > now()`,
|
||||
[ownerId],
|
||||
);
|
||||
return Number(result.rows[0].count);
|
||||
}
|
||||
|
||||
async linkState(
|
||||
chatId: string,
|
||||
ids: string[],
|
||||
): Promise<'ok' | 'failed' | 'missing'> {
|
||||
const result = await this.database.query<{
|
||||
linkable: string;
|
||||
failed: string;
|
||||
}>(
|
||||
'attachments.link_state',
|
||||
`SELECT count(*) FILTER (WHERE status IN ('queued','processing','ready')) AS linkable,
|
||||
count(*) FILTER (WHERE status = 'failed') AS failed
|
||||
FROM attachments
|
||||
WHERE chat_id = $1 AND id = ANY($2::uuid[])`,
|
||||
[chatId, ids],
|
||||
);
|
||||
const linkable = Number(result.rows[0]?.linkable ?? 0);
|
||||
const failed = Number(result.rows[0]?.failed ?? 0);
|
||||
if (linkable === ids.length) return 'ok';
|
||||
if (linkable + failed === ids.length) return 'failed';
|
||||
return 'missing';
|
||||
}
|
||||
|
||||
async claimExpired(
|
||||
graceSeconds: number,
|
||||
limit: number,
|
||||
): Promise<
|
||||
{ id: string; object_key: string; staging_key: string | null }[]
|
||||
> {
|
||||
const result = await this.database.query<{
|
||||
id: string;
|
||||
object_key: string;
|
||||
staging_key: string | null;
|
||||
}>(
|
||||
'attachments.claim_expired',
|
||||
`UPDATE attachments SET status = 'deleting', updated_at = now()
|
||||
WHERE id IN (
|
||||
SELECT id FROM attachments
|
||||
WHERE (status IN ('pending', 'rejected')
|
||||
AND expires_at < now() - make_interval(secs => $1::double precision))
|
||||
OR (status = 'deleting' AND updated_at < now() - interval '1 minute')
|
||||
ORDER BY expires_at NULLS FIRST
|
||||
LIMIT $2
|
||||
FOR UPDATE SKIP LOCKED)
|
||||
RETURNING id, object_key, staging_key`,
|
||||
[graceSeconds, limit],
|
||||
);
|
||||
return result.rows;
|
||||
}
|
||||
|
||||
async deleteClaimed(id: string): Promise<void> {
|
||||
await this.database.query(
|
||||
'attachments.delete_claimed',
|
||||
`DELETE FROM attachments WHERE id = $1 AND status = 'deleting'`,
|
||||
[id],
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,133 @@
|
||||
import { readFileSync } from 'node:fs';
|
||||
import { dirname, join } from 'node:path';
|
||||
import { Readable } from 'node:stream';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import { describe, expect, it, vi } from 'vitest';
|
||||
import type { AttachmentRow } from './attachment.js';
|
||||
import { AttachmentsService } from './attachments.service.js';
|
||||
|
||||
const here = dirname(fileURLToPath(import.meta.url));
|
||||
const png = readFileSync(join(here, '../../../tests/fixtures/sample.png'));
|
||||
const lostReply = () =>
|
||||
Object.assign(new Error('Connection terminated unexpectedly'), {});
|
||||
|
||||
const settings: Record<string, number> = {
|
||||
MAX_UPLOAD_BYTES: 10_485_760,
|
||||
MAX_IMAGE_PIXELS: 40_000_000,
|
||||
UPLOAD_TIMEOUT_MS: 10_000,
|
||||
PIPELINE_VERSION: 1,
|
||||
JOB_MAX_ATTEMPTS: 5,
|
||||
};
|
||||
|
||||
function setup(
|
||||
repository: Record<string, unknown>,
|
||||
jobs: Record<string, unknown> = {},
|
||||
) {
|
||||
const storage = {
|
||||
objectKey: (id: string) => `attachments/${id}/original`,
|
||||
stagingKey: (id: string) => `staging/${id}/original`,
|
||||
putStream: vi.fn(async (_key: string, body: Readable) => {
|
||||
for await (const _chunk of body) {
|
||||
// drain like S3 does
|
||||
}
|
||||
}),
|
||||
delete: vi.fn().mockResolvedValue(undefined),
|
||||
};
|
||||
const service = new AttachmentsService(
|
||||
{ getOrThrow: (key: string) => settings[key] } as never,
|
||||
{
|
||||
insertPending: vi.fn(),
|
||||
deleteClaimed: vi.fn(),
|
||||
...repository,
|
||||
} as never,
|
||||
{ acceptUpload: vi.fn(), ...jobs } as never,
|
||||
{ kick: vi.fn() } as never,
|
||||
{ point: vi.fn() } as never,
|
||||
storage as never,
|
||||
{ getForUser: vi.fn() } as never,
|
||||
{ increment: vi.fn(), add: vi.fn() } as never,
|
||||
{ verify: vi.fn() } as never,
|
||||
{
|
||||
run: (_user: string, work: () => Promise<unknown>) => work(),
|
||||
} as never,
|
||||
);
|
||||
const upload = () =>
|
||||
service.uploadProxy(
|
||||
'chat',
|
||||
{ id: 'user' } as never,
|
||||
'a.png',
|
||||
'image/png',
|
||||
String(png.length),
|
||||
Readable.from([png]),
|
||||
);
|
||||
return { storage, upload };
|
||||
}
|
||||
|
||||
const readyRow = (id: string): AttachmentRow => ({
|
||||
id,
|
||||
chat_id: 'chat',
|
||||
owner_id: 'user',
|
||||
filename: 'a.png',
|
||||
content_type: 'image/png',
|
||||
upload_mode: 'proxy',
|
||||
status: 'ready',
|
||||
object_key: `attachments/${id}/original`,
|
||||
staging_key: `staging/${id}/original`,
|
||||
size_bytes: png.length,
|
||||
sha256: 'a'.repeat(64),
|
||||
width: 320,
|
||||
height: 200,
|
||||
error_code: null,
|
||||
ready_pipeline_version: 1,
|
||||
expires_at: null,
|
||||
created_at: new Date(),
|
||||
updated_at: new Date(),
|
||||
});
|
||||
|
||||
describe('AttachmentsService compensation', () => {
|
||||
it('keeps the object and answers 202 when the job commit reply was lost', async () => {
|
||||
const { storage, upload } = setup(
|
||||
{
|
||||
claimForCompensation: vi.fn().mockResolvedValue(false),
|
||||
findById: vi.fn((id: string) => Promise.resolve(readyRow(id))),
|
||||
},
|
||||
{ acceptUpload: vi.fn().mockRejectedValue(lostReply()) },
|
||||
);
|
||||
const attachment = await upload();
|
||||
expect(attachment.status).toBe('ready');
|
||||
expect(storage.delete).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('deletes nothing while PostgreSQL is unreachable', async () => {
|
||||
const { storage, upload } = setup(
|
||||
{
|
||||
claimForCompensation: vi.fn().mockRejectedValue(lostReply()),
|
||||
findById: vi.fn().mockRejectedValue(lostReply()),
|
||||
},
|
||||
{ acceptUpload: vi.fn().mockRejectedValue(lostReply()) },
|
||||
);
|
||||
await expect(upload()).rejects.toThrow();
|
||||
expect(storage.delete).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('deletes the object only after claiming the pending row', async () => {
|
||||
const order: string[] = [];
|
||||
const { storage, upload } = setup(
|
||||
{
|
||||
claimForCompensation: vi.fn(async () => {
|
||||
order.push('claim');
|
||||
return true;
|
||||
}),
|
||||
deleteClaimed: vi.fn(async () => {
|
||||
order.push('delete-row');
|
||||
}),
|
||||
},
|
||||
{ acceptUpload: vi.fn().mockRejectedValue(lostReply()) },
|
||||
);
|
||||
storage.delete.mockImplementation(async () => {
|
||||
order.push('delete-object');
|
||||
});
|
||||
await expect(upload()).rejects.toThrow();
|
||||
expect(order).toEqual(['claim', 'delete-object', 'delete-row']);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,435 @@
|
||||
import {
|
||||
BadRequestException,
|
||||
ConflictException,
|
||||
HttpException,
|
||||
Injectable,
|
||||
InternalServerErrorException,
|
||||
Logger,
|
||||
NotFoundException,
|
||||
PayloadTooLargeException,
|
||||
ServiceUnavailableException,
|
||||
UnsupportedMediaTypeException,
|
||||
} from '@nestjs/common';
|
||||
import { ConfigService } from '@nestjs/config';
|
||||
import { createHash, randomUUID } from 'node:crypto';
|
||||
import { pipeline, Readable, Transform } from 'node:stream';
|
||||
import type { AppConfig } from '../config/app-config.schema.js';
|
||||
import { ChatsService } from '../chats/chats.service.js';
|
||||
import { FaultInjector } from '../jobs/fault-injector.js';
|
||||
import { JobsRepository } from '../jobs/jobs.repository.js';
|
||||
import { OutboxRelay } from '../jobs/outbox-relay.service.js';
|
||||
import { MetricsService } from '../metrics/metrics.service.js';
|
||||
import { ObjectStorage } from '../storage/object-storage.js';
|
||||
import type { User } from '../users/user.js';
|
||||
import {
|
||||
toAttachment,
|
||||
type Attachment,
|
||||
type AttachmentRow,
|
||||
} from './attachment.js';
|
||||
import { validateFilename } from './filename.js';
|
||||
import {
|
||||
ImageHeader,
|
||||
ImageInspector,
|
||||
ImageRejectedError,
|
||||
} from './image/image-inspector.js';
|
||||
import { AttachmentsRepository } from './attachments.repository.js';
|
||||
import { ImageDecoder } from './image/image-decoder.js';
|
||||
import { UploadSlots } from './upload-slots.js';
|
||||
|
||||
const MEDIA_TYPES = new Set(['image/png', 'image/jpeg']);
|
||||
|
||||
export function mediaType(header: unknown): string | undefined {
|
||||
if (typeof header !== 'string') return undefined;
|
||||
return header.split(';', 1)[0].trim().toLowerCase();
|
||||
}
|
||||
|
||||
export function imageRejection(error: ImageRejectedError): HttpException {
|
||||
return error.reason === 'too_many_pixels'
|
||||
? new PayloadTooLargeException(error.message)
|
||||
: new UnsupportedMediaTypeException(error.message);
|
||||
}
|
||||
|
||||
class UploadInterruptedError extends Error {}
|
||||
|
||||
@Injectable()
|
||||
export class AttachmentsService {
|
||||
private readonly logger = new Logger(AttachmentsService.name);
|
||||
private readonly maxUploadBytes: number;
|
||||
private readonly maxPixels: number;
|
||||
private readonly uploadTimeoutMs: number;
|
||||
private readonly pipelineVersion: number;
|
||||
private readonly maxAttempts: number;
|
||||
|
||||
constructor(
|
||||
config: ConfigService<AppConfig>,
|
||||
private readonly repository: AttachmentsRepository,
|
||||
private readonly jobs: JobsRepository,
|
||||
private readonly relay: OutboxRelay,
|
||||
private readonly faults: FaultInjector,
|
||||
private readonly storage: ObjectStorage,
|
||||
private readonly chats: ChatsService,
|
||||
private readonly metrics: MetricsService,
|
||||
private readonly decoder: ImageDecoder,
|
||||
private readonly slots: UploadSlots,
|
||||
) {
|
||||
this.maxUploadBytes = config.getOrThrow('MAX_UPLOAD_BYTES');
|
||||
this.maxPixels = config.getOrThrow('MAX_IMAGE_PIXELS');
|
||||
this.uploadTimeoutMs = config.getOrThrow('UPLOAD_TIMEOUT_MS');
|
||||
this.pipelineVersion = config.getOrThrow('PIPELINE_VERSION');
|
||||
this.maxAttempts = config.getOrThrow('JOB_MAX_ATTEMPTS');
|
||||
}
|
||||
|
||||
async uploadProxy(
|
||||
chatId: string,
|
||||
user: User,
|
||||
rawFilename: unknown,
|
||||
contentTypeHeader: unknown,
|
||||
contentLengthHeader: unknown,
|
||||
body: Readable | undefined,
|
||||
): Promise<Attachment> {
|
||||
const filename = validateFilename(rawFilename);
|
||||
const declared = mediaType(contentTypeHeader);
|
||||
if (declared === undefined || !MEDIA_TYPES.has(declared))
|
||||
throw new UnsupportedMediaTypeException(
|
||||
'Content-Type must be image/png or image/jpeg',
|
||||
);
|
||||
if (
|
||||
typeof contentLengthHeader !== 'string' ||
|
||||
!/^\d{1,15}$/u.test(contentLengthHeader)
|
||||
)
|
||||
throw new BadRequestException('Content-Length is required');
|
||||
const contentLength = Number(contentLengthHeader);
|
||||
if (contentLength === 0 || body === undefined)
|
||||
throw new UnsupportedMediaTypeException('Image body is empty');
|
||||
if (contentLength > this.maxUploadBytes)
|
||||
throw new PayloadTooLargeException('Image is too large');
|
||||
|
||||
return this.slots.run(user.id, async () => {
|
||||
await this.chats.getForUser(chatId, user);
|
||||
return this.receive(
|
||||
chatId,
|
||||
user,
|
||||
filename,
|
||||
declared,
|
||||
contentLength,
|
||||
body,
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
private async receive(
|
||||
chatId: string,
|
||||
user: User,
|
||||
filename: string,
|
||||
declared: string,
|
||||
contentLength: number,
|
||||
body: Readable,
|
||||
): Promise<Attachment> {
|
||||
const controller = new AbortController();
|
||||
let timedOut = false;
|
||||
const timer = setTimeout(() => {
|
||||
timedOut = true;
|
||||
body.destroy(new Error('upload timeout'));
|
||||
controller.abort();
|
||||
}, this.uploadTimeoutMs);
|
||||
const iterator = body[Symbol.asyncIterator]() as AsyncIterator<Buffer>;
|
||||
const next = async (): Promise<IteratorResult<Buffer>> => {
|
||||
try {
|
||||
return await iterator.next();
|
||||
} catch {
|
||||
throw new UploadInterruptedError('Upload was interrupted');
|
||||
}
|
||||
};
|
||||
|
||||
const inspector = new ImageInspector(this.maxPixels);
|
||||
const head: Buffer[] = [];
|
||||
let id: string | undefined;
|
||||
let key = '';
|
||||
let rejection: unknown;
|
||||
try {
|
||||
// Header phase: nothing exists in the database or S3 yet. It is
|
||||
// bounded by Content-Length (<= MAX_UPLOAD_BYTES): a JPEG may carry
|
||||
// megabytes of APPn metadata before its frame header.
|
||||
try {
|
||||
while (inspector.header === undefined) {
|
||||
const step = await next();
|
||||
if (step.done) {
|
||||
inspector.finish();
|
||||
throw new ImageRejectedError(
|
||||
'unsupported_media_type',
|
||||
'Image header is missing',
|
||||
);
|
||||
}
|
||||
head.push(step.value);
|
||||
inspector.push(step.value);
|
||||
}
|
||||
} catch (error) {
|
||||
throw this.mapUploadError(error, timedOut);
|
||||
}
|
||||
const header = inspector.header as ImageHeader;
|
||||
if (header.contentType !== declared)
|
||||
throw new UnsupportedMediaTypeException(
|
||||
'Image content does not match Content-Type',
|
||||
);
|
||||
|
||||
id = randomUUID();
|
||||
key = this.storage.stagingKey(id);
|
||||
await this.repository.insertPending({
|
||||
id,
|
||||
chatId,
|
||||
ownerId: user.id,
|
||||
filename,
|
||||
contentType: declared,
|
||||
uploadMode: 'proxy',
|
||||
objectKey: this.storage.objectKey(id),
|
||||
stagingKey: key,
|
||||
sizeBytes: null,
|
||||
sha256: null,
|
||||
ttlMs: this.uploadTimeoutMs,
|
||||
});
|
||||
|
||||
const sha256 = createHash('sha256');
|
||||
const decoder = this.decoder;
|
||||
// Copy for the full decode check; bounded by MAX_UPLOAD_BYTES and
|
||||
// UPLOAD_MAX_IN_FLIGHT, released as soon as the decode starts.
|
||||
const copy: Buffer[] = [];
|
||||
let total = 0;
|
||||
let allSent = false;
|
||||
const account = (chunk: Buffer, inspect: boolean): boolean => {
|
||||
total += chunk.length;
|
||||
if (total > contentLength)
|
||||
throw new UploadInterruptedError(
|
||||
'Body exceeds Content-Length',
|
||||
);
|
||||
if (inspect) inspector.push(chunk);
|
||||
sha256.update(chunk);
|
||||
this.metrics.add('attachments.proxy.bytes_in', chunk.length);
|
||||
copy.push(chunk);
|
||||
return total === contentLength;
|
||||
};
|
||||
// Validate before the last bytes reach S3: it cannot commit an object
|
||||
// whose body is incomplete, so an invalid image never becomes an object.
|
||||
const finalize = async (): Promise<void> => {
|
||||
inspector.finish();
|
||||
const data = Buffer.concat(copy, total);
|
||||
copy.length = 0;
|
||||
await decoder.verify(data, controller.signal);
|
||||
// From here S3 may receive the whole body and commit the object
|
||||
// even if this request fails or times out afterwards.
|
||||
allSent = true;
|
||||
};
|
||||
async function* generate(): AsyncGenerator<Buffer> {
|
||||
for (const chunk of head) {
|
||||
if (account(chunk, false)) await finalize();
|
||||
yield chunk;
|
||||
}
|
||||
for (;;) {
|
||||
const step = await next();
|
||||
if (step.done) break;
|
||||
if (account(step.value, true)) await finalize();
|
||||
yield step.value;
|
||||
}
|
||||
if (total < contentLength)
|
||||
throw new UploadInterruptedError('Upload was interrupted');
|
||||
}
|
||||
const readable = Readable.from(
|
||||
(async function* () {
|
||||
try {
|
||||
yield* generate();
|
||||
} catch (error) {
|
||||
rejection = error;
|
||||
throw error;
|
||||
}
|
||||
})(),
|
||||
);
|
||||
|
||||
// A failing body must abort the PUT at once: the SDK does not end a
|
||||
// request whose Content-Length can no longer be satisfied, it would
|
||||
// hang until the request timeout. The error itself is reported via
|
||||
// `rejection`; the listener also keeps a late error from crashing us.
|
||||
readable.on('error', () => controller.abort());
|
||||
let claimed = false;
|
||||
let stored = false;
|
||||
try {
|
||||
await this.storage.putStream(
|
||||
key,
|
||||
readable,
|
||||
contentLength,
|
||||
declared,
|
||||
controller.signal,
|
||||
);
|
||||
this.metrics.add(
|
||||
'attachments.proxy.bytes_to_s3',
|
||||
contentLength,
|
||||
);
|
||||
stored = true;
|
||||
const row = await this.jobs.acceptUpload(
|
||||
id,
|
||||
{
|
||||
sizeBytes: total,
|
||||
sha256: sha256.digest('hex'),
|
||||
width: header.width,
|
||||
height: header.height,
|
||||
},
|
||||
this.pipelineVersion,
|
||||
this.maxAttempts,
|
||||
);
|
||||
if (row === undefined) {
|
||||
claimed = true;
|
||||
throw new ServiceUnavailableException();
|
||||
}
|
||||
// Durable: staging object + attachment + job + outbox are committed.
|
||||
// The fault point deliberately sits between COMMIT and publish; the
|
||||
// relay recovers the outbox row after a crash.
|
||||
await this.faults.point('api.after_commit_before_publish');
|
||||
this.relay.kick();
|
||||
this.metrics.increment('attachments.upload.queued');
|
||||
return toAttachment(row);
|
||||
} catch (error) {
|
||||
readable.destroy();
|
||||
if (!claimed) {
|
||||
// After the last byte was handed to S3, a PUT aborted on our
|
||||
// side may still be committed by S3 after our DELETE: keep the
|
||||
// row so the sweeper deletes the key again later.
|
||||
const outcome = await this.compensate(id, key, {
|
||||
keepRow: allSent && !stored,
|
||||
});
|
||||
// acceptUpload failed on our side, but the row is no longer
|
||||
// pending: the commit may have succeeded with its reply lost.
|
||||
// If so, the upload is accepted and 202 is the truth.
|
||||
if (outcome === 'not_pending' && stored) {
|
||||
const row = await this.repository
|
||||
.findById(id)
|
||||
.catch(() => undefined);
|
||||
if (
|
||||
row &&
|
||||
[
|
||||
'queued',
|
||||
'processing',
|
||||
'ready',
|
||||
'failed',
|
||||
].includes(row.status)
|
||||
) {
|
||||
this.metrics.increment(
|
||||
'attachments.upload.recovered',
|
||||
);
|
||||
return toAttachment(row);
|
||||
}
|
||||
}
|
||||
}
|
||||
throw this.mapUploadError(rejection ?? error, timedOut);
|
||||
}
|
||||
} finally {
|
||||
clearTimeout(timer);
|
||||
}
|
||||
}
|
||||
|
||||
private mapUploadError(error: unknown, timedOut: boolean): unknown {
|
||||
if (error instanceof ImageRejectedError) return imageRejection(error);
|
||||
if (error instanceof UploadInterruptedError || timedOut)
|
||||
return new BadRequestException('Upload was interrupted');
|
||||
return error;
|
||||
}
|
||||
|
||||
/**
|
||||
* Undoes an upload that did not reach `ready`. The row is claimed first
|
||||
* (pending -> deleting) and the object is deleted only after a successful
|
||||
* claim: if PostgreSQL committed `ready` but the reply was lost, the claim
|
||||
* matches nothing and the object stays. If PostgreSQL is unreachable,
|
||||
* nothing is deleted; the sweeper finishes the job after expiry + grace
|
||||
* (a row stuck in `deleting` is re-claimed after a minute).
|
||||
*/
|
||||
async compensate(
|
||||
id: string,
|
||||
key: string,
|
||||
options: { keepRow?: boolean } = {},
|
||||
): Promise<'compensated' | 'not_pending' | 'deferred'> {
|
||||
try {
|
||||
if (!(await this.repository.claimForCompensation(id))) {
|
||||
this.metrics.increment('attachments.compensation_skipped');
|
||||
return 'not_pending';
|
||||
}
|
||||
await this.storage.delete(key);
|
||||
// A kept `deleting` row is re-claimed by the sweeper after a minute,
|
||||
// which deletes the key once more and only then the row.
|
||||
if (!options.keepRow) await this.repository.deleteClaimed(id);
|
||||
this.metrics.increment('attachments.compensated');
|
||||
return 'compensated';
|
||||
} catch (error) {
|
||||
this.metrics.increment('attachments.compensation_deferred');
|
||||
this.logger.warn(
|
||||
`Compensation of attachment ${id} deferred: ${error instanceof Error ? error.message : 'unknown error'}`,
|
||||
);
|
||||
return 'deferred';
|
||||
}
|
||||
}
|
||||
|
||||
async getMetadata(id: string, user: User): Promise<Attachment> {
|
||||
const row = await this.repository.findVisible(id, user.id);
|
||||
if (!row) throw new NotFoundException();
|
||||
const variants =
|
||||
row.status === 'ready' && row.ready_pipeline_version !== null
|
||||
? await this.jobs.variantsFor(
|
||||
row.id,
|
||||
row.ready_pipeline_version,
|
||||
)
|
||||
: [];
|
||||
return toAttachment(row, variants);
|
||||
}
|
||||
|
||||
async openContent(
|
||||
id: string,
|
||||
user: User,
|
||||
variant: string | undefined,
|
||||
): Promise<{
|
||||
row: AttachmentRow;
|
||||
contentType: string;
|
||||
sizeBytes: number;
|
||||
stream: Readable;
|
||||
}> {
|
||||
if (
|
||||
variant !== undefined &&
|
||||
!['original', 'thumbnail', 'crop', 'optimized'].includes(variant)
|
||||
)
|
||||
throw new BadRequestException('variant is invalid');
|
||||
const row = await this.repository.findVisible(id, user.id);
|
||||
if (!row) throw new NotFoundException();
|
||||
if (row.status !== 'ready')
|
||||
throw new ConflictException('Attachment is not ready');
|
||||
|
||||
let key = row.object_key;
|
||||
let contentType: string = row.content_type;
|
||||
let sizeBytes = Number(row.size_bytes);
|
||||
if (variant !== undefined && variant !== 'original') {
|
||||
if (row.ready_pipeline_version === null)
|
||||
throw new NotFoundException();
|
||||
const variants = await this.jobs.variantsFor(
|
||||
row.id,
|
||||
row.ready_pipeline_version,
|
||||
);
|
||||
const match = variants.find((item) => item.variant === variant);
|
||||
if (!match) throw new NotFoundException();
|
||||
key = match.object_key;
|
||||
contentType = match.content_type;
|
||||
sizeBytes = Number(match.size_bytes);
|
||||
}
|
||||
|
||||
const source = await this.storage.getStream(key);
|
||||
if (source === null) {
|
||||
this.metrics.increment('attachments.object_missing');
|
||||
this.logger.error(
|
||||
`Object for ready attachment ${row.id} is missing in storage`,
|
||||
);
|
||||
throw new InternalServerErrorException();
|
||||
}
|
||||
const metrics = this.metrics;
|
||||
const counter = new Transform({
|
||||
transform(chunk: Buffer, _encoding, callback) {
|
||||
metrics.add('attachments.download.bytes_out', chunk.length);
|
||||
callback(null, chunk);
|
||||
},
|
||||
});
|
||||
pipeline(source, counter, () => undefined);
|
||||
return { row, contentType, sizeBytes, stream: counter };
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,293 @@
|
||||
import {
|
||||
ConflictException,
|
||||
HttpException,
|
||||
Injectable,
|
||||
NotFoundException,
|
||||
PayloadTooLargeException,
|
||||
UnsupportedMediaTypeException,
|
||||
} from '@nestjs/common';
|
||||
import { ConfigService } from '@nestjs/config';
|
||||
import { randomUUID } from 'node:crypto';
|
||||
import type { AppConfig } from '../config/app-config.schema.js';
|
||||
import { ChatsService } from '../chats/chats.service.js';
|
||||
import { JobsRepository } from '../jobs/jobs.repository.js';
|
||||
import { OutboxRelay } from '../jobs/outbox-relay.service.js';
|
||||
import { MetricsService } from '../metrics/metrics.service.js';
|
||||
import {
|
||||
ObjectStorage,
|
||||
ObjectStorageUnavailableError,
|
||||
} from '../storage/object-storage.js';
|
||||
import type { User } from '../users/user.js';
|
||||
import {
|
||||
toAttachment,
|
||||
type Attachment,
|
||||
type AttachmentRow,
|
||||
} from './attachment.js';
|
||||
import { AttachmentsRepository } from './attachments.repository.js';
|
||||
import { imageRejection } from './attachments.service.js';
|
||||
import type { CreateDirectUploadDto } from './dto/create-direct-upload.dto.js';
|
||||
import { validateFilename } from './filename.js';
|
||||
import { ImageDecoder } from './image/image-decoder.js';
|
||||
import { UploadSlots } from './upload-slots.js';
|
||||
import {
|
||||
ImageInspector,
|
||||
ImageRejectedError,
|
||||
type ImageHeader,
|
||||
} from './image/image-inspector.js';
|
||||
|
||||
export type DirectUpload = {
|
||||
attachment_id: string;
|
||||
method: 'PUT';
|
||||
url: string;
|
||||
headers: Record<string, string>;
|
||||
expires_at: string;
|
||||
};
|
||||
|
||||
const MEDIA_TYPES = new Set(['image/png', 'image/jpeg']);
|
||||
|
||||
@Injectable()
|
||||
export class DirectUploadsService {
|
||||
private readonly maxUploadBytes: number;
|
||||
private readonly maxPixels: number;
|
||||
private readonly uploadTimeoutMs: number;
|
||||
private readonly ttlSeconds: number;
|
||||
private readonly maxPending: number;
|
||||
private readonly pipelineVersion: number;
|
||||
private readonly maxAttempts: number;
|
||||
|
||||
constructor(
|
||||
config: ConfigService<AppConfig>,
|
||||
private readonly repository: AttachmentsRepository,
|
||||
private readonly jobs: JobsRepository,
|
||||
private readonly relay: OutboxRelay,
|
||||
private readonly storage: ObjectStorage,
|
||||
private readonly chats: ChatsService,
|
||||
private readonly metrics: MetricsService,
|
||||
private readonly decoder: ImageDecoder,
|
||||
private readonly slots: UploadSlots,
|
||||
) {
|
||||
this.maxUploadBytes = config.getOrThrow('MAX_UPLOAD_BYTES');
|
||||
this.maxPixels = config.getOrThrow('MAX_IMAGE_PIXELS');
|
||||
this.uploadTimeoutMs = config.getOrThrow('UPLOAD_TIMEOUT_MS');
|
||||
this.ttlSeconds = config.getOrThrow('DIRECT_UPLOAD_URL_TTL_SECONDS');
|
||||
this.maxPending = config.getOrThrow(
|
||||
'DIRECT_UPLOAD_MAX_PENDING_PER_USER',
|
||||
);
|
||||
this.pipelineVersion = config.getOrThrow('PIPELINE_VERSION');
|
||||
this.maxAttempts = config.getOrThrow('JOB_MAX_ATTEMPTS');
|
||||
}
|
||||
|
||||
async init(
|
||||
chatId: string,
|
||||
user: User,
|
||||
dto: CreateDirectUploadDto,
|
||||
): Promise<DirectUpload> {
|
||||
const filename = validateFilename(dto.filename);
|
||||
if (!MEDIA_TYPES.has(dto.content_type))
|
||||
throw new UnsupportedMediaTypeException(
|
||||
'content_type must be image/png or image/jpeg',
|
||||
);
|
||||
if (dto.size_bytes > this.maxUploadBytes)
|
||||
throw new PayloadTooLargeException('Image is too large');
|
||||
await this.chats.getForUser(chatId, user);
|
||||
const id = randomUUID();
|
||||
const stagingKey = this.storage.stagingKey(id);
|
||||
const inserted = await this.repository.insertPendingLimited(
|
||||
{
|
||||
id,
|
||||
chatId,
|
||||
ownerId: user.id,
|
||||
filename,
|
||||
contentType: dto.content_type,
|
||||
uploadMode: 'direct',
|
||||
objectKey: this.storage.objectKey(id),
|
||||
stagingKey,
|
||||
sizeBytes: dto.size_bytes,
|
||||
sha256: dto.sha256,
|
||||
ttlMs: this.ttlSeconds * 1000,
|
||||
},
|
||||
this.maxPending,
|
||||
);
|
||||
if (!inserted)
|
||||
throw new HttpException(
|
||||
{
|
||||
message: 'Too many unfinished direct uploads',
|
||||
retryAfter: this.ttlSeconds,
|
||||
},
|
||||
429,
|
||||
);
|
||||
const checksum = Buffer.from(dto.sha256, 'hex').toString('base64');
|
||||
let url: string;
|
||||
try {
|
||||
url = await this.storage.presignPut(stagingKey, {
|
||||
contentType: dto.content_type,
|
||||
contentLength: dto.size_bytes,
|
||||
checksumSha256Base64: checksum,
|
||||
expiresInSeconds: this.ttlSeconds,
|
||||
});
|
||||
} catch (error) {
|
||||
await this.repository.deletePending(id);
|
||||
throw error;
|
||||
}
|
||||
this.metrics.increment('attachments.direct.initiated');
|
||||
return {
|
||||
attachment_id: id,
|
||||
method: 'PUT',
|
||||
url,
|
||||
headers: {
|
||||
'Content-Type': dto.content_type,
|
||||
'x-amz-checksum-sha256': checksum,
|
||||
},
|
||||
expires_at: new Date(
|
||||
Date.now() + this.ttlSeconds * 1000,
|
||||
).toISOString(),
|
||||
};
|
||||
}
|
||||
|
||||
async complete(attachmentId: string, user: User): Promise<Attachment> {
|
||||
const row = await this.repository.findForComplete(
|
||||
attachmentId,
|
||||
user.id,
|
||||
);
|
||||
if (!row) throw new NotFoundException();
|
||||
if (row.status !== 'pending') return this.describe(row);
|
||||
return this.slots.run(user.id, () => this.verifyAndAccept(row));
|
||||
}
|
||||
|
||||
private async describe(row: AttachmentRow): Promise<Attachment> {
|
||||
const variants =
|
||||
row.status === 'ready' && row.ready_pipeline_version !== null
|
||||
? await this.jobs.variantsFor(
|
||||
row.id,
|
||||
row.ready_pipeline_version,
|
||||
)
|
||||
: [];
|
||||
return toAttachment(row, variants);
|
||||
}
|
||||
|
||||
private async verifyAndAccept(row: AttachmentRow): Promise<Attachment> {
|
||||
const size = Number(row.size_bytes);
|
||||
const stagingKey = row.staging_key;
|
||||
if (stagingKey === null)
|
||||
throw new ConflictException('Upload has not been received');
|
||||
const expectedChecksum = Buffer.from(
|
||||
row.sha256 as string,
|
||||
'hex',
|
||||
).toString('base64');
|
||||
const head = await this.storage.head(stagingKey);
|
||||
if (head === null)
|
||||
throw new ConflictException('Upload has not been received');
|
||||
if (head.size !== size || head.checksumSha256 !== expectedChecksum) {
|
||||
await this.reject(row, stagingKey);
|
||||
throw new ConflictException(
|
||||
'Uploaded object does not match the declaration',
|
||||
);
|
||||
}
|
||||
|
||||
let header: ImageHeader;
|
||||
try {
|
||||
header = await this.verifyObject(stagingKey, size);
|
||||
} catch (error) {
|
||||
if (error instanceof ImageRejectedError) {
|
||||
await this.reject(row, stagingKey);
|
||||
throw imageRejection(error);
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
if (header.contentType !== row.content_type) {
|
||||
await this.reject(row, stagingKey);
|
||||
throw new UnsupportedMediaTypeException(
|
||||
'Image content does not match content_type',
|
||||
);
|
||||
}
|
||||
const accepted = await this.jobs.acceptUpload(
|
||||
row.id,
|
||||
{
|
||||
sizeBytes: size,
|
||||
sha256: row.sha256 as string,
|
||||
width: header.width,
|
||||
height: header.height,
|
||||
},
|
||||
this.pipelineVersion,
|
||||
this.maxAttempts,
|
||||
);
|
||||
if (!accepted) {
|
||||
// A parallel complete of the same upload won: answer like a repeat.
|
||||
const current = await this.repository.findForComplete(
|
||||
row.id,
|
||||
row.owner_id,
|
||||
);
|
||||
if (current && current.status !== 'pending')
|
||||
return this.describe(current);
|
||||
throw new ConflictException('Upload is no longer pending');
|
||||
}
|
||||
this.relay.kick();
|
||||
this.metrics.increment('attachments.direct.completed');
|
||||
return toAttachment(accepted);
|
||||
}
|
||||
|
||||
/**
|
||||
* Reads the whole object: a correct signed checksum only proves that the
|
||||
* bytes are the ones the client declared, not that they form an image. The
|
||||
* stream goes through ImageInspector (structure, CRCs, completeness), then
|
||||
* the copy is fully decoded. Memory is bounded like a proxy upload.
|
||||
*/
|
||||
private async verifyObject(
|
||||
key: string,
|
||||
size: number,
|
||||
): Promise<ImageHeader> {
|
||||
// Same deadline as a proxy upload: a slow or hung read releases its slot.
|
||||
const signal = AbortSignal.timeout(this.uploadTimeoutMs);
|
||||
const stream = await this.storage.getStream(key, signal);
|
||||
if (stream === null)
|
||||
throw new ConflictException('Upload has not been received');
|
||||
const inspector = new ImageInspector(this.maxPixels);
|
||||
const chunks: Buffer[] = [];
|
||||
let read = 0;
|
||||
try {
|
||||
for await (const chunk of stream as AsyncIterable<Buffer>) {
|
||||
read += chunk.length;
|
||||
if (read > size)
|
||||
throw new ImageRejectedError(
|
||||
'unsupported_media_type',
|
||||
'Object is larger than declared',
|
||||
);
|
||||
inspector.push(chunk);
|
||||
chunks.push(chunk);
|
||||
}
|
||||
} catch (error) {
|
||||
if (error instanceof ImageRejectedError) throw error;
|
||||
// The object stream broke mid-read (S3 outage, timeout): the image
|
||||
// was not judged, so this is an outage, not a client error.
|
||||
throw new ObjectStorageUnavailableError();
|
||||
} finally {
|
||||
stream.destroy();
|
||||
this.metrics.add('attachments.direct.verify_bytes', read);
|
||||
}
|
||||
const header = inspector.finish();
|
||||
const data = Buffer.concat(chunks, read);
|
||||
chunks.length = 0;
|
||||
await this.decoder.verify(data, signal);
|
||||
return header;
|
||||
}
|
||||
|
||||
/**
|
||||
* A rejected claim keeps its row (status `rejected`, original expiry) while
|
||||
* the signed URL may still be used: a later re-PUT would otherwise leave an
|
||||
* object that no row refers to. The sweeper removes both after expiry + grace.
|
||||
*/
|
||||
private async reject(
|
||||
row: AttachmentRow,
|
||||
stagingKey: string,
|
||||
): Promise<void> {
|
||||
// Row first: a row that is not pending any more (accepted after a parallel
|
||||
// complete, or claimed by the sweeper) keeps its object.
|
||||
if (!(await this.repository.rejectPending(row.id))) return;
|
||||
try {
|
||||
await this.storage.delete(stagingKey);
|
||||
} catch {
|
||||
// The sweeper deletes the object again after expiry.
|
||||
}
|
||||
this.metrics.increment('attachments.direct.rejected');
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
import { IsInt, IsString, Matches, Min } from 'class-validator';
|
||||
|
||||
export class CreateDirectUploadDto {
|
||||
@IsString()
|
||||
filename!: string;
|
||||
|
||||
@IsString()
|
||||
content_type!: string;
|
||||
|
||||
@IsInt()
|
||||
@Min(1)
|
||||
size_bytes!: number;
|
||||
|
||||
@Matches(/^[a-f0-9]{64}$/u)
|
||||
sha256!: string;
|
||||
}
|
||||
@@ -0,0 +1,34 @@
|
||||
import { BadRequestException } from '@nestjs/common';
|
||||
|
||||
const FORBIDDEN =
|
||||
// eslint-disable-next-line no-control-regex
|
||||
/[/\\\u0000-\u001f\u007f-\u009f\u200e\u200f\u202a-\u202e\u2066-\u2069]/u;
|
||||
|
||||
/** The name is stored as-is and used for display only, never in paths or keys. */
|
||||
export function validateFilename(value: unknown): string {
|
||||
if (typeof value !== 'string') throw invalid();
|
||||
const length = [...value].length;
|
||||
if (
|
||||
length < 1 ||
|
||||
length > 255 ||
|
||||
/^\s*$/u.test(value) ||
|
||||
value === '.' ||
|
||||
value === '..' ||
|
||||
FORBIDDEN.test(value)
|
||||
)
|
||||
throw invalid();
|
||||
return value;
|
||||
}
|
||||
|
||||
function invalid(): BadRequestException {
|
||||
return new BadRequestException('filename is invalid');
|
||||
}
|
||||
|
||||
export function contentDisposition(filename: string): string {
|
||||
const fallback = filename.replace(/[^\x20-\x7e]|["\\]/gu, '_');
|
||||
const encoded = encodeURIComponent(filename).replace(
|
||||
/['()*]/gu,
|
||||
(character) => `%${character.charCodeAt(0).toString(16).toUpperCase()}`,
|
||||
);
|
||||
return `inline; filename="${fallback}"; filename*=UTF-8''${encoded}`;
|
||||
}
|
||||
@@ -0,0 +1,70 @@
|
||||
import { readFileSync } from 'node:fs';
|
||||
import { dirname, join } from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import { describe, expect, it } from 'vitest';
|
||||
import { jpegWithoutScanData } from '../../../test/support/jpeg.js';
|
||||
import {
|
||||
maxPixelsRgba16Png,
|
||||
noisePng,
|
||||
pngWithGarbageIdat,
|
||||
} from '../../../test/support/png.js';
|
||||
import { ImageDecoder } from './image-decoder.js';
|
||||
import { ImageInspector, ImageRejectedError } from './image-inspector.js';
|
||||
|
||||
const here = dirname(fileURLToPath(import.meta.url));
|
||||
const png = readFileSync(join(here, '../../../../tests/fixtures/sample.png'));
|
||||
const jpeg = readFileSync(join(here, '../../../test/fixtures/sample.jpg'));
|
||||
const progressive = readFileSync(
|
||||
join(here, '../../../test/fixtures/sample-progressive.jpg'),
|
||||
);
|
||||
|
||||
const settings: Record<string, number> = {
|
||||
MAX_IMAGE_PIXELS: 40_000_000,
|
||||
IMAGE_DECODE_MAX_CONCURRENT: 1,
|
||||
};
|
||||
const decoder = new ImageDecoder(
|
||||
{ getOrThrow: (key: string) => settings[key] } as never,
|
||||
{ increment: () => undefined, add: () => undefined } as never,
|
||||
);
|
||||
|
||||
describe('ImageDecoder', () => {
|
||||
it('accepts valid PNG and JPEG files, also concurrently', async () => {
|
||||
await Promise.all(
|
||||
[png, jpeg, progressive, noisePng(1_048_576)].map((data) =>
|
||||
decoder.verify(data),
|
||||
),
|
||||
);
|
||||
});
|
||||
|
||||
// These files pass the structural inspector; only a decode catches them.
|
||||
it.each([
|
||||
['PNG with a garbage zlib stream', pngWithGarbageIdat()],
|
||||
['JPEG without scan data', jpegWithoutScanData(jpeg)],
|
||||
[
|
||||
'progressive JPEG without scan data',
|
||||
jpegWithoutScanData(progressive),
|
||||
],
|
||||
])('rejects a %s', async (_name, data) => {
|
||||
const inspector = new ImageInspector(40_000_000);
|
||||
inspector.push(data);
|
||||
expect(inspector.finish().width).toBe(320);
|
||||
await expect(decoder.verify(data)).rejects.toBeInstanceOf(
|
||||
ImageRejectedError,
|
||||
);
|
||||
});
|
||||
|
||||
it('accepts the heaviest valid input: 40 MP RGBA 16-bit', async () => {
|
||||
await decoder.verify(await maxPixelsRgba16Png());
|
||||
}, 30_000);
|
||||
|
||||
it('drops an aborted waiter from the queue without running its decode', async () => {
|
||||
const busy = decoder.verify(noisePng(4_000_000));
|
||||
const controller = new AbortController();
|
||||
const waiting = decoder.verify(png, controller.signal);
|
||||
controller.abort();
|
||||
await expect(waiting).rejects.toMatchObject({ status: 503 });
|
||||
await busy;
|
||||
// The slot was not leaked: the next decode runs.
|
||||
await decoder.verify(png);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,119 @@
|
||||
import {
|
||||
Injectable,
|
||||
InternalServerErrorException,
|
||||
Logger,
|
||||
ServiceUnavailableException,
|
||||
} from '@nestjs/common';
|
||||
import { ConfigService } from '@nestjs/config';
|
||||
import sharp from 'sharp';
|
||||
import type { AppConfig } from '../../config/app-config.schema.js';
|
||||
import { MetricsService } from '../../metrics/metrics.service.js';
|
||||
import { ImageRejectedError } from './image-inspector.js';
|
||||
|
||||
// One libvips worker per decode and no operation cache: decodes are one-off,
|
||||
// and memory must stay predictable under the per-replica limit.
|
||||
sharp.cache(false);
|
||||
sharp.concurrency(1);
|
||||
|
||||
// Failures of the replica itself, not of the image: they must not turn into a
|
||||
// client 415. libvips spills large images to a temp file unless
|
||||
// VIPS_DISC_THRESHOLD (compose.yaml) keeps them in memory; a full /tmp or an
|
||||
// allocation failure ends up here.
|
||||
export const SERVER_FAULT =
|
||||
/no space left|out of memory|unable to allocate|cannot allocate|write failed|unable to open|tmp/iu;
|
||||
|
||||
/**
|
||||
* Full decode check. ImageInspector only proves the container structure
|
||||
* (signature, chunk CRCs, markers, dimensions); a PNG with a garbage zlib stream
|
||||
* or a JPEG without scan data passes it. Decoding every scanline with
|
||||
* failOn=warning rejects those, including truncated or corrupted compressed
|
||||
* data. A downscale to a thumbnail is the sink: it decodes all input but keeps
|
||||
* far less in memory than a full-size pipeline (measured in
|
||||
* evidence/lab4/decode-memory.txt). Decodes are bounded by
|
||||
* IMAGE_DECODE_MAX_CONCURRENT; a waiter whose request was aborted leaves the
|
||||
* queue and releases its buffer.
|
||||
*/
|
||||
@Injectable()
|
||||
export class ImageDecoder {
|
||||
private readonly logger = new Logger(ImageDecoder.name);
|
||||
private readonly maxPixels: number;
|
||||
private readonly maxConcurrent: number;
|
||||
private active = 0;
|
||||
private readonly waiting = new Set<() => void>();
|
||||
|
||||
constructor(
|
||||
config: ConfigService<AppConfig>,
|
||||
private readonly metrics: MetricsService,
|
||||
) {
|
||||
this.maxPixels = config.getOrThrow('MAX_IMAGE_PIXELS');
|
||||
this.maxConcurrent = config.getOrThrow('IMAGE_DECODE_MAX_CONCURRENT');
|
||||
}
|
||||
|
||||
async verify(data: Buffer, signal?: AbortSignal): Promise<void> {
|
||||
await this.acquire(signal);
|
||||
const started = performance.now();
|
||||
try {
|
||||
await sharp(data, {
|
||||
failOn: 'warning',
|
||||
sequentialRead: true,
|
||||
limitInputPixels: this.maxPixels,
|
||||
})
|
||||
.resize(64, 64, { fit: 'inside' })
|
||||
.toBuffer();
|
||||
} catch (error) {
|
||||
const message =
|
||||
error instanceof Error ? error.message : String(error);
|
||||
if (SERVER_FAULT.test(message)) {
|
||||
this.metrics.increment('attachments.decode.server_error');
|
||||
this.logger.error(
|
||||
`Image decode failed on the server: ${message}`,
|
||||
);
|
||||
throw new InternalServerErrorException();
|
||||
}
|
||||
this.metrics.increment('attachments.decode.rejected');
|
||||
throw new ImageRejectedError(
|
||||
'unsupported_media_type',
|
||||
'Image data cannot be decoded',
|
||||
);
|
||||
} finally {
|
||||
this.metrics.add(
|
||||
'attachments.decode.ms',
|
||||
performance.now() - started,
|
||||
);
|
||||
this.release();
|
||||
}
|
||||
}
|
||||
|
||||
private acquire(signal?: AbortSignal): Promise<void> {
|
||||
const abandoned = new ServiceUnavailableException(
|
||||
'Image check was abandoned',
|
||||
);
|
||||
if (signal?.aborted) return Promise.reject(abandoned);
|
||||
if (this.active < this.maxConcurrent) {
|
||||
this.active += 1;
|
||||
return Promise.resolve();
|
||||
}
|
||||
const { promise, resolve, reject } = Promise.withResolvers<void>();
|
||||
const cancel = () => {
|
||||
this.waiting.delete(grant);
|
||||
reject(abandoned);
|
||||
};
|
||||
const grant = () => {
|
||||
signal?.removeEventListener('abort', cancel);
|
||||
resolve();
|
||||
};
|
||||
this.waiting.add(grant);
|
||||
signal?.addEventListener('abort', cancel, { once: true });
|
||||
return promise;
|
||||
}
|
||||
|
||||
private release(): void {
|
||||
const next = this.waiting.values().next();
|
||||
if (next.done) {
|
||||
this.active -= 1;
|
||||
return;
|
||||
}
|
||||
this.waiting.delete(next.value);
|
||||
next.value();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,109 @@
|
||||
import { readFileSync } from 'node:fs';
|
||||
import { dirname, join } from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import { describe, expect, it } from 'vitest';
|
||||
import { noisePng, pixelBombPng } from '../../../test/support/png.js';
|
||||
import { ImageInspector, ImageRejectedError } from './image-inspector.js';
|
||||
|
||||
const here = dirname(fileURLToPath(import.meta.url));
|
||||
const png = readFileSync(join(here, '../../../../tests/fixtures/sample.png'));
|
||||
const jpeg = readFileSync(join(here, '../../../test/fixtures/sample.jpg'));
|
||||
const progressive = readFileSync(
|
||||
join(here, '../../../test/fixtures/sample-progressive.jpg'),
|
||||
);
|
||||
const MAX = 40_000_000;
|
||||
|
||||
function inspect(data: Buffer, step = data.length, max = MAX) {
|
||||
const inspector = new ImageInspector(max);
|
||||
for (let offset = 0; offset < data.length; offset += step)
|
||||
inspector.push(data.subarray(offset, offset + step));
|
||||
return inspector.finish();
|
||||
}
|
||||
|
||||
function rejection(action: () => unknown): ImageRejectedError {
|
||||
try {
|
||||
action();
|
||||
} catch (error) {
|
||||
expect(error).toBeInstanceOf(ImageRejectedError);
|
||||
return error as ImageRejectedError;
|
||||
}
|
||||
throw new Error('expected rejection');
|
||||
}
|
||||
|
||||
describe('ImageInspector', () => {
|
||||
it.each([
|
||||
['png', png, 'image/png'],
|
||||
['jpeg', jpeg, 'image/jpeg'],
|
||||
['progressive jpeg', progressive, 'image/jpeg'],
|
||||
])('accepts %s and reports its size', (_name, data, contentType) => {
|
||||
const expected = { contentType, width: 320, height: 200 };
|
||||
expect(inspect(data)).toEqual(expected);
|
||||
expect(inspect(data, 1)).toEqual(expected);
|
||||
expect(inspect(data, 7)).toEqual(expected);
|
||||
});
|
||||
|
||||
it('rejects truncated files on finish', () => {
|
||||
for (const data of [png, jpeg, progressive])
|
||||
for (const cut of [10, 40, data.length - 100, data.length - 1]) {
|
||||
const reason = rejection(() =>
|
||||
inspect(data.subarray(0, cut)),
|
||||
).reason;
|
||||
expect(reason).toBe('unsupported_media_type');
|
||||
}
|
||||
});
|
||||
|
||||
it('rejects a flipped byte inside IDAT by checksum', () => {
|
||||
const damaged = Buffer.from(png);
|
||||
damaged[Math.floor(png.length / 2)] ^= 0xff;
|
||||
expect(rejection(() => inspect(damaged)).reason).toBe(
|
||||
'unsupported_media_type',
|
||||
);
|
||||
});
|
||||
|
||||
it('rejects a pixel bomb right after IHDR, before any IDAT', () => {
|
||||
const bomb = pixelBombPng();
|
||||
const inspector = new ImageInspector(MAX);
|
||||
// signature + IHDR chunk = 8 + 25 bytes
|
||||
const reason = rejection(() =>
|
||||
inspector.push(bomb.subarray(0, 33)),
|
||||
).reason;
|
||||
expect(reason).toBe('too_many_pixels');
|
||||
expect(inspector.header).toBeUndefined();
|
||||
});
|
||||
|
||||
it('applies the pixel limit to JPEG frame headers', () => {
|
||||
expect(rejection(() => inspect(jpeg, jpeg.length, 63_999)).reason).toBe(
|
||||
'too_many_pixels',
|
||||
);
|
||||
expect(inspect(jpeg, jpeg.length, 64_000).width).toBe(320);
|
||||
});
|
||||
|
||||
it('rejects foreign formats and trailing data after IEND', () => {
|
||||
for (const data of [
|
||||
Buffer.from('GIF89a' + 'x'.repeat(40)),
|
||||
Buffer.from('plain text, not an image'),
|
||||
Buffer.alloc(0),
|
||||
])
|
||||
expect(rejection(() => inspect(data)).reason).toBe(
|
||||
'unsupported_media_type',
|
||||
);
|
||||
expect(
|
||||
rejection(() => inspect(Buffer.concat([png, Buffer.from([0])])))
|
||||
.reason,
|
||||
).toBe('unsupported_media_type');
|
||||
});
|
||||
|
||||
it('accepts padded and noise PNGs of exact sizes', () => {
|
||||
for (const size of [131_072, 1_048_576]) {
|
||||
const data = noisePng(size);
|
||||
expect(data.length).toBe(size);
|
||||
expect(inspect(data).width).toBe(1024);
|
||||
}
|
||||
});
|
||||
|
||||
it('tolerates trailing data after JPEG EOI', () => {
|
||||
expect(
|
||||
inspect(Buffer.concat([jpeg, Buffer.from('MPF trailer')])).height,
|
||||
).toBe(200);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,387 @@
|
||||
import { crc32 } from 'node:zlib';
|
||||
|
||||
export type ImageHeader = {
|
||||
contentType: 'image/png' | 'image/jpeg';
|
||||
width: number;
|
||||
height: number;
|
||||
};
|
||||
|
||||
export class ImageRejectedError extends Error {
|
||||
constructor(
|
||||
readonly reason: 'unsupported_media_type' | 'too_many_pixels',
|
||||
message: string,
|
||||
) {
|
||||
super(message);
|
||||
this.name = 'ImageRejectedError';
|
||||
}
|
||||
}
|
||||
|
||||
const PNG_SIGNATURE = Buffer.from([
|
||||
0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a,
|
||||
]);
|
||||
const PNG_MAX_DIMENSION = 2 ** 31 - 1;
|
||||
const PNG_BIT_DEPTHS: Record<number, readonly number[]> = {
|
||||
0: [1, 2, 4, 8, 16],
|
||||
2: [8, 16],
|
||||
3: [1, 2, 4, 8],
|
||||
4: [8, 16],
|
||||
6: [8, 16],
|
||||
};
|
||||
const SOF_MARKERS = new Set([
|
||||
0xc0, 0xc1, 0xc2, 0xc3, 0xc5, 0xc6, 0xc7, 0xc9, 0xca, 0xcb, 0xcd, 0xce,
|
||||
0xcf,
|
||||
]);
|
||||
|
||||
type State =
|
||||
| 'start'
|
||||
// PNG
|
||||
| 'png_chunk_head'
|
||||
| 'png_ihdr'
|
||||
| 'png_skip'
|
||||
| 'png_crc'
|
||||
| 'png_done'
|
||||
// JPEG
|
||||
| 'jpeg_marker'
|
||||
| 'jpeg_code'
|
||||
| 'jpeg_length'
|
||||
| 'jpeg_sof'
|
||||
| 'jpeg_skip'
|
||||
| 'jpeg_entropy'
|
||||
| 'jpeg_entropy_ff'
|
||||
| 'jpeg_trailer';
|
||||
|
||||
function unsupported(message: string): ImageRejectedError {
|
||||
return new ImageRejectedError('unsupported_media_type', message);
|
||||
}
|
||||
|
||||
/**
|
||||
* Streaming structural validator for PNG and JPEG. It never buffers the body:
|
||||
* only fixed-size fields (chunk headers, IHDR, SOF) are accumulated, and
|
||||
* nothing is decompressed. It is the early gate (type, dimensions, pixel limit
|
||||
* before anything is stored), not proof of a valid image: compressed pixel
|
||||
* data is checked by ImageDecoder.
|
||||
*/
|
||||
export class ImageInspector {
|
||||
private state: State = 'start';
|
||||
private imageHeader: ImageHeader | undefined;
|
||||
|
||||
// Accumulator for fixed-length fields.
|
||||
private field = Buffer.alloc(16);
|
||||
private fieldNeed = 0;
|
||||
private fieldLength = 0;
|
||||
|
||||
// Remaining bytes to skip in png_skip / jpeg_skip.
|
||||
private skip = 0;
|
||||
|
||||
// PNG
|
||||
private chunkType = '';
|
||||
private chunkLength = 0;
|
||||
private crc = 0;
|
||||
private firstChunk = true;
|
||||
private seenIdat = false;
|
||||
|
||||
// JPEG
|
||||
private marker = 0;
|
||||
private scans = 0;
|
||||
private afterSkip: State = 'jpeg_marker';
|
||||
|
||||
constructor(private readonly maxPixels: number) {}
|
||||
|
||||
get header(): ImageHeader | undefined {
|
||||
return this.imageHeader;
|
||||
}
|
||||
|
||||
push(chunk: Buffer): void {
|
||||
let offset = 0;
|
||||
while (offset < chunk.length) offset = this.step(chunk, offset);
|
||||
}
|
||||
|
||||
finish(): ImageHeader {
|
||||
const done = this.state === 'png_done' || this.state === 'jpeg_trailer';
|
||||
if (!done || this.imageHeader === undefined)
|
||||
throw unsupported('Image data is incomplete');
|
||||
return this.imageHeader;
|
||||
}
|
||||
|
||||
private collect(need: number): void {
|
||||
this.fieldNeed = need;
|
||||
this.fieldLength = 0;
|
||||
}
|
||||
|
||||
/** Copies up to fieldNeed bytes; returns new offset and whether complete. */
|
||||
private fill(chunk: Buffer, offset: number): [number, boolean] {
|
||||
const take = Math.min(
|
||||
this.fieldNeed - this.fieldLength,
|
||||
chunk.length - offset,
|
||||
);
|
||||
chunk.copy(this.field, this.fieldLength, offset, offset + take);
|
||||
this.fieldLength += take;
|
||||
return [offset + take, this.fieldLength === this.fieldNeed];
|
||||
}
|
||||
|
||||
private step(chunk: Buffer, offset: number): number {
|
||||
switch (this.state) {
|
||||
case 'start':
|
||||
return this.stepStart(chunk, offset);
|
||||
case 'png_chunk_head':
|
||||
case 'png_ihdr':
|
||||
case 'png_crc':
|
||||
case 'jpeg_length':
|
||||
case 'jpeg_sof': {
|
||||
const [next, complete] = this.fill(chunk, offset);
|
||||
if (complete) this.fieldDone();
|
||||
return next;
|
||||
}
|
||||
case 'png_skip':
|
||||
case 'jpeg_skip': {
|
||||
const take = Math.min(this.skip, chunk.length - offset);
|
||||
if (this.state === 'png_skip')
|
||||
this.crc = crc32(
|
||||
chunk.subarray(offset, offset + take),
|
||||
this.crc,
|
||||
);
|
||||
this.skip -= take;
|
||||
if (this.skip === 0) this.skipDone();
|
||||
return offset + take;
|
||||
}
|
||||
case 'png_done':
|
||||
throw unsupported('Data after the PNG end chunk');
|
||||
case 'jpeg_marker':
|
||||
if (chunk[offset] !== 0xff)
|
||||
throw unsupported('Unexpected byte between JPEG segments');
|
||||
this.state = 'jpeg_code';
|
||||
return offset + 1;
|
||||
case 'jpeg_code':
|
||||
this.onMarkerCode(chunk[offset]);
|
||||
return offset + 1;
|
||||
case 'jpeg_entropy': {
|
||||
const next = chunk.indexOf(0xff, offset);
|
||||
if (next === -1) return chunk.length;
|
||||
this.state = 'jpeg_entropy_ff';
|
||||
return next + 1;
|
||||
}
|
||||
case 'jpeg_entropy_ff': {
|
||||
const byte = chunk[offset];
|
||||
if (byte === 0xff) return offset + 1;
|
||||
if (byte === 0x00 || (byte >= 0xd0 && byte <= 0xd7)) {
|
||||
this.state = 'jpeg_entropy';
|
||||
return offset + 1;
|
||||
}
|
||||
this.onMarkerCode(byte);
|
||||
return offset + 1;
|
||||
}
|
||||
case 'jpeg_trailer':
|
||||
return chunk.length;
|
||||
}
|
||||
}
|
||||
|
||||
private stepStart(chunk: Buffer, offset: number): number {
|
||||
// Accumulate until the type is certain: 3 bytes for JPEG, 8 for PNG.
|
||||
if (this.fieldLength === 0 && this.fieldNeed === 0) this.collect(8);
|
||||
const have = this.fieldLength;
|
||||
const take = Math.min(this.fieldNeed - have, chunk.length - offset);
|
||||
chunk.copy(this.field, have, offset, offset + take);
|
||||
this.fieldLength += take;
|
||||
const first = this.field[0];
|
||||
if (first === 0xff) {
|
||||
if (this.fieldLength >= 3) {
|
||||
if (this.field[1] !== 0xd8 || this.field[2] !== 0xff)
|
||||
throw unsupported('Not a JPEG file');
|
||||
// The third byte opens the next marker.
|
||||
this.state = 'jpeg_code';
|
||||
this.fieldNeed = 0;
|
||||
this.fieldLength = 0;
|
||||
return offset + (3 - have);
|
||||
}
|
||||
return offset + take;
|
||||
}
|
||||
if (first === PNG_SIGNATURE[0]) {
|
||||
for (let index = 0; index < this.fieldLength; index += 1)
|
||||
if (this.field[index] !== PNG_SIGNATURE[index])
|
||||
throw unsupported('Not a PNG file');
|
||||
if (this.fieldLength === 8) {
|
||||
this.state = 'png_chunk_head';
|
||||
this.collect(8);
|
||||
}
|
||||
return offset + take;
|
||||
}
|
||||
throw unsupported('Unsupported image format');
|
||||
}
|
||||
|
||||
private fieldDone(): void {
|
||||
switch (this.state) {
|
||||
case 'png_chunk_head':
|
||||
this.onPngChunkHead();
|
||||
return;
|
||||
case 'png_ihdr':
|
||||
this.crc = crc32(this.field.subarray(0, 13), this.crc);
|
||||
this.parseIhdr();
|
||||
this.state = 'png_crc';
|
||||
this.collect(4);
|
||||
return;
|
||||
case 'png_crc': {
|
||||
const expected = this.field.readUInt32BE(0);
|
||||
if (this.crc >>> 0 !== expected)
|
||||
throw unsupported('PNG chunk checksum mismatch');
|
||||
this.onPngChunkEnd();
|
||||
return;
|
||||
}
|
||||
case 'jpeg_length':
|
||||
this.onJpegLength();
|
||||
return;
|
||||
case 'jpeg_sof':
|
||||
this.onJpegSof();
|
||||
return;
|
||||
default:
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
private skipDone(): void {
|
||||
if (this.state === 'png_skip') {
|
||||
this.state = 'png_crc';
|
||||
this.collect(4);
|
||||
} else {
|
||||
this.state = this.afterSkip;
|
||||
}
|
||||
}
|
||||
|
||||
// ---- PNG ----
|
||||
|
||||
private onPngChunkHead(): void {
|
||||
const length = this.field.readUInt32BE(0);
|
||||
const type = this.field.toString('latin1', 4, 8);
|
||||
if (length > PNG_MAX_DIMENSION)
|
||||
throw unsupported('PNG chunk is too long');
|
||||
if (!/^[A-Za-z]{4}$/u.test(type))
|
||||
throw unsupported('PNG chunk type is invalid');
|
||||
if (this.firstChunk) {
|
||||
if (type !== 'IHDR' || length !== 13)
|
||||
throw unsupported('PNG must start with IHDR');
|
||||
this.firstChunk = false;
|
||||
} else if (type === 'IHDR') {
|
||||
throw unsupported('Duplicate PNG IHDR');
|
||||
}
|
||||
if (type === 'IEND') {
|
||||
if (length !== 0 || !this.seenIdat)
|
||||
throw unsupported('PNG IEND is invalid');
|
||||
}
|
||||
if (type === 'IDAT') this.seenIdat = true;
|
||||
this.chunkType = type;
|
||||
this.chunkLength = length;
|
||||
this.crc = crc32(this.field.subarray(4, 8), 0);
|
||||
if (type === 'IHDR') {
|
||||
this.state = 'png_ihdr';
|
||||
this.collect(13);
|
||||
} else if (length === 0) {
|
||||
this.state = 'png_crc';
|
||||
this.collect(4);
|
||||
} else {
|
||||
this.state = 'png_skip';
|
||||
this.skip = length;
|
||||
}
|
||||
}
|
||||
|
||||
private parseIhdr(): void {
|
||||
const width = this.field.readUInt32BE(0);
|
||||
const height = this.field.readUInt32BE(4);
|
||||
const bitDepth = this.field[8];
|
||||
const colorType = this.field[9];
|
||||
if (width < 1 || width > PNG_MAX_DIMENSION)
|
||||
throw unsupported('PNG width is invalid');
|
||||
if (height < 1 || height > PNG_MAX_DIMENSION)
|
||||
throw unsupported('PNG height is invalid');
|
||||
if (!PNG_BIT_DEPTHS[colorType]?.includes(bitDepth))
|
||||
throw unsupported('PNG color type / bit depth is invalid');
|
||||
if (this.field[10] !== 0 || this.field[11] !== 0)
|
||||
throw unsupported('PNG compression/filter method is invalid');
|
||||
if (this.field[12] > 1)
|
||||
throw unsupported('PNG interlace method is invalid');
|
||||
this.pendingHeader = { contentType: 'image/png', width, height };
|
||||
}
|
||||
|
||||
private pendingHeader: ImageHeader | undefined;
|
||||
|
||||
private onPngChunkEnd(): void {
|
||||
if (this.chunkType === 'IHDR' && this.pendingHeader) {
|
||||
const { width, height } = this.pendingHeader;
|
||||
if (width * height > this.maxPixels)
|
||||
throw new ImageRejectedError(
|
||||
'too_many_pixels',
|
||||
'Image has too many pixels',
|
||||
);
|
||||
this.imageHeader = this.pendingHeader;
|
||||
}
|
||||
if (this.chunkType === 'IEND') {
|
||||
this.state = 'png_done';
|
||||
return;
|
||||
}
|
||||
this.state = 'png_chunk_head';
|
||||
this.collect(8);
|
||||
}
|
||||
|
||||
// ---- JPEG ----
|
||||
|
||||
private onMarkerCode(code: number): void {
|
||||
if (code === 0xff) return; // fill byte
|
||||
this.marker = code;
|
||||
if (code === 0x00 || code === 0xd8)
|
||||
throw unsupported('Unexpected JPEG marker');
|
||||
if (code === 0x01 || (code >= 0xd0 && code <= 0xd7)) {
|
||||
this.state = 'jpeg_marker';
|
||||
return;
|
||||
}
|
||||
if (code === 0xd9) {
|
||||
if (this.imageHeader === undefined || this.scans === 0)
|
||||
throw unsupported('JPEG ended before image data');
|
||||
this.state = 'jpeg_trailer';
|
||||
return;
|
||||
}
|
||||
if (code === 0xda && this.imageHeader === undefined)
|
||||
throw unsupported('JPEG scan before frame header');
|
||||
this.state = 'jpeg_length';
|
||||
this.collect(2);
|
||||
}
|
||||
|
||||
private onJpegLength(): void {
|
||||
const length = this.field.readUInt16BE(0);
|
||||
if (length < 2) throw unsupported('JPEG segment length is invalid');
|
||||
const dataLength = length - 2;
|
||||
if (SOF_MARKERS.has(this.marker)) {
|
||||
if (this.imageHeader !== undefined || this.pendingHeader)
|
||||
throw unsupported('Duplicate JPEG frame header');
|
||||
if (dataLength < 6) throw unsupported('JPEG frame header is short');
|
||||
this.skip = dataLength - 6;
|
||||
this.afterSkip = 'jpeg_marker';
|
||||
this.state = 'jpeg_sof';
|
||||
this.collect(6);
|
||||
return;
|
||||
}
|
||||
if (this.marker === 0xda) {
|
||||
this.scans += 1;
|
||||
this.afterSkip = 'jpeg_entropy';
|
||||
} else {
|
||||
this.afterSkip = 'jpeg_marker';
|
||||
}
|
||||
if (dataLength === 0) this.state = this.afterSkip;
|
||||
else {
|
||||
this.skip = dataLength;
|
||||
this.state = 'jpeg_skip';
|
||||
}
|
||||
}
|
||||
|
||||
private onJpegSof(): void {
|
||||
const height = this.field.readUInt16BE(1);
|
||||
const width = this.field.readUInt16BE(3);
|
||||
if (width === 0 || height === 0 || this.field[5] === 0)
|
||||
throw unsupported('JPEG dimensions are invalid');
|
||||
if (width * height > this.maxPixels)
|
||||
throw new ImageRejectedError(
|
||||
'too_many_pixels',
|
||||
'Image has too many pixels',
|
||||
);
|
||||
this.imageHeader = { contentType: 'image/jpeg', width, height };
|
||||
if (this.skip === 0) this.state = 'jpeg_marker';
|
||||
else this.state = 'jpeg_skip';
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
/** Request of the upload route; see the onRequest hook in main.ts. */
|
||||
export type UploadRequest = {
|
||||
method: string;
|
||||
routeOptions?: { url?: string };
|
||||
headers: Record<string, string | string[] | undefined>;
|
||||
/** Content-Type as sent by the client (the header itself is rewritten). */
|
||||
uploadContentType?: string | string[];
|
||||
body?: unknown;
|
||||
};
|
||||
@@ -0,0 +1,54 @@
|
||||
import { describe, expect, it } from 'vitest';
|
||||
import { UploadSlots } from './upload-slots.js';
|
||||
|
||||
const slots = (max: number, perUser: number) =>
|
||||
new UploadSlots({
|
||||
getOrThrow: (key: string) =>
|
||||
key === 'UPLOAD_MAX_IN_FLIGHT' ? max : perUser,
|
||||
} as never);
|
||||
|
||||
function hold() {
|
||||
const { promise, resolve } = Promise.withResolvers<void>();
|
||||
return { work: () => promise, release: resolve };
|
||||
}
|
||||
|
||||
describe('UploadSlots', () => {
|
||||
it('caps one user without starving others, and frees slots after work', async () => {
|
||||
const limiter = slots(10, 2);
|
||||
const held = [hold(), hold()];
|
||||
const running = held.map((h) => limiter.run('a', h.work));
|
||||
await expect(limiter.run('a', async () => 'x')).rejects.toMatchObject({
|
||||
status: 429,
|
||||
});
|
||||
await expect(limiter.run('b', async () => 'ok')).resolves.toBe('ok');
|
||||
held.forEach((h) => h.release());
|
||||
await Promise.all(running);
|
||||
await expect(limiter.run('a', async () => 'again')).resolves.toBe(
|
||||
'again',
|
||||
);
|
||||
});
|
||||
|
||||
it('applies the replica-wide cap across users', async () => {
|
||||
const limiter = slots(2, 2);
|
||||
const held = [hold(), hold()];
|
||||
const running = [
|
||||
limiter.run('a', held[0].work),
|
||||
limiter.run('b', held[1].work),
|
||||
];
|
||||
await expect(limiter.run('c', async () => 'x')).rejects.toMatchObject({
|
||||
status: 429,
|
||||
});
|
||||
held.forEach((h) => h.release());
|
||||
await Promise.all(running);
|
||||
});
|
||||
|
||||
it('releases a slot when the work fails', async () => {
|
||||
const limiter = slots(1, 1);
|
||||
await expect(
|
||||
limiter.run('a', async () => {
|
||||
throw new Error('boom');
|
||||
}),
|
||||
).rejects.toThrow('boom');
|
||||
await expect(limiter.run('a', async () => 'ok')).resolves.toBe('ok');
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,41 @@
|
||||
import { HttpException, Injectable } from '@nestjs/common';
|
||||
import { ConfigService } from '@nestjs/config';
|
||||
import type { AppConfig } from '../config/app-config.schema.js';
|
||||
|
||||
/**
|
||||
* Per-replica caps on requests that hold an image in memory (proxy upload,
|
||||
* direct complete): each may keep up to MAX_UPLOAD_BYTES for the decode check.
|
||||
* The per-user cap keeps one client (e.g. slow senders holding slots until
|
||||
* UPLOAD_TIMEOUT_MS) from starving everyone else.
|
||||
*/
|
||||
@Injectable()
|
||||
export class UploadSlots {
|
||||
private inFlight = 0;
|
||||
private readonly perUser = new Map<string, number>();
|
||||
private readonly max: number;
|
||||
private readonly maxPerUser: number;
|
||||
|
||||
constructor(config: ConfigService<AppConfig>) {
|
||||
this.max = config.getOrThrow('UPLOAD_MAX_IN_FLIGHT');
|
||||
this.maxPerUser = config.getOrThrow('UPLOAD_MAX_IN_FLIGHT_PER_USER');
|
||||
}
|
||||
|
||||
async run<T>(userId: string, work: () => Promise<T>): Promise<T> {
|
||||
const mine = this.perUser.get(userId) ?? 0;
|
||||
if (this.inFlight >= this.max || mine >= this.maxPerUser)
|
||||
throw new HttpException(
|
||||
{ message: 'Too many concurrent uploads', retryAfter: 1 },
|
||||
429,
|
||||
);
|
||||
this.inFlight += 1;
|
||||
this.perUser.set(userId, mine + 1);
|
||||
try {
|
||||
return await work();
|
||||
} finally {
|
||||
this.inFlight -= 1;
|
||||
const left = (this.perUser.get(userId) ?? 1) - 1;
|
||||
if (left === 0) this.perUser.delete(userId);
|
||||
else this.perUser.set(userId, left);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,79 @@
|
||||
import {
|
||||
BadRequestException,
|
||||
Body,
|
||||
Controller,
|
||||
Delete,
|
||||
Headers,
|
||||
HttpCode,
|
||||
HttpStatus,
|
||||
Post,
|
||||
UnauthorizedException,
|
||||
UseGuards,
|
||||
} from '@nestjs/common';
|
||||
import { TextDecoder } from 'node:util';
|
||||
import { AuthService } from './auth.service.js';
|
||||
import { CurrentUserGuard } from '../common/auth/current-user.guard.js';
|
||||
|
||||
const BASE64 =
|
||||
/^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$/u;
|
||||
const BASIC_CHALLENGE = 'Basic realm="messenger", charset="UTF-8"';
|
||||
|
||||
function basicUnauthorized(): UnauthorizedException {
|
||||
return new UnauthorizedException({
|
||||
message: 'Unauthorized',
|
||||
challenge: BASIC_CHALLENGE,
|
||||
});
|
||||
}
|
||||
|
||||
function bearerUnauthorized(): UnauthorizedException {
|
||||
return new UnauthorizedException({
|
||||
message: 'Unauthorized',
|
||||
challenge: 'Bearer realm="messenger"',
|
||||
});
|
||||
}
|
||||
|
||||
function parseBasic(value: string | undefined): [string, string] {
|
||||
if (typeof value !== 'string') throw basicUnauthorized();
|
||||
const match = /^Basic[ \t]+([^ \t]+)$/iu.exec(value);
|
||||
if (!match || !BASE64.test(match[1])) throw basicUnauthorized();
|
||||
const bytes = Buffer.from(match[1], 'base64');
|
||||
if (bytes.toString('base64') !== match[1]) throw basicUnauthorized();
|
||||
let decoded: string;
|
||||
try {
|
||||
decoded = new TextDecoder('utf-8', { fatal: true }).decode(bytes);
|
||||
} catch {
|
||||
throw basicUnauthorized();
|
||||
}
|
||||
const separator = decoded.indexOf(':');
|
||||
if (separator < 1) throw basicUnauthorized();
|
||||
return [decoded.slice(0, separator), decoded.slice(separator + 1)];
|
||||
}
|
||||
|
||||
@Controller('api/v1/auth/sessions')
|
||||
export class AuthController {
|
||||
constructor(private readonly auth: AuthService) {}
|
||||
|
||||
@Post()
|
||||
create(
|
||||
@Headers('authorization') authorization: string | undefined,
|
||||
@Body() body: unknown,
|
||||
) {
|
||||
const [username, password] = parseBasic(authorization);
|
||||
if (body !== undefined)
|
||||
throw new BadRequestException(
|
||||
'Session creation does not accept a body',
|
||||
);
|
||||
return this.auth.createSession(username, password);
|
||||
}
|
||||
|
||||
@Delete('current')
|
||||
@UseGuards(CurrentUserGuard)
|
||||
@HttpCode(HttpStatus.NO_CONTENT)
|
||||
async delete(
|
||||
@Headers('authorization') authorization: string | undefined,
|
||||
): Promise<void> {
|
||||
if (!authorization || !/^Bearer\s+[^\s]+$/iu.test(authorization))
|
||||
throw bearerUnauthorized();
|
||||
await this.auth.deleteSession(authorization.slice(7).trim());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
import { Module } from '@nestjs/common';
|
||||
import { DatabaseModule } from '../database/database.module.js';
|
||||
import { RedisModule } from '../redis/redis.module.js';
|
||||
import { UsersModule } from '../users/users.module.js';
|
||||
import { AuthController } from './auth.controller.js';
|
||||
import { AuthService } from './auth.service.js';
|
||||
import { SessionRepository } from './session.repository.js';
|
||||
import { MetricsModule } from '../metrics/metrics.module.js';
|
||||
|
||||
@Module({
|
||||
imports: [UsersModule, DatabaseModule, RedisModule, MetricsModule],
|
||||
controllers: [AuthController],
|
||||
providers: [AuthService, SessionRepository],
|
||||
exports: [AuthService, SessionRepository],
|
||||
})
|
||||
export class AuthModule {}
|
||||
@@ -0,0 +1,63 @@
|
||||
import {
|
||||
Injectable,
|
||||
ServiceUnavailableException,
|
||||
UnauthorizedException,
|
||||
} from '@nestjs/common';
|
||||
import argon2 from 'argon2';
|
||||
import type { User } from '../users/user.js';
|
||||
import { UsersService } from '../users/users.service.js';
|
||||
import { SessionRepository } from './session.repository.js';
|
||||
|
||||
export type SessionResponse = {
|
||||
token: string;
|
||||
token_type: 'Bearer';
|
||||
expires_at: string;
|
||||
user: User;
|
||||
};
|
||||
|
||||
@Injectable()
|
||||
export class AuthService {
|
||||
constructor(
|
||||
private readonly users: UsersService,
|
||||
private readonly sessions: SessionRepository,
|
||||
) {}
|
||||
|
||||
async createSession(
|
||||
username: string,
|
||||
password: string,
|
||||
): Promise<SessionResponse> {
|
||||
const storedUser = await this.users.findByUsername(username);
|
||||
if (
|
||||
!storedUser ||
|
||||
!(await argon2.verify(storedUser.password_hash, password))
|
||||
) {
|
||||
throw new UnauthorizedException();
|
||||
}
|
||||
const user: User = {
|
||||
id: storedUser.id,
|
||||
username: storedUser.username,
|
||||
display_name: storedUser.display_name,
|
||||
created_at: storedUser.created_at,
|
||||
};
|
||||
let session: { token: string; expiresAt: string };
|
||||
try {
|
||||
session = await this.sessions.create(user.id);
|
||||
} catch {
|
||||
throw new ServiceUnavailableException('Session store unavailable');
|
||||
}
|
||||
return {
|
||||
token: session.token,
|
||||
token_type: 'Bearer',
|
||||
expires_at: session.expiresAt,
|
||||
user,
|
||||
};
|
||||
}
|
||||
|
||||
async deleteSession(token: string): Promise<void> {
|
||||
try {
|
||||
await this.sessions.delete(token);
|
||||
} catch {
|
||||
throw new ServiceUnavailableException('Session store unavailable');
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,35 @@
|
||||
import { Injectable } from '@nestjs/common';
|
||||
import { ConfigService } from '@nestjs/config';
|
||||
import { randomBytes } from 'node:crypto';
|
||||
import type { AppConfig } from '../config/app-config.schema.js';
|
||||
import { RedisClient } from '../redis/redis.client.js';
|
||||
import { MetricsService } from '../metrics/metrics.service.js';
|
||||
|
||||
@Injectable()
|
||||
export class SessionRepository {
|
||||
constructor(
|
||||
private readonly redis: RedisClient,
|
||||
private readonly config: ConfigService<AppConfig>,
|
||||
private readonly metrics: MetricsService,
|
||||
) {}
|
||||
|
||||
async create(
|
||||
userId: string,
|
||||
): Promise<{ token: string; expiresAt: string }> {
|
||||
const token = randomBytes(32).toString('base64url');
|
||||
const ttl = this.config.getOrThrow('SESSION_TTL_SECONDS');
|
||||
const expiresAt = new Date(Date.now() + ttl * 1000).toISOString();
|
||||
this.metrics.increment('session_set');
|
||||
await this.redis.set(
|
||||
`session:${token}`,
|
||||
JSON.stringify({ user_id: userId, expires_at: expiresAt }),
|
||||
{ EX: ttl },
|
||||
);
|
||||
return { token, expiresAt };
|
||||
}
|
||||
|
||||
delete(token: string): Promise<number> {
|
||||
this.metrics.increment('session_del');
|
||||
return this.redis.del(`session:${token}`);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,6 @@
|
||||
export type Chat = {
|
||||
id: string;
|
||||
title: string;
|
||||
member_ids: string[];
|
||||
created_at: string;
|
||||
};
|
||||
@@ -0,0 +1,49 @@
|
||||
import {
|
||||
BadRequestException,
|
||||
Body,
|
||||
Controller,
|
||||
DefaultValuePipe,
|
||||
Get,
|
||||
Param,
|
||||
ParseIntPipe,
|
||||
ParseUUIDPipe,
|
||||
Post,
|
||||
Query,
|
||||
UseGuards,
|
||||
} from '@nestjs/common';
|
||||
import { CurrentUser } from '../common/auth/current-user.decorator.js';
|
||||
import { CurrentUserGuard } from '../common/auth/current-user.guard.js';
|
||||
import { ChatsService } from './chats.service.js';
|
||||
import { CreateChatDto } from './dto/create-chat.dto.js';
|
||||
import type { User } from '../users/user.js';
|
||||
|
||||
@Controller('api/v1/chats')
|
||||
@UseGuards(CurrentUserGuard)
|
||||
export class ChatsController {
|
||||
constructor(private readonly chatsService: ChatsService) {}
|
||||
|
||||
@Post()
|
||||
create(@Body() body: CreateChatDto, @CurrentUser() currentUser: User) {
|
||||
return this.chatsService.create(body, currentUser);
|
||||
}
|
||||
|
||||
@Get()
|
||||
list(
|
||||
@CurrentUser() currentUser: User,
|
||||
@Query('limit', new DefaultValuePipe(50), new ParseIntPipe())
|
||||
limit: number,
|
||||
@Query('cursor') cursor?: string,
|
||||
) {
|
||||
if (limit < 1 || limit > 100)
|
||||
throw new BadRequestException('limit must be between 1 and 100');
|
||||
return this.chatsService.listForUser(currentUser, limit, cursor);
|
||||
}
|
||||
|
||||
@Get(':chatId')
|
||||
get(
|
||||
@Param('chatId', new ParseUUIDPipe()) chatId: string,
|
||||
@CurrentUser() currentUser: User,
|
||||
) {
|
||||
return this.chatsService.getForUser(chatId, currentUser);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
import { Module } from '@nestjs/common';
|
||||
import { DatabaseModule } from '../database/database.module.js';
|
||||
import { RedisModule } from '../redis/redis.module.js';
|
||||
import { MetricsModule } from '../metrics/metrics.module.js';
|
||||
import { UsersModule } from '../users/users.module.js';
|
||||
import { ChatsController } from './chats.controller.js';
|
||||
import { ChatsRepository } from './chats.repository.js';
|
||||
import { ChatsService } from './chats.service.js';
|
||||
|
||||
@Module({
|
||||
imports: [UsersModule, DatabaseModule, RedisModule, MetricsModule],
|
||||
controllers: [ChatsController],
|
||||
providers: [ChatsRepository, ChatsService],
|
||||
exports: [ChatsService],
|
||||
})
|
||||
export class ChatsModule {}
|
||||
@@ -0,0 +1,109 @@
|
||||
import { BadRequestException, Injectable } from '@nestjs/common';
|
||||
import { randomUUID } from 'node:crypto';
|
||||
import { DatabaseClient } from '../database/database.client.js';
|
||||
import type { Cursor } from '../common/http/cursor.js';
|
||||
import type { User } from '../users/user.js';
|
||||
import type { Chat } from './chat.js';
|
||||
|
||||
type ChatRow = {
|
||||
id: string;
|
||||
title: string;
|
||||
created_at: Date | string;
|
||||
member_ids: string[];
|
||||
};
|
||||
|
||||
function toChat(row: ChatRow): Chat {
|
||||
return {
|
||||
id: row.id,
|
||||
title: row.title,
|
||||
created_at: new Date(row.created_at).toISOString(),
|
||||
member_ids: row.member_ids,
|
||||
};
|
||||
}
|
||||
|
||||
@Injectable()
|
||||
export class ChatsRepository {
|
||||
constructor(private readonly database: DatabaseClient) {}
|
||||
|
||||
async create(title: string, memberIds: string[]): Promise<Chat> {
|
||||
const id = randomUUID();
|
||||
return this.database.withTransaction(async (client) => {
|
||||
const users = await client.query<{ id: string }>({
|
||||
name: 'chats.validateMembers',
|
||||
text: 'SELECT id FROM users WHERE id = ANY($1::uuid[])',
|
||||
values: [memberIds],
|
||||
});
|
||||
if (users.rowCount !== memberIds.length)
|
||||
throw new BadRequestException('Chat member does not exist');
|
||||
const inserted = await client.query<ChatRow>({
|
||||
name: 'chats.insert',
|
||||
text: 'INSERT INTO chats (id, title) VALUES ($1, $2) RETURNING id, title, created_at',
|
||||
values: [id, title],
|
||||
});
|
||||
for (const memberId of memberIds) {
|
||||
await client.query({
|
||||
name: 'chats.insertMember',
|
||||
text: 'INSERT INTO chat_members (chat_id, user_id) VALUES ($1, $2)',
|
||||
values: [id, memberId],
|
||||
});
|
||||
}
|
||||
return {
|
||||
...toChat({ ...inserted.rows[0], member_ids: memberIds }),
|
||||
member_ids: memberIds,
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
async findForUser(id: string, user: User): Promise<Chat | undefined> {
|
||||
const result = await this.database.query<ChatRow>(
|
||||
'chats.findForUser',
|
||||
`SELECT c.id, c.title, c.created_at,
|
||||
COALESCE(array_agg(cm.user_id ORDER BY cm.user_id)
|
||||
FILTER (WHERE cm.user_id IS NOT NULL), '{}') AS member_ids
|
||||
FROM chats c
|
||||
LEFT JOIN chat_members cm ON cm.chat_id = c.id
|
||||
WHERE c.id = $1
|
||||
AND EXISTS (
|
||||
SELECT 1 FROM chat_members own
|
||||
WHERE own.chat_id = c.id AND own.user_id = $2
|
||||
)
|
||||
GROUP BY c.id`,
|
||||
[id, user.id],
|
||||
);
|
||||
return result.rows[0] ? toChat(result.rows[0]) : undefined;
|
||||
}
|
||||
|
||||
async findPageForUser(
|
||||
user: User,
|
||||
limit: number,
|
||||
cursor?: Cursor,
|
||||
): Promise<{ items: Chat[]; hasMore: boolean }> {
|
||||
const values: unknown[] = [user.id];
|
||||
let predicate = '';
|
||||
if (cursor) {
|
||||
values.push(cursor.created_at, cursor.id);
|
||||
predicate =
|
||||
'AND (c.created_at, c.id) > ($2::timestamptz, $3::uuid)';
|
||||
}
|
||||
values.push(limit + 1);
|
||||
const result = await this.database.query<ChatRow>(
|
||||
'chats.listForUser',
|
||||
`SELECT c.id, c.title, c.created_at,
|
||||
COALESCE(array_agg(cm.user_id ORDER BY cm.user_id)
|
||||
FILTER (WHERE cm.user_id IS NOT NULL), '{}') AS member_ids
|
||||
FROM chats c
|
||||
LEFT JOIN chat_members cm ON cm.chat_id = c.id
|
||||
WHERE EXISTS (
|
||||
SELECT 1 FROM chat_members own
|
||||
WHERE own.chat_id = c.id AND own.user_id = $1
|
||||
)
|
||||
${predicate}
|
||||
GROUP BY c.id ORDER BY c.created_at, c.id LIMIT $${values.length}`,
|
||||
values,
|
||||
);
|
||||
return {
|
||||
items: result.rows.slice(0, limit).map(toChat),
|
||||
hasMore: result.rows.length > limit,
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,61 @@
|
||||
import {
|
||||
BadRequestException,
|
||||
Injectable,
|
||||
NotFoundException,
|
||||
} from '@nestjs/common';
|
||||
import { encodeCursor, decodeCursor } from '../common/http/cursor.js';
|
||||
import type { Page } from '../common/http/page.js';
|
||||
import type { User } from '../users/user.js';
|
||||
import type { CreateChatDto } from './dto/create-chat.dto.js';
|
||||
import type { Chat } from './chat.js';
|
||||
import { ChatsRepository } from './chats.repository.js';
|
||||
|
||||
@Injectable()
|
||||
export class ChatsService {
|
||||
constructor(private readonly chatsRepository: ChatsRepository) {}
|
||||
|
||||
create(input: CreateChatDto, currentUser: User): Promise<Chat> {
|
||||
if (!input.member_ids.includes(currentUser.id))
|
||||
throw new BadRequestException('Current user must be a chat member');
|
||||
if (new Set(input.member_ids).size !== input.member_ids.length)
|
||||
throw new BadRequestException('Duplicate chat member');
|
||||
return this.chatsRepository.create(input.title, input.member_ids);
|
||||
}
|
||||
|
||||
async getForUser(chatId: string, currentUser: User): Promise<Chat> {
|
||||
const chat = await this.chatsRepository.findForUser(
|
||||
chatId,
|
||||
currentUser,
|
||||
);
|
||||
if (!chat) throw new NotFoundException();
|
||||
return chat;
|
||||
}
|
||||
|
||||
async listForUser(
|
||||
currentUser: User,
|
||||
limit: number,
|
||||
cursorValue?: string,
|
||||
): Promise<Page<Chat>> {
|
||||
const scope = `chats:${currentUser.id}`;
|
||||
const cursor = cursorValue
|
||||
? decodeCursor(cursorValue, scope)
|
||||
: undefined;
|
||||
const page = await this.chatsRepository.findPageForUser(
|
||||
currentUser,
|
||||
limit,
|
||||
cursor,
|
||||
);
|
||||
const last = page.items.at(-1);
|
||||
return {
|
||||
items: page.items,
|
||||
next_cursor:
|
||||
page.hasMore && last
|
||||
? encodeCursor({
|
||||
scope,
|
||||
created_at: last.created_at,
|
||||
id: last.id,
|
||||
})
|
||||
: null,
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,26 @@
|
||||
import {
|
||||
ArrayMaxSize,
|
||||
ArrayMinSize,
|
||||
ArrayUnique,
|
||||
IsArray,
|
||||
IsString,
|
||||
IsUUID,
|
||||
Matches,
|
||||
} from 'class-validator';
|
||||
import { CodePointLength } from '../../common/validation/code-point-length.js';
|
||||
|
||||
export class CreateChatDto {
|
||||
@IsString()
|
||||
@CodePointLength(1, 100)
|
||||
@Matches(/\S/, {
|
||||
message: 'title must contain a non-whitespace character',
|
||||
})
|
||||
title: string;
|
||||
|
||||
@IsArray()
|
||||
@ArrayMinSize(1)
|
||||
@ArrayMaxSize(100)
|
||||
@ArrayUnique()
|
||||
@IsUUID('4', { each: true })
|
||||
member_ids: string[];
|
||||
}
|
||||
@@ -0,0 +1,10 @@
|
||||
import { createParamDecorator, ExecutionContext } from '@nestjs/common';
|
||||
import type { User } from '../../users/user.js';
|
||||
import { FastifyRequest } from 'fastify';
|
||||
|
||||
export const CurrentUser = createParamDecorator(
|
||||
(_data: unknown, context: ExecutionContext): User => {
|
||||
const request = context.switchToHttp().getRequest<FastifyRequest>();
|
||||
return request.currentUser!;
|
||||
},
|
||||
);
|
||||
@@ -0,0 +1,85 @@
|
||||
import {
|
||||
CanActivate,
|
||||
ExecutionContext,
|
||||
Injectable,
|
||||
ServiceUnavailableException,
|
||||
UnauthorizedException,
|
||||
} from '@nestjs/common';
|
||||
import type { FastifyRequest } from 'fastify';
|
||||
import type { User } from '../../users/user.js';
|
||||
import { RedisClient } from '../../redis/redis.client.js';
|
||||
import { UsersService } from '../../users/users.service.js';
|
||||
import { MetricsService } from '../../metrics/metrics.service.js';
|
||||
|
||||
@Injectable()
|
||||
export class CurrentUserGuard implements CanActivate {
|
||||
constructor(
|
||||
private readonly redis: RedisClient,
|
||||
private readonly users: UsersService,
|
||||
private readonly metrics: MetricsService,
|
||||
) {}
|
||||
|
||||
async canActivate(context: ExecutionContext): Promise<boolean> {
|
||||
const request = context.switchToHttp().getRequest<FastifyRequest>();
|
||||
const authorization = request.headers.authorization;
|
||||
if (
|
||||
typeof authorization !== 'string' ||
|
||||
!/^Bearer\s+[^\s]+$/u.test(authorization)
|
||||
) {
|
||||
throw new UnauthorizedException({
|
||||
message: 'Unauthorized',
|
||||
challenge: 'Bearer realm="messenger"',
|
||||
});
|
||||
}
|
||||
let session: string | null;
|
||||
try {
|
||||
this.metrics.increment('session_get');
|
||||
session = await this.redis.get(
|
||||
`session:${authorization.slice(7).trim()}`,
|
||||
);
|
||||
} catch {
|
||||
throw new ServiceUnavailableException('Session store unavailable');
|
||||
}
|
||||
if (!session)
|
||||
throw new UnauthorizedException({
|
||||
message: 'Unauthorized',
|
||||
challenge: 'Bearer realm="messenger"',
|
||||
});
|
||||
let userId: string;
|
||||
try {
|
||||
const parsed: unknown = JSON.parse(session);
|
||||
if (
|
||||
typeof parsed !== 'object' ||
|
||||
parsed === null ||
|
||||
!('user_id' in parsed) ||
|
||||
typeof parsed.user_id !== 'string' ||
|
||||
!('expires_at' in parsed) ||
|
||||
typeof parsed.expires_at !== 'string' ||
|
||||
!/^(?:\d{4})-(?:0[1-9]|1[0-2])-(?:0[1-9]|[12]\d|3[01])T(?:[01]\d|2[0-3]):[0-5]\d:[0-5]\d(?:\.\d+)?(?:Z|[+-](?:[01]\d|2[0-3]):[0-5]\d)$/u.test(
|
||||
parsed.expires_at,
|
||||
)
|
||||
)
|
||||
throw new Error('invalid session');
|
||||
const expiresAt = Date.parse(parsed.expires_at);
|
||||
if (!Number.isFinite(expiresAt) || Date.now() >= expiresAt)
|
||||
throw new Error('expired session');
|
||||
userId = parsed.user_id;
|
||||
} catch {
|
||||
throw new UnauthorizedException({
|
||||
message: 'Unauthorized',
|
||||
challenge: 'Bearer realm="messenger"',
|
||||
});
|
||||
}
|
||||
let user: User | undefined;
|
||||
try {
|
||||
user = await this.users.findById(userId);
|
||||
} catch {
|
||||
throw new ServiceUnavailableException(
|
||||
'Authentication service unavailable',
|
||||
);
|
||||
}
|
||||
if (!user) throw new UnauthorizedException();
|
||||
request.currentUser = user;
|
||||
return true;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,7 @@
|
||||
export type ApiErrorBody = {
|
||||
error: {
|
||||
code: string;
|
||||
message: string;
|
||||
request_id: string;
|
||||
};
|
||||
};
|
||||
@@ -0,0 +1,181 @@
|
||||
import {
|
||||
ArgumentsHost,
|
||||
Catch,
|
||||
ExceptionFilter,
|
||||
HttpException,
|
||||
HttpStatus,
|
||||
Logger,
|
||||
} from '@nestjs/common';
|
||||
import { HttpAdapterHost } from '@nestjs/core';
|
||||
import { ConfigService } from '@nestjs/config';
|
||||
import { randomUUID } from 'node:crypto';
|
||||
import type { AppConfig } from '../../config/app-config.schema.js';
|
||||
import { resolveInstanceId } from '../../config/instance-id.js';
|
||||
import type { ApiErrorBody } from './api-error.js';
|
||||
|
||||
type RequestWithId = { requestId?: string; method?: string; url?: string };
|
||||
type ExceptionBody = {
|
||||
message?: unknown;
|
||||
challenge?: unknown;
|
||||
retryAfter?: unknown;
|
||||
};
|
||||
|
||||
// Connection/session loss codes reported by node-postgres and the network layer.
|
||||
const UNAVAILABLE_ERROR_CODES: Record<string, true> = {
|
||||
ECONNREFUSED: true,
|
||||
ECONNRESET: true,
|
||||
ENOTFOUND: true,
|
||||
ETIMEDOUT: true,
|
||||
EPIPE: true,
|
||||
'08000': true,
|
||||
'08001': true,
|
||||
'08003': true,
|
||||
'08004': true,
|
||||
'08006': true,
|
||||
'57P01': true,
|
||||
'57P02': true,
|
||||
'57P03': true,
|
||||
'53300': true,
|
||||
// Transient aborts: the statement was cancelled by statement_timeout, the
|
||||
// transaction was rolled back by a serialization failure or a deadlock. The
|
||||
// work was never confirmed, so callers see a retryable 503 instead of an
|
||||
// internal error. Without this, such an abort would be recorded as the final
|
||||
// result of an idempotent operation.
|
||||
'57014': true,
|
||||
'40001': true,
|
||||
'40P01': true,
|
||||
};
|
||||
|
||||
// Some driver failures carry no `code` at all: node-postgres raises plain errors
|
||||
// when a connection dies or its pool attempt times out, and node-redis reports a
|
||||
// closed socket the same way. Without these patterns a stopped database would
|
||||
// surface as an internal fault instead of a bounded outage.
|
||||
const UNAVAILABLE_ERROR_MESSAGES = [
|
||||
'Connection terminated unexpectedly',
|
||||
'Connection terminated due to connection timeout',
|
||||
'timeout exceeded when trying to connect',
|
||||
'Client has encountered a connection error and is not queryable',
|
||||
'Socket closed unexpectedly',
|
||||
'Connection is closed',
|
||||
];
|
||||
|
||||
function isDependencyOutage(exception: unknown): boolean {
|
||||
if (typeof exception !== 'object' || exception === null) return false;
|
||||
const code = (exception as { code?: unknown }).code;
|
||||
if (typeof code === 'string' && UNAVAILABLE_ERROR_CODES[code] === true)
|
||||
return true;
|
||||
const message = (exception as { message?: unknown }).message;
|
||||
return (
|
||||
typeof message === 'string' &&
|
||||
UNAVAILABLE_ERROR_MESSAGES.some((pattern) => message.includes(pattern))
|
||||
);
|
||||
}
|
||||
|
||||
function errorCodeForStatus(status: number): string {
|
||||
const codes: Record<number, string> = {
|
||||
[HttpStatus.BAD_REQUEST]: 'invalid_request',
|
||||
[HttpStatus.UNAUTHORIZED]: 'unauthorized',
|
||||
[HttpStatus.NOT_FOUND]: 'not_found',
|
||||
[HttpStatus.CONFLICT]: 'conflict',
|
||||
[HttpStatus.PAYLOAD_TOO_LARGE]: 'payload_too_large',
|
||||
[HttpStatus.UNSUPPORTED_MEDIA_TYPE]: 'unsupported_media_type',
|
||||
[HttpStatus.TOO_MANY_REQUESTS]: 'rate_limited',
|
||||
[HttpStatus.SERVICE_UNAVAILABLE]: 'unavailable',
|
||||
};
|
||||
return codes[status] ?? 'internal_error';
|
||||
}
|
||||
|
||||
@Catch()
|
||||
export class ApiExceptionFilter implements ExceptionFilter {
|
||||
private readonly logger = new Logger(ApiExceptionFilter.name);
|
||||
// Guards and pipes run before interceptors, so an error response may be the
|
||||
// first thing the filter writes: it must carry the replica identity too.
|
||||
private readonly instanceId: string;
|
||||
|
||||
constructor(
|
||||
private readonly adapterHost: HttpAdapterHost,
|
||||
config: ConfigService<AppConfig>,
|
||||
) {
|
||||
this.instanceId = resolveInstanceId(config);
|
||||
}
|
||||
|
||||
catch(exception: unknown, host: ArgumentsHost): void {
|
||||
const http = host.switchToHttp();
|
||||
const request = http.getRequest<RequestWithId>();
|
||||
const response = http.getResponse<unknown>();
|
||||
const adapter = this.adapterHost.httpAdapter;
|
||||
const requestId = request.requestId ?? randomUUID();
|
||||
request.requestId = requestId;
|
||||
|
||||
let status = HttpStatus.INTERNAL_SERVER_ERROR;
|
||||
let message = 'Internal server error';
|
||||
let challenge: string | undefined;
|
||||
let retryAfter = 1;
|
||||
if (exception instanceof HttpException) {
|
||||
status = exception.getStatus();
|
||||
const body = exception.getResponse();
|
||||
const details: ExceptionBody =
|
||||
typeof body === 'object' && body !== null
|
||||
? (body as ExceptionBody)
|
||||
: {};
|
||||
const rawMessage =
|
||||
'message' in details ? details.message : undefined;
|
||||
challenge =
|
||||
typeof details.challenge === 'string'
|
||||
? details.challenge
|
||||
: undefined;
|
||||
if (
|
||||
typeof details.retryAfter === 'number' &&
|
||||
Number.isFinite(details.retryAfter) &&
|
||||
details.retryAfter >= 1
|
||||
)
|
||||
retryAfter = Math.ceil(details.retryAfter);
|
||||
message = Array.isArray(rawMessage)
|
||||
? rawMessage.join('; ')
|
||||
: typeof rawMessage === 'string'
|
||||
? rawMessage
|
||||
: 'Request failed';
|
||||
if (status === HttpStatus.NOT_FOUND) message = 'Resource not found';
|
||||
if (status >= HttpStatus.INTERNAL_SERVER_ERROR)
|
||||
message =
|
||||
status === HttpStatus.SERVICE_UNAVAILABLE
|
||||
? 'Service unavailable'
|
||||
: 'Internal server error';
|
||||
} else if (isDependencyOutage(exception)) {
|
||||
// A dropped dependency is a bounded outage, not an internal fault.
|
||||
status = HttpStatus.SERVICE_UNAVAILABLE;
|
||||
message = 'Service unavailable';
|
||||
this.logger.warn(
|
||||
`Dependency outage on ${request.method ?? ''} ${request.url ?? ''}: ${exception instanceof Error ? exception.message : 'unknown error'}`,
|
||||
);
|
||||
} else {
|
||||
this.logger.error(
|
||||
`Unhandled exception on ${request.method ?? ''} ${request.url ?? ''}`,
|
||||
exception instanceof Error
|
||||
? exception.stack
|
||||
: String(exception),
|
||||
);
|
||||
}
|
||||
if (status === HttpStatus.UNAUTHORIZED) {
|
||||
const authChallenge =
|
||||
challenge ??
|
||||
(request.method === 'POST' &&
|
||||
request.url?.startsWith('/api/v1/auth/sessions')
|
||||
? 'Basic realm="messenger", charset="UTF-8"'
|
||||
: 'Bearer realm="messenger"');
|
||||
adapter.setHeader(response, 'WWW-Authenticate', authChallenge);
|
||||
}
|
||||
if (status === HttpStatus.TOO_MANY_REQUESTS)
|
||||
adapter.setHeader(response, 'Retry-After', String(retryAfter));
|
||||
adapter.setHeader(response, 'X-Request-Id', requestId);
|
||||
adapter.setHeader(response, 'X-Instance-Id', this.instanceId);
|
||||
const payload: ApiErrorBody = {
|
||||
error: {
|
||||
code: errorCodeForStatus(status),
|
||||
message,
|
||||
request_id: requestId,
|
||||
},
|
||||
};
|
||||
adapter.reply(response, payload, status);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,85 @@
|
||||
import { BadRequestException } from '@nestjs/common';
|
||||
import { randomUUID } from 'node:crypto';
|
||||
import { describe, expect, it } from 'vitest';
|
||||
import { decodeCursor, encodeCursor } from './cursor.js';
|
||||
|
||||
const scope = 'messages:6f1c2a3e-1111-4222-8333-444455556666';
|
||||
|
||||
function encode(createdAt: string, cursorScope = scope): string {
|
||||
return encodeCursor({
|
||||
scope: cursorScope,
|
||||
created_at: createdAt,
|
||||
id: randomUUID(),
|
||||
});
|
||||
}
|
||||
|
||||
describe('decodeCursor', () => {
|
||||
it('accepts valid instants and normalizes them to canonical milliseconds', () => {
|
||||
const accepted: Array<[string, string]> = [
|
||||
['2026-01-01T00:00:00.000Z', '2026-01-01T00:00:00.000Z'],
|
||||
['2026-01-01T00:00:00Z', '2026-01-01T00:00:00.000Z'],
|
||||
['2026-01-01T00:00:00.999999Z', '2026-01-01T00:00:00.999Z'],
|
||||
['2024-02-29T23:59:59.000Z', '2024-02-29T23:59:59.000Z'],
|
||||
['2026-12-31T23:59:59.999Z', '2026-12-31T23:59:59.999Z'],
|
||||
];
|
||||
for (const [createdAt, expected] of accepted) {
|
||||
const decoded = decodeCursor(encode(createdAt), scope);
|
||||
expect(decoded.created_at).toBe(expected);
|
||||
expect(decoded.scope).toBe(scope);
|
||||
}
|
||||
});
|
||||
|
||||
it('rejects calendar dates that the platform parser rolls over', () => {
|
||||
// 2026-02-31 would silently become 2026-03-03 and then fail inside SQL.
|
||||
for (const createdAt of [
|
||||
'2026-02-31T00:00:00.000Z',
|
||||
'2025-02-29T00:00:00.000Z',
|
||||
'2026-04-31T00:00:00.000Z',
|
||||
'2026-06-31T12:00:00.000Z',
|
||||
]) {
|
||||
expect(() => decodeCursor(encode(createdAt), scope)).toThrow(
|
||||
BadRequestException,
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
it('rejects out-of-range and malformed timestamps', () => {
|
||||
for (const createdAt of [
|
||||
'2026-01-01T24:00:00.000Z',
|
||||
'2026-01-01T00:00:60.000Z',
|
||||
'2026-13-01T00:00:00.000Z',
|
||||
'2026-00-01T00:00:00.000Z',
|
||||
'2026-01-01T00:00:00',
|
||||
'2026-01-01T00:00:00+02:00',
|
||||
'2026-1-1T00:00:00.000Z',
|
||||
'2026-01-01',
|
||||
'not-a-timestamp',
|
||||
]) {
|
||||
expect(() => decodeCursor(encode(createdAt), scope)).toThrow(
|
||||
BadRequestException,
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
it('rejects a cursor from another scope, a malformed id, and non-cursor input', () => {
|
||||
expect(() =>
|
||||
decodeCursor(
|
||||
encode('2026-01-01T00:00:00.000Z', 'messages:other'),
|
||||
scope,
|
||||
),
|
||||
).toThrow(BadRequestException);
|
||||
expect(() =>
|
||||
decodeCursor(
|
||||
encodeCursor({
|
||||
scope,
|
||||
created_at: '2026-01-01T00:00:00.000Z',
|
||||
id: 'not-a-uuid',
|
||||
}),
|
||||
scope,
|
||||
),
|
||||
).toThrow(BadRequestException);
|
||||
expect(() => decodeCursor('not-a-cursor', scope)).toThrow(
|
||||
BadRequestException,
|
||||
);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,54 @@
|
||||
import { BadRequestException } from '@nestjs/common';
|
||||
import { isValid, parseISO } from 'date-fns';
|
||||
|
||||
export type Cursor = {
|
||||
scope: string;
|
||||
created_at: string;
|
||||
id: string;
|
||||
};
|
||||
|
||||
// Format check for the canonical API shape: `Date.prototype.toISOString()` with
|
||||
// optional fractional seconds. Explicit ranges reject values such as 24:00:00.
|
||||
// Calendar validity (2026-02-31, 2025-02-29, 2026-04-31) is checked by
|
||||
// date-fns `parseISO`, which refuses impossible dates that the platform parser
|
||||
// would silently roll over into another day.
|
||||
const CURSOR_TIMESTAMP =
|
||||
/^\d{4}-(?:0[1-9]|1[0-2])-(?:0[1-9]|[12]\d|3[01])T(?:[01]\d|2[0-3]):[0-5]\d:[0-5]\d(?:\.\d+)?Z$/u;
|
||||
|
||||
export function encodeCursor(cursor: Cursor): string {
|
||||
return Buffer.from(JSON.stringify(cursor), 'utf8').toString('base64url');
|
||||
}
|
||||
|
||||
export function decodeCursor(value: string, expectedScope: string): Cursor {
|
||||
try {
|
||||
const parsed: unknown = JSON.parse(
|
||||
Buffer.from(value, 'base64url').toString('utf8'),
|
||||
);
|
||||
if (
|
||||
typeof parsed !== 'object' ||
|
||||
parsed === null ||
|
||||
!('scope' in parsed) ||
|
||||
!('created_at' in parsed) ||
|
||||
!('id' in parsed) ||
|
||||
typeof parsed.scope !== 'string' ||
|
||||
typeof parsed.created_at !== 'string' ||
|
||||
typeof parsed.id !== 'string' ||
|
||||
parsed.scope !== expectedScope ||
|
||||
!/^[\da-f]{8}-[\da-f]{4}-[\da-f]{4}-[\da-f]{4}-[\da-f]{12}$/iu.test(
|
||||
parsed.id,
|
||||
) ||
|
||||
!CURSOR_TIMESTAMP.test(parsed.created_at)
|
||||
)
|
||||
throw new Error('invalid cursor');
|
||||
const instant = parseISO(parsed.created_at);
|
||||
if (!isValid(instant)) throw new Error('invalid cursor');
|
||||
|
||||
return {
|
||||
scope: parsed.scope,
|
||||
created_at: instant.toISOString(),
|
||||
id: parsed.id,
|
||||
};
|
||||
} catch {
|
||||
throw new BadRequestException('Invalid cursor');
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,4 @@
|
||||
export type Page<T> = {
|
||||
items: T[];
|
||||
next_cursor: string | null;
|
||||
};
|
||||
@@ -0,0 +1,59 @@
|
||||
import {
|
||||
CallHandler,
|
||||
ExecutionContext,
|
||||
Injectable,
|
||||
NestInterceptor,
|
||||
} from '@nestjs/common';
|
||||
import { ConfigService } from '@nestjs/config';
|
||||
import { HttpAdapterHost } from '@nestjs/core';
|
||||
import { randomUUID } from 'node:crypto';
|
||||
import { finalize, Observable } from 'rxjs';
|
||||
import type { AppConfig } from '../../config/app-config.schema.js';
|
||||
import { resolveInstanceId } from '../../config/instance-id.js';
|
||||
import { MetricsService } from '../../metrics/metrics.service.js';
|
||||
|
||||
type RequestWithId = { requestId?: string };
|
||||
|
||||
@Injectable()
|
||||
export class RequestIdInterceptor implements NestInterceptor {
|
||||
// Resolved once: the identity must be stable for the whole process lifetime,
|
||||
// otherwise a client cannot tell whether it moved to another replica.
|
||||
private readonly instanceId: string;
|
||||
|
||||
constructor(
|
||||
private readonly adapterHost: HttpAdapterHost,
|
||||
private readonly metrics: MetricsService,
|
||||
config: ConfigService<AppConfig>,
|
||||
) {
|
||||
this.instanceId = resolveInstanceId(config);
|
||||
}
|
||||
|
||||
intercept(
|
||||
context: ExecutionContext,
|
||||
next: CallHandler,
|
||||
): Observable<unknown> {
|
||||
const request = context.switchToHttp().getRequest<RequestWithId>();
|
||||
const response = context.switchToHttp().getResponse<unknown>();
|
||||
const requestId = randomUUID();
|
||||
const startedAt = performance.now();
|
||||
request.requestId = requestId;
|
||||
this.adapterHost.httpAdapter.setHeader(
|
||||
response,
|
||||
'X-Request-Id',
|
||||
requestId,
|
||||
);
|
||||
this.adapterHost.httpAdapter.setHeader(
|
||||
response,
|
||||
'X-Instance-Id',
|
||||
this.instanceId,
|
||||
);
|
||||
this.metrics.increment('request_count');
|
||||
return next
|
||||
.handle()
|
||||
.pipe(
|
||||
finalize(() =>
|
||||
this.metrics.observeLatency(performance.now() - startedAt),
|
||||
),
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,43 @@
|
||||
import { validate } from 'class-validator';
|
||||
import { describe, expect, it } from 'vitest';
|
||||
import { CodePointLength } from './code-point-length.js';
|
||||
|
||||
class Sample {
|
||||
@CodePointLength(3, 5)
|
||||
value!: string;
|
||||
}
|
||||
|
||||
async function fails(value: unknown): Promise<boolean> {
|
||||
const sample = new Sample();
|
||||
(sample as { value: unknown }).value = value;
|
||||
const errors = await validate(sample);
|
||||
return errors.length > 0;
|
||||
}
|
||||
|
||||
describe('CodePointLength', () => {
|
||||
it('counts Unicode code points instead of UTF-16 units', async () => {
|
||||
// '🔐' is one code point but two UTF-16 units.
|
||||
expect(await fails('🔐🔐')).toBe(true);
|
||||
expect(await fails('🔐🔐🔐')).toBe(false);
|
||||
expect(await fails('🔐🔐🔐🔐🔐')).toBe(false);
|
||||
expect(await fails('🔐🔐🔐🔐🔐🔐')).toBe(true);
|
||||
// Combining marks are separate code points, not grapheme clusters:
|
||||
// three 'é' sequences are six code points and exceed the maximum.
|
||||
expect(await fails('e\u0301e\u0301')).toBe(false);
|
||||
expect(await fails('e\u0301e\u0301e\u0301')).toBe(true);
|
||||
});
|
||||
|
||||
it('applies to BMP text boundaries', async () => {
|
||||
expect(await fails('ab')).toBe(true);
|
||||
expect(await fails('abc')).toBe(false);
|
||||
expect(await fails('abcde')).toBe(false);
|
||||
expect(await fails('abcdef')).toBe(true);
|
||||
});
|
||||
|
||||
it('rejects empty and non-string values', async () => {
|
||||
expect(await fails('')).toBe(true);
|
||||
expect(await fails(undefined)).toBe(true);
|
||||
expect(await fails(12345)).toBe(true);
|
||||
expect(await fails(['a', 'b', 'c'])).toBe(true);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,38 @@
|
||||
import {
|
||||
registerDecorator,
|
||||
ValidatorConstraint,
|
||||
type ValidationArguments,
|
||||
type ValidatorConstraintInterface,
|
||||
} from 'class-validator';
|
||||
|
||||
// JSON Schema (and the course contract) measure string lengths in Unicode code
|
||||
// points, while String.prototype.length counts UTF-16 units: one emoji outside
|
||||
// the BMP is two units. `@Length` therefore accepted an 11-emoji password
|
||||
// (22 units) and rejected 4000 emoji (8000 units), so these fields use an
|
||||
// explicit code-point validator instead.
|
||||
@ValidatorConstraint({ name: 'codePointLength', async: false })
|
||||
class CodePointLengthConstraint implements ValidatorConstraintInterface {
|
||||
validate(value: unknown, args: ValidationArguments): boolean {
|
||||
const [min, max] = args.constraints as [number, number];
|
||||
if (typeof value !== 'string') return false;
|
||||
const length = [...value].length;
|
||||
return length >= min && length <= max;
|
||||
}
|
||||
|
||||
defaultMessage(args: ValidationArguments): string {
|
||||
const [min, max] = args.constraints as [number, number];
|
||||
return `${args.property} must be between ${min} and ${max} code points`;
|
||||
}
|
||||
}
|
||||
|
||||
export function CodePointLength(min: number, max: number): PropertyDecorator {
|
||||
return (target, propertyKey) => {
|
||||
registerDecorator({
|
||||
name: 'codePointLength',
|
||||
target: target.constructor,
|
||||
propertyName: String(propertyKey),
|
||||
constraints: [min, max],
|
||||
validator: CodePointLengthConstraint,
|
||||
});
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
import { Global, Module } from '@nestjs/common';
|
||||
import { ConfigModule } from '@nestjs/config';
|
||||
import { parseAppConfig } from './app-config.schema.js';
|
||||
|
||||
@Global()
|
||||
@Module({
|
||||
imports: [
|
||||
ConfigModule.forRoot({
|
||||
isGlobal: true,
|
||||
cache: true,
|
||||
validate: parseAppConfig,
|
||||
}),
|
||||
],
|
||||
exports: [ConfigModule],
|
||||
})
|
||||
export class AppConfigModule {}
|
||||
@@ -0,0 +1,255 @@
|
||||
import { readFileSync } from 'node:fs';
|
||||
import { z } from 'zod';
|
||||
|
||||
function numeric(
|
||||
name: string,
|
||||
minimum: number,
|
||||
maximum: number | undefined,
|
||||
fallback: number,
|
||||
) {
|
||||
let schema = z
|
||||
.number({ message: `${name} must be a number` })
|
||||
.int()
|
||||
.min(minimum);
|
||||
if (maximum !== undefined) schema = schema.max(maximum);
|
||||
return z.preprocess(
|
||||
(value) =>
|
||||
typeof value === 'string' && value.trim() !== ''
|
||||
? Number(value)
|
||||
: value,
|
||||
schema.default(fallback),
|
||||
);
|
||||
}
|
||||
|
||||
function requiredString(name: string) {
|
||||
return z
|
||||
.string({ message: `${name} is required` })
|
||||
.min(1, `${name} is required`);
|
||||
}
|
||||
|
||||
export const appConfigSchema = z
|
||||
.object({
|
||||
APP_HOST: z.string().min(1).default('0.0.0.0'),
|
||||
APP_PORT: numeric('APP_PORT', 1, 65535, 8080),
|
||||
APP_ENV: z.string().min(1).default('lab'),
|
||||
SESSION_TTL_SECONDS: numeric('SESSION_TTL_SECONDS', 2, undefined, 3600),
|
||||
DATABASE_URL: requiredString('DATABASE_URL'),
|
||||
REDIS_URL: requiredString('REDIS_URL'),
|
||||
BROKER_URL: requiredString('BROKER_URL'),
|
||||
BROKER_PUBLISH_TIMEOUT_MS: numeric(
|
||||
'BROKER_PUBLISH_TIMEOUT_MS',
|
||||
100,
|
||||
60000,
|
||||
5000,
|
||||
),
|
||||
DATABASE_POOL_MAX: numeric('DATABASE_POOL_MAX', 1, 100, 10),
|
||||
DATABASE_CONNECTION_TIMEOUT_MS: numeric(
|
||||
'DATABASE_CONNECTION_TIMEOUT_MS',
|
||||
1,
|
||||
undefined,
|
||||
5000,
|
||||
),
|
||||
DATABASE_STATEMENT_TIMEOUT_MS: numeric(
|
||||
'DATABASE_STATEMENT_TIMEOUT_MS',
|
||||
1,
|
||||
undefined,
|
||||
5000,
|
||||
),
|
||||
DATABASE_IDLE_TX_TIMEOUT_MS: numeric(
|
||||
'DATABASE_IDLE_TX_TIMEOUT_MS',
|
||||
1000,
|
||||
undefined,
|
||||
30000,
|
||||
),
|
||||
// Бюджет зависимостей в /health/ready: LB проверяет ту же ручку, поэтому
|
||||
// проба готовности обязана уложиться в timeout check HAProxy (500 ms).
|
||||
HEALTH_TIMEOUT_MS: numeric('HEALTH_TIMEOUT_MS', 1, undefined, 400),
|
||||
CACHE_TTL_SECONDS: numeric('CACHE_TTL_SECONDS', 1, undefined, 30),
|
||||
CACHE_MODE: z.enum(['protected', 'naive']).default('protected'),
|
||||
CACHE_LOCK_TTL_MS: numeric('CACHE_LOCK_TTL_MS', 1, undefined, 1000),
|
||||
CACHE_MISS_DELAY_MS: numeric('CACHE_MISS_DELAY_MS', 0, undefined, 0),
|
||||
METRICS_TOKEN: z.string().min(1).optional(),
|
||||
DRAIN_DELAY_MS: numeric('DRAIN_DELAY_MS', 0, undefined, 5000),
|
||||
INSTANCE_ID: z.string().min(1).optional(),
|
||||
IDEMPOTENCY_TTL_SECONDS: numeric(
|
||||
'IDEMPOTENCY_TTL_SECONDS',
|
||||
60,
|
||||
undefined,
|
||||
86400,
|
||||
),
|
||||
S3_ENDPOINT: requiredString('S3_ENDPOINT'),
|
||||
S3_PUBLIC_ENDPOINT: requiredString('S3_PUBLIC_ENDPOINT'),
|
||||
S3_REGION: z.string().min(1).default('us-east-1'),
|
||||
S3_BUCKET: requiredString('S3_BUCKET'),
|
||||
S3_ACCESS_KEY_ID: requiredString('S3_ACCESS_KEY_ID'),
|
||||
S3_SECRET_ACCESS_KEY: requiredString('S3_SECRET_ACCESS_KEY'),
|
||||
S3_CONNECT_TIMEOUT_MS: numeric(
|
||||
'S3_CONNECT_TIMEOUT_MS',
|
||||
1,
|
||||
undefined,
|
||||
1000,
|
||||
),
|
||||
S3_REQUEST_TIMEOUT_MS: numeric(
|
||||
'S3_REQUEST_TIMEOUT_MS',
|
||||
1,
|
||||
undefined,
|
||||
15000,
|
||||
),
|
||||
MAX_UPLOAD_BYTES: numeric('MAX_UPLOAD_BYTES', 1, 10485760, 10485760),
|
||||
MAX_IMAGE_PIXELS: numeric('MAX_IMAGE_PIXELS', 1, 40000000, 40000000),
|
||||
UPLOAD_TIMEOUT_MS: numeric('UPLOAD_TIMEOUT_MS', 1000, undefined, 25000),
|
||||
UPLOAD_MAX_IN_FLIGHT: numeric('UPLOAD_MAX_IN_FLIGHT', 1, 1024, 16),
|
||||
UPLOAD_MAX_IN_FLIGHT_PER_USER: numeric(
|
||||
'UPLOAD_MAX_IN_FLIGHT_PER_USER',
|
||||
1,
|
||||
1024,
|
||||
4,
|
||||
),
|
||||
IMAGE_DECODE_MAX_CONCURRENT: numeric(
|
||||
'IMAGE_DECODE_MAX_CONCURRENT',
|
||||
1,
|
||||
64,
|
||||
1,
|
||||
),
|
||||
// 0 отключает фоновый проход уборщика (остаётся ручной CLI).
|
||||
ATTACHMENT_SWEEP_INTERVAL_MS: numeric(
|
||||
'ATTACHMENT_SWEEP_INTERVAL_MS',
|
||||
0,
|
||||
undefined,
|
||||
60000,
|
||||
),
|
||||
ATTACHMENT_SWEEP_GRACE_SECONDS: numeric(
|
||||
'ATTACHMENT_SWEEP_GRACE_SECONDS',
|
||||
0,
|
||||
undefined,
|
||||
900,
|
||||
),
|
||||
DIRECT_UPLOAD_URL_TTL_SECONDS: numeric(
|
||||
'DIRECT_UPLOAD_URL_TTL_SECONDS',
|
||||
1,
|
||||
3600,
|
||||
300,
|
||||
),
|
||||
DIRECT_UPLOAD_MAX_PENDING_PER_USER: numeric(
|
||||
'DIRECT_UPLOAD_MAX_PENDING_PER_USER',
|
||||
1,
|
||||
1000,
|
||||
10,
|
||||
),
|
||||
WORKER_CONCURRENCY: numeric('WORKER_CONCURRENCY', 1, 16, 2),
|
||||
JOB_MAX_ATTEMPTS: numeric('JOB_MAX_ATTEMPTS', 1, 20, 5),
|
||||
JOB_TIMEOUT_SECONDS: numeric('JOB_TIMEOUT_SECONDS', 1, 600, 60),
|
||||
JOB_LEASE_SECONDS: numeric('JOB_LEASE_SECONDS', 3, 600, 30),
|
||||
JOB_RETRY_BASE_MS: numeric('JOB_RETRY_BASE_MS', 10, undefined, 1000),
|
||||
JOB_RETRY_MAX_MS: numeric('JOB_RETRY_MAX_MS', 10, undefined, 30000),
|
||||
PIPELINE_VERSION: numeric('PIPELINE_VERSION', 1, 2, 1),
|
||||
BROKER_DELIVERY_LIMIT: numeric('BROKER_DELIVERY_LIMIT', 1, 100, 10),
|
||||
OUTBOX_POLL_INTERVAL_MS: numeric(
|
||||
'OUTBOX_POLL_INTERVAL_MS',
|
||||
50,
|
||||
undefined,
|
||||
500,
|
||||
),
|
||||
LEASE_REAPER_INTERVAL_MS: numeric(
|
||||
'LEASE_REAPER_INTERVAL_MS',
|
||||
100,
|
||||
undefined,
|
||||
2000,
|
||||
),
|
||||
// 0 отключает фоновую уборку staging (ручной запуск остаётся).
|
||||
STAGING_CLEANUP_INTERVAL_MS: numeric(
|
||||
'STAGING_CLEANUP_INTERVAL_MS',
|
||||
0,
|
||||
undefined,
|
||||
60000,
|
||||
),
|
||||
STAGING_RETENTION_SECONDS: numeric(
|
||||
'STAGING_RETENTION_SECONDS',
|
||||
60,
|
||||
undefined,
|
||||
86400,
|
||||
),
|
||||
WORKER_HEALTH_PORT: numeric('WORKER_HEALTH_PORT', 1, 65535, 8081),
|
||||
FAULT_INJECTION_ENABLED: numeric('FAULT_INJECTION_ENABLED', 0, 1, 0),
|
||||
QUEUED_RECONCILE_GRACE_SECONDS: numeric(
|
||||
'QUEUED_RECONCILE_GRACE_SECONDS',
|
||||
5,
|
||||
3600,
|
||||
15,
|
||||
),
|
||||
OUTBOX_RETENTION_SECONDS: numeric(
|
||||
'OUTBOX_RETENTION_SECONDS',
|
||||
60,
|
||||
undefined,
|
||||
86400,
|
||||
),
|
||||
JOB_EVENTS_RETENTION_SECONDS: numeric(
|
||||
'JOB_EVENTS_RETENTION_SECONDS',
|
||||
3600,
|
||||
undefined,
|
||||
2592000,
|
||||
),
|
||||
})
|
||||
.strip();
|
||||
|
||||
export const databaseConfigSchema = appConfigSchema.pick({
|
||||
DATABASE_URL: true,
|
||||
DATABASE_POOL_MAX: true,
|
||||
DATABASE_CONNECTION_TIMEOUT_MS: true,
|
||||
DATABASE_STATEMENT_TIMEOUT_MS: true,
|
||||
DATABASE_IDLE_TX_TIMEOUT_MS: true,
|
||||
HEALTH_TIMEOUT_MS: true,
|
||||
IDEMPOTENCY_TTL_SECONDS: true,
|
||||
});
|
||||
export type DatabaseConfig = z.infer<typeof databaseConfigSchema>;
|
||||
|
||||
export type AppConfig = z.infer<typeof appConfigSchema>;
|
||||
export type AppConfigKey = keyof AppConfig;
|
||||
|
||||
function resolveSecret(
|
||||
raw: Record<string, unknown>,
|
||||
key:
|
||||
| 'DATABASE_URL'
|
||||
| 'REDIS_URL'
|
||||
| 'BROKER_URL'
|
||||
| 'S3_ACCESS_KEY_ID'
|
||||
| 'S3_SECRET_ACCESS_KEY',
|
||||
): unknown {
|
||||
const fileKey = `${key}_FILE`;
|
||||
const path = raw[fileKey];
|
||||
if (typeof path !== 'string' || path.length === 0) return raw[key];
|
||||
try {
|
||||
const value = readFileSync(path, 'utf8').trim();
|
||||
if (value.length === 0)
|
||||
throw new Error(`${fileKey} contains an empty value`);
|
||||
return value;
|
||||
} catch (error) {
|
||||
if (
|
||||
error instanceof Error &&
|
||||
error.message.includes('contains an empty value')
|
||||
)
|
||||
throw error;
|
||||
throw new Error(`${fileKey} could not be read`);
|
||||
}
|
||||
}
|
||||
|
||||
export function parseAppConfig(raw: Record<string, unknown>): AppConfig {
|
||||
return appConfigSchema.parse({
|
||||
...raw,
|
||||
DATABASE_URL: resolveSecret(raw, 'DATABASE_URL'),
|
||||
REDIS_URL: resolveSecret(raw, 'REDIS_URL'),
|
||||
BROKER_URL: resolveSecret(raw, 'BROKER_URL'),
|
||||
S3_ACCESS_KEY_ID: resolveSecret(raw, 'S3_ACCESS_KEY_ID'),
|
||||
S3_SECRET_ACCESS_KEY: resolveSecret(raw, 'S3_SECRET_ACCESS_KEY'),
|
||||
});
|
||||
}
|
||||
|
||||
// Для процессов, которым нужна только БД (миграции, GC идемпотентности).
|
||||
export function parseDatabaseConfig(
|
||||
raw: Record<string, unknown>,
|
||||
): DatabaseConfig {
|
||||
return databaseConfigSchema.parse({
|
||||
...raw,
|
||||
DATABASE_URL: resolveSecret(raw, 'DATABASE_URL'),
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,14 @@
|
||||
import { randomUUID } from 'node:crypto';
|
||||
import { hostname } from 'node:os';
|
||||
import type { ConfigService } from '@nestjs/config';
|
||||
import type { AppConfig } from './app-config.schema.js';
|
||||
|
||||
// A replica must be identifiable from the outside. Inside Docker the hostname is
|
||||
// the short container ID, so X-Instance-Id can be mapped back to exactly one
|
||||
// container; INSTANCE_ID exists for platforms where that mapping is not useful.
|
||||
export function resolveInstanceId(config: ConfigService<AppConfig>): string {
|
||||
const configured = config.get('INSTANCE_ID');
|
||||
if (configured) return configured;
|
||||
const host = hostname();
|
||||
return host.length > 0 ? host : randomUUID();
|
||||
}
|
||||
@@ -0,0 +1,105 @@
|
||||
import { Injectable, Logger, OnModuleDestroy } from '@nestjs/common';
|
||||
import { ConfigService } from '@nestjs/config';
|
||||
import pg from 'pg';
|
||||
import { MetricsService } from '../metrics/metrics.service.js';
|
||||
import type { AppConfig } from '../config/app-config.schema.js';
|
||||
|
||||
@Injectable()
|
||||
export class DatabaseClient implements OnModuleDestroy {
|
||||
readonly pool: pg.Pool;
|
||||
private readonly healthPool: pg.Pool;
|
||||
private readonly logger = new Logger(DatabaseClient.name);
|
||||
|
||||
constructor(
|
||||
private readonly config: ConfigService<AppConfig>,
|
||||
private readonly metrics: MetricsService,
|
||||
) {
|
||||
this.pool = new pg.Pool({
|
||||
connectionString: this.config.getOrThrow('DATABASE_URL'),
|
||||
max: this.config.getOrThrow('DATABASE_POOL_MAX'),
|
||||
connectionTimeoutMillis: this.config.getOrThrow(
|
||||
'DATABASE_CONNECTION_TIMEOUT_MS',
|
||||
),
|
||||
statement_timeout: this.config.getOrThrow(
|
||||
'DATABASE_STATEMENT_TIMEOUT_MS',
|
||||
),
|
||||
// A relay that waits on the broker must not leave the transaction
|
||||
// idle forever if confirm handling stalls.
|
||||
options: `-c idle_in_transaction_session_timeout=${this.config.getOrThrow('DATABASE_IDLE_TX_TIMEOUT_MS')}`,
|
||||
});
|
||||
// Idle clients emit 'error' on the pool when the database drops; without a
|
||||
// listener Node treats it as uncaught and terminates the process.
|
||||
this.pool.on('error', (error: Error) => {
|
||||
this.metrics.increment('database.pool_error');
|
||||
this.logger.warn(`PostgreSQL pool error: ${error.message}`);
|
||||
});
|
||||
// Проба готовности не должна стоять в очереди за бизнес-запросами: занятый
|
||||
// общий пул — это перегрузка, а не потеря зависимости.
|
||||
this.healthPool = new pg.Pool({
|
||||
connectionString: this.config.getOrThrow('DATABASE_URL'),
|
||||
max: 1,
|
||||
idleTimeoutMillis: 0,
|
||||
connectionTimeoutMillis:
|
||||
this.config.getOrThrow('HEALTH_TIMEOUT_MS'),
|
||||
query_timeout: this.config.getOrThrow('HEALTH_TIMEOUT_MS'),
|
||||
});
|
||||
this.healthPool.on('error', (error: Error) => {
|
||||
this.metrics.increment('database.health_pool_error');
|
||||
this.logger.warn(
|
||||
`PostgreSQL health connection error: ${error.message}`,
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
query<T extends pg.QueryResultRow = pg.QueryResultRow>(
|
||||
label: string,
|
||||
text: string,
|
||||
values: readonly unknown[] = [],
|
||||
): Promise<pg.QueryResult<T>> {
|
||||
this.metrics.increment(label);
|
||||
return this.pool.query<T>({ text, values: [...values] });
|
||||
}
|
||||
|
||||
async health(): Promise<void> {
|
||||
this.metrics.increment('health.database');
|
||||
await this.healthPool.query('SELECT 1');
|
||||
}
|
||||
|
||||
async withTransaction<T>(
|
||||
work: (client: pg.PoolClient) => Promise<T>,
|
||||
): Promise<T> {
|
||||
const client = await this.pool.connect();
|
||||
// A server-side idle_in_transaction_session_timeout or a network drop
|
||||
// can emit 'error' on a checked-out client after pg-pool removed its
|
||||
// idle listener. Without this listener Node treats it as uncaught.
|
||||
let fatal: Error | undefined;
|
||||
const onError = (error: Error): void => {
|
||||
fatal = error;
|
||||
};
|
||||
client.on('error', onError);
|
||||
try {
|
||||
await client.query('BEGIN');
|
||||
const result = await work(client);
|
||||
await client.query('COMMIT');
|
||||
return result;
|
||||
} catch (error) {
|
||||
try {
|
||||
await client.query('ROLLBACK');
|
||||
} catch (rollbackError) {
|
||||
this.metrics.increment('database.rollback_error');
|
||||
this.logger.warn(
|
||||
`Rollback failed: ${rollbackError instanceof Error ? rollbackError.message : 'unknown error'}`,
|
||||
);
|
||||
}
|
||||
// Callers must see the operation error, not the rollback error.
|
||||
throw error;
|
||||
} finally {
|
||||
client.removeListener('error', onError);
|
||||
client.release(fatal);
|
||||
}
|
||||
}
|
||||
|
||||
async onModuleDestroy(): Promise<void> {
|
||||
await Promise.all([this.pool.end(), this.healthPool.end()]);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,10 @@
|
||||
import { Module } from '@nestjs/common';
|
||||
import { MetricsModule } from '../metrics/metrics.module.js';
|
||||
import { DatabaseClient } from './database.client.js';
|
||||
|
||||
@Module({
|
||||
imports: [MetricsModule],
|
||||
providers: [DatabaseClient],
|
||||
exports: [DatabaseClient],
|
||||
})
|
||||
export class DatabaseModule {}
|
||||
@@ -0,0 +1,63 @@
|
||||
import { readdir, readFile } from 'node:fs/promises';
|
||||
import { dirname, join } from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import { DatabaseClient } from './database.client.js';
|
||||
import { MetricsService } from '../metrics/metrics.service.js';
|
||||
import { parseDatabaseConfig } from '../config/app-config.schema.js';
|
||||
|
||||
const migrationsDirectory = join(
|
||||
dirname(fileURLToPath(import.meta.url)),
|
||||
'migrations',
|
||||
);
|
||||
|
||||
async function migrationFiles(): Promise<string[]> {
|
||||
const files = await readdir(migrationsDirectory);
|
||||
return files.filter((file) => /^\d+_.+\.sql$/u.test(file)).sort();
|
||||
}
|
||||
|
||||
export async function runMigrations(database: DatabaseClient): Promise<void> {
|
||||
await database.query(
|
||||
'migrations.ensure_table',
|
||||
`CREATE TABLE IF NOT EXISTS schema_migrations (
|
||||
version text PRIMARY KEY,
|
||||
applied_at timestamptz NOT NULL DEFAULT now()
|
||||
)`,
|
||||
);
|
||||
|
||||
for (const version of await migrationFiles()) {
|
||||
await database.withTransaction(async (client) => {
|
||||
const applied = await client.query({
|
||||
text: 'SELECT version FROM schema_migrations WHERE version = $1',
|
||||
values: [version],
|
||||
});
|
||||
if (applied.rowCount) return;
|
||||
|
||||
const sql = await readFile(
|
||||
join(migrationsDirectory, version),
|
||||
'utf8',
|
||||
);
|
||||
await client.query(sql);
|
||||
await client.query({
|
||||
text: 'INSERT INTO schema_migrations (version) VALUES ($1)',
|
||||
values: [version],
|
||||
});
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
async function main(): Promise<void> {
|
||||
const config = parseDatabaseConfig(process.env as Record<string, unknown>);
|
||||
const database = new DatabaseClient(
|
||||
{
|
||||
getOrThrow: (key: keyof typeof config) => config[key],
|
||||
} as never,
|
||||
new MetricsService(),
|
||||
);
|
||||
try {
|
||||
await runMigrations(database);
|
||||
} finally {
|
||||
await database.onModuleDestroy();
|
||||
}
|
||||
}
|
||||
|
||||
if (process.argv[1] === fileURLToPath(import.meta.url)) await main();
|
||||
@@ -0,0 +1,41 @@
|
||||
CREATE TABLE IF NOT EXISTS schema_migrations (
|
||||
version text PRIMARY KEY,
|
||||
applied_at timestamptz NOT NULL DEFAULT now()
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS users (
|
||||
id uuid PRIMARY KEY,
|
||||
username text NOT NULL UNIQUE,
|
||||
display_name text NOT NULL,
|
||||
password_hash text NOT NULL,
|
||||
created_at timestamptz NOT NULL DEFAULT now()
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS chats (
|
||||
id uuid PRIMARY KEY,
|
||||
title text NOT NULL,
|
||||
created_at timestamptz NOT NULL DEFAULT now(),
|
||||
message_version bigint NOT NULL DEFAULT 0
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS chat_members (
|
||||
chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE,
|
||||
user_id uuid NOT NULL REFERENCES users(id) ON DELETE CASCADE,
|
||||
PRIMARY KEY (chat_id, user_id)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS messages (
|
||||
id uuid PRIMARY KEY,
|
||||
chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE,
|
||||
sender_id uuid NOT NULL REFERENCES users(id) ON DELETE RESTRICT,
|
||||
text text NOT NULL,
|
||||
created_at timestamptz NOT NULL DEFAULT now(),
|
||||
CONSTRAINT messages_sender_member_fk
|
||||
FOREIGN KEY (chat_id, sender_id) REFERENCES chat_members(chat_id, user_id)
|
||||
ON DELETE RESTRICT
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS messages_chat_created_id_idx
|
||||
ON messages (chat_id, created_at, id);
|
||||
CREATE INDEX IF NOT EXISTS chat_members_user_chat_idx
|
||||
ON chat_members (user_id, chat_id);
|
||||
@@ -0,0 +1,5 @@
|
||||
-- HTTP timestamps and keyset cursors use JavaScript millisecond precision.
|
||||
-- Store the same precision so a returned cursor cannot point before its own row.
|
||||
ALTER TABLE users ALTER COLUMN created_at TYPE timestamptz(3);
|
||||
ALTER TABLE chats ALTER COLUMN created_at TYPE timestamptz(3);
|
||||
ALTER TABLE messages ALTER COLUMN created_at TYPE timestamptz(3);
|
||||
@@ -0,0 +1,20 @@
|
||||
-- Shared idempotency ledger for POST /api/v1/chats/{id}/messages.
|
||||
-- The key scope is (user, method, path, key): the same key from another user or
|
||||
-- another chat is independent, as the contract requires.
|
||||
-- claim + response are committed in one transaction, so a committed row always
|
||||
-- carries status_code and response_body (there is no "poisoned" intermediate state).
|
||||
CREATE TABLE IF NOT EXISTS idempotency_keys (
|
||||
user_id uuid NOT NULL REFERENCES users(id) ON DELETE CASCADE,
|
||||
method text NOT NULL,
|
||||
path text NOT NULL,
|
||||
key text NOT NULL,
|
||||
request_fingerprint text NOT NULL,
|
||||
status_code integer,
|
||||
response_body jsonb,
|
||||
message_id uuid,
|
||||
created_at timestamptz NOT NULL DEFAULT now(),
|
||||
PRIMARY KEY (user_id, method, path, key)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idempotency_keys_created_at_idx
|
||||
ON idempotency_keys (created_at);
|
||||
@@ -0,0 +1,9 @@
|
||||
-- Instrumentation for the scaling experiments. pg_stat_statements only works
|
||||
-- when the server is started with shared_preload_libraries=pg_stat_statements
|
||||
-- (compose.yaml does that); without the preload the CREATE fails with a plain
|
||||
-- notice, so a database started without the preload still migrates cleanly.
|
||||
DO $$ BEGIN
|
||||
CREATE EXTENSION IF NOT EXISTS pg_stat_statements;
|
||||
EXCEPTION WHEN OTHERS THEN
|
||||
RAISE NOTICE 'pg_stat_statements unavailable: %', SQLERRM;
|
||||
END $$;
|
||||
@@ -0,0 +1,42 @@
|
||||
ALTER TABLE messages ADD CONSTRAINT messages_id_chat_key UNIQUE (id, chat_id);
|
||||
|
||||
CREATE TABLE attachments (
|
||||
id uuid PRIMARY KEY,
|
||||
chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE,
|
||||
owner_id uuid NOT NULL REFERENCES users(id) ON DELETE RESTRICT,
|
||||
filename text NOT NULL CHECK (char_length(filename) BETWEEN 1 AND 255),
|
||||
content_type text NOT NULL CHECK (content_type IN ('image/png', 'image/jpeg')),
|
||||
upload_mode text NOT NULL CHECK (upload_mode IN ('proxy', 'direct')),
|
||||
status text NOT NULL CHECK (status IN ('pending', 'ready', 'deleting')),
|
||||
object_key text NOT NULL UNIQUE,
|
||||
size_bytes bigint CHECK (size_bytes BETWEEN 1 AND 10485760),
|
||||
sha256 text CHECK (sha256 ~ '^[a-f0-9]{64}$'),
|
||||
width integer CHECK (width > 0),
|
||||
height integer CHECK (height > 0),
|
||||
expires_at timestamptz(3),
|
||||
created_at timestamptz(3) NOT NULL DEFAULT now(),
|
||||
updated_at timestamptz(3) NOT NULL DEFAULT now(),
|
||||
CONSTRAINT attachments_id_chat_key UNIQUE (id, chat_id),
|
||||
CONSTRAINT attachments_owner_member_fk FOREIGN KEY (chat_id, owner_id)
|
||||
REFERENCES chat_members (chat_id, user_id),
|
||||
CONSTRAINT attachments_ready_complete CHECK (status <> 'ready' OR
|
||||
(size_bytes IS NOT NULL AND sha256 IS NOT NULL AND width IS NOT NULL AND height IS NOT NULL)),
|
||||
CONSTRAINT attachments_unfinished_expiry CHECK (status = 'ready' OR expires_at IS NOT NULL)
|
||||
);
|
||||
CREATE INDEX attachments_unfinished_idx ON attachments (expires_at) WHERE status <> 'ready';
|
||||
CREATE INDEX attachments_owner_direct_pending_idx ON attachments (owner_id)
|
||||
WHERE status = 'pending' AND upload_mode = 'direct';
|
||||
|
||||
CREATE TABLE message_attachments (
|
||||
message_id uuid NOT NULL,
|
||||
chat_id uuid NOT NULL,
|
||||
attachment_id uuid NOT NULL,
|
||||
ordinal smallint NOT NULL CHECK (ordinal BETWEEN 1 AND 10),
|
||||
PRIMARY KEY (message_id, attachment_id),
|
||||
CONSTRAINT message_attachments_ordinal_key UNIQUE (message_id, ordinal),
|
||||
CONSTRAINT message_attachments_message_fk FOREIGN KEY (message_id, chat_id)
|
||||
REFERENCES messages (id, chat_id) ON DELETE CASCADE,
|
||||
CONSTRAINT message_attachments_attachment_fk FOREIGN KEY (attachment_id, chat_id)
|
||||
REFERENCES attachments (id, chat_id)
|
||||
);
|
||||
CREATE INDEX message_attachments_attachment_idx ON message_attachments (attachment_id);
|
||||
@@ -0,0 +1,8 @@
|
||||
-- Rejected direct uploads keep their row until the signed URL can no longer be
|
||||
-- used, so an object re-PUT after rejection is still found by the sweeper.
|
||||
ALTER TABLE attachments DROP CONSTRAINT attachments_status_check;
|
||||
ALTER TABLE attachments ADD CONSTRAINT attachments_status_check
|
||||
CHECK (status IN ('pending', 'ready', 'deleting', 'rejected'));
|
||||
DROP INDEX attachments_owner_direct_pending_idx;
|
||||
CREATE INDEX attachments_owner_direct_pending_idx ON attachments (owner_id)
|
||||
WHERE status IN ('pending', 'rejected') AND upload_mode = 'direct';
|
||||
@@ -0,0 +1,81 @@
|
||||
ALTER TABLE attachments DROP CONSTRAINT attachments_status_check;
|
||||
ALTER TABLE attachments ADD CONSTRAINT attachments_status_check CHECK (status IN
|
||||
('pending','queued','processing','ready','failed','deleting','rejected'));
|
||||
ALTER TABLE attachments DROP CONSTRAINT attachments_ready_complete;
|
||||
ALTER TABLE attachments ADD CONSTRAINT attachments_accepted_complete CHECK (
|
||||
status NOT IN ('queued','processing','ready','failed') OR
|
||||
(size_bytes IS NOT NULL AND sha256 IS NOT NULL AND width IS NOT NULL AND height IS NOT NULL));
|
||||
ALTER TABLE attachments DROP CONSTRAINT attachments_unfinished_expiry;
|
||||
ALTER TABLE attachments ADD CONSTRAINT attachments_unfinished_expiry CHECK (
|
||||
status NOT IN ('pending','rejected','deleting') OR expires_at IS NOT NULL);
|
||||
ALTER TABLE attachments
|
||||
ADD COLUMN staging_key text UNIQUE,
|
||||
ADD COLUMN error_code text CHECK (error_code ~ '^[a-z_]{1,64}$'),
|
||||
ADD COLUMN ready_pipeline_version integer CHECK (ready_pipeline_version >= 1);
|
||||
ALTER TABLE attachments ADD CONSTRAINT attachments_failed_code
|
||||
CHECK (status <> 'failed' OR error_code IS NOT NULL);
|
||||
DROP INDEX attachments_unfinished_idx;
|
||||
CREATE INDEX attachments_unfinished_idx ON attachments (expires_at)
|
||||
WHERE status IN ('pending','rejected','deleting');
|
||||
CREATE INDEX attachments_staging_idx ON attachments (updated_at)
|
||||
WHERE staging_key IS NOT NULL AND status IN ('ready','failed');
|
||||
|
||||
-- Rows accepted by lab4 already have their final original and complete metadata.
|
||||
UPDATE attachments SET ready_pipeline_version = 1
|
||||
WHERE status = 'ready' AND ready_pipeline_version IS NULL;
|
||||
|
||||
CREATE TABLE attachment_jobs (
|
||||
job_id uuid PRIMARY KEY,
|
||||
attachment_id uuid NOT NULL REFERENCES attachments(id) ON DELETE CASCADE,
|
||||
pipeline_version integer NOT NULL CHECK (pipeline_version >= 1),
|
||||
status text NOT NULL CHECK (status IN ('queued','processing','succeeded','failed','superseded')),
|
||||
attempt integer NOT NULL DEFAULT 1 CHECK (attempt >= 1),
|
||||
max_attempts integer NOT NULL CHECK (max_attempts >= 1),
|
||||
next_attempt_at timestamptz(3) NOT NULL DEFAULT now(),
|
||||
lease_owner text, lease_token uuid, lease_until timestamptz(3),
|
||||
last_error_code text,
|
||||
created_at timestamptz(3) NOT NULL DEFAULT now(),
|
||||
updated_at timestamptz(3) NOT NULL DEFAULT now(),
|
||||
finished_at timestamptz(3),
|
||||
CONSTRAINT attachment_jobs_effect_key UNIQUE (attachment_id, pipeline_version),
|
||||
CONSTRAINT attachment_jobs_lease CHECK ((status = 'processing') =
|
||||
(lease_owner IS NOT NULL AND lease_token IS NOT NULL AND lease_until IS NOT NULL))
|
||||
);
|
||||
CREATE INDEX attachment_jobs_lease_idx ON attachment_jobs (lease_until) WHERE status = 'processing';
|
||||
CREATE INDEX attachment_jobs_failed_idx ON attachment_jobs (updated_at) WHERE status = 'failed';
|
||||
|
||||
CREATE TABLE attachment_variants (
|
||||
attachment_id uuid NOT NULL REFERENCES attachments(id) ON DELETE CASCADE,
|
||||
pipeline_version integer NOT NULL CHECK (pipeline_version >= 1),
|
||||
variant text NOT NULL CHECK (variant IN ('thumbnail','crop','optimized')),
|
||||
object_key text NOT NULL UNIQUE,
|
||||
content_type text NOT NULL CHECK (content_type IN ('image/jpeg','image/png','image/webp')),
|
||||
size_bytes bigint NOT NULL CHECK (size_bytes >= 1),
|
||||
width integer NOT NULL CHECK (width >= 1),
|
||||
height integer NOT NULL CHECK (height >= 1),
|
||||
sha256 text NOT NULL CHECK (sha256 ~ '^[a-f0-9]{64}$'),
|
||||
job_id uuid NOT NULL REFERENCES attachment_jobs(job_id) ON DELETE CASCADE,
|
||||
created_at timestamptz(3) NOT NULL DEFAULT now(),
|
||||
PRIMARY KEY (attachment_id, pipeline_version, variant)
|
||||
);
|
||||
|
||||
CREATE TABLE job_outbox (
|
||||
id bigserial PRIMARY KEY,
|
||||
job_id uuid NOT NULL REFERENCES attachment_jobs(job_id) ON DELETE CASCADE,
|
||||
payload jsonb NOT NULL,
|
||||
available_at timestamptz(3) NOT NULL DEFAULT now(),
|
||||
published_at timestamptz(3),
|
||||
created_at timestamptz(3) NOT NULL DEFAULT now()
|
||||
);
|
||||
CREATE INDEX job_outbox_pending_idx ON job_outbox (available_at, id) WHERE published_at IS NULL;
|
||||
|
||||
CREATE TABLE job_events (
|
||||
id bigserial PRIMARY KEY,
|
||||
job_id uuid NOT NULL REFERENCES attachment_jobs(job_id) ON DELETE CASCADE,
|
||||
event text NOT NULL CHECK (event IN ('created','claimed','retry_scheduled','lease_expired',
|
||||
'succeeded','failed','superseded','replayed','duplicate_skipped','staging_deleted')),
|
||||
attempt integer NOT NULL,
|
||||
detail jsonb NOT NULL DEFAULT '{}'::jsonb,
|
||||
created_at timestamptz(3) NOT NULL DEFAULT now()
|
||||
);
|
||||
CREATE INDEX job_events_job_idx ON job_events (job_id, id);
|
||||
@@ -0,0 +1,37 @@
|
||||
ALTER TABLE attachment_jobs ADD COLUMN last_reconciled_at timestamptz(3);
|
||||
CREATE INDEX job_outbox_job_idx ON job_outbox (job_id);
|
||||
|
||||
-- lab4 rows are ready but predate variants. Requeue them once so the worker
|
||||
-- materializes the same v1 variants every other ready attachment has. The
|
||||
-- INSERT ... RETURNING CTE keeps the job, outbox and audit event atomic.
|
||||
WITH legacy AS (
|
||||
INSERT INTO attachment_jobs
|
||||
(job_id, attachment_id, pipeline_version, status, attempt, max_attempts,
|
||||
next_attempt_at, created_at)
|
||||
SELECT gen_random_uuid(), a.id, 1, 'queued', 1, 5, now(), now()
|
||||
FROM attachments a
|
||||
WHERE a.status = 'ready'
|
||||
AND NOT EXISTS (
|
||||
SELECT 1 FROM attachment_variants v WHERE v.attachment_id = a.id)
|
||||
AND NOT EXISTS (
|
||||
SELECT 1 FROM attachment_jobs j
|
||||
WHERE j.attachment_id = a.id AND j.pipeline_version = 1)
|
||||
RETURNING job_id, attachment_id, pipeline_version, attempt, created_at
|
||||
), queued AS (
|
||||
INSERT INTO job_outbox (job_id, payload, available_at)
|
||||
SELECT job_id,
|
||||
jsonb_build_object(
|
||||
'schema_version', 1,
|
||||
'job_id', job_id,
|
||||
'type', 'attachment.process.v1',
|
||||
'attachment_id', attachment_id,
|
||||
'pipeline_version', pipeline_version,
|
||||
'attempt', attempt,
|
||||
'created_at', to_char(created_at AT TIME ZONE 'UTC',
|
||||
'YYYY-MM-DD"T"HH24:MI:SS.MS"Z"')),
|
||||
now()
|
||||
FROM legacy
|
||||
RETURNING job_id
|
||||
)
|
||||
INSERT INTO job_events (job_id, event, attempt, detail)
|
||||
SELECT job_id, 'created', 1, '{"backfill":true}'::jsonb FROM legacy;
|
||||
@@ -0,0 +1,10 @@
|
||||
CREATE INDEX attachment_jobs_queued_idx
|
||||
ON attachment_jobs (next_attempt_at) WHERE status = 'queued';
|
||||
CREATE INDEX job_events_created_idx ON job_events (created_at);
|
||||
CREATE INDEX job_outbox_published_idx
|
||||
ON job_outbox (published_at) WHERE published_at IS NOT NULL;
|
||||
|
||||
ALTER TABLE job_events DROP CONSTRAINT job_events_event_check;
|
||||
ALTER TABLE job_events ADD CONSTRAINT job_events_event_check CHECK (event IN
|
||||
('created','claimed','retry_scheduled','lease_expired','succeeded','failed',
|
||||
'superseded','replayed','duplicate_skipped','staging_deleted','reconciled'));
|
||||
@@ -0,0 +1,17 @@
|
||||
import { Controller, Get } from '@nestjs/common';
|
||||
import { HealthService } from './health.service.js';
|
||||
|
||||
@Controller('health')
|
||||
export class HealthController {
|
||||
constructor(private readonly healthService: HealthService) {}
|
||||
|
||||
@Get('live')
|
||||
getLive() {
|
||||
return this.healthService.getLiveness();
|
||||
}
|
||||
|
||||
@Get('ready')
|
||||
async getReady() {
|
||||
return this.healthService.getReadiness();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,12 @@
|
||||
import { Module } from '@nestjs/common';
|
||||
import { HealthController } from './health.controller.js';
|
||||
import { HealthService } from './health.service.js';
|
||||
import { DatabaseModule } from '../database/database.module.js';
|
||||
import { RedisModule } from '../redis/redis.module.js';
|
||||
|
||||
@Module({
|
||||
imports: [DatabaseModule, RedisModule],
|
||||
controllers: [HealthController],
|
||||
providers: [HealthService],
|
||||
})
|
||||
export class HealthModule {}
|
||||
@@ -0,0 +1,91 @@
|
||||
import { describe, expect, it, vi } from 'vitest';
|
||||
import { ServiceUnavailableException } from '@nestjs/common';
|
||||
import { HealthService } from './health.service.js';
|
||||
|
||||
describe('HealthService', () => {
|
||||
it('reports liveness and readiness when dependencies are healthy', async () => {
|
||||
const database = { health: vi.fn().mockResolvedValue(undefined) };
|
||||
const redis = { health: vi.fn().mockResolvedValue(undefined) };
|
||||
const service = new HealthService(
|
||||
database as never,
|
||||
redis as never,
|
||||
{ getOrThrow: () => 50 } as never,
|
||||
);
|
||||
expect(service.getLiveness()).toEqual({ status: 'ok' });
|
||||
await expect(service.getReadiness()).resolves.toEqual({ status: 'ok' });
|
||||
});
|
||||
|
||||
it('rejects readiness when a dependency is unavailable', async () => {
|
||||
const database = {
|
||||
health: vi.fn().mockRejectedValue(new Error('down')),
|
||||
};
|
||||
const redis = { health: vi.fn().mockResolvedValue(undefined) };
|
||||
const service = new HealthService(
|
||||
database as never,
|
||||
redis as never,
|
||||
{ getOrThrow: () => 50 } as never,
|
||||
);
|
||||
await expect(service.getReadiness()).rejects.toBeInstanceOf(
|
||||
ServiceUnavailableException,
|
||||
);
|
||||
});
|
||||
|
||||
it('rejects readiness when Redis alone is unavailable', async () => {
|
||||
const database = { health: vi.fn().mockResolvedValue(undefined) };
|
||||
const redis = {
|
||||
health: vi.fn().mockRejectedValue(new Error('down')),
|
||||
};
|
||||
const service = new HealthService(
|
||||
database as never,
|
||||
redis as never,
|
||||
{ getOrThrow: () => 50 } as never,
|
||||
);
|
||||
await expect(service.getReadiness()).rejects.toBeInstanceOf(
|
||||
ServiceUnavailableException,
|
||||
);
|
||||
});
|
||||
|
||||
it('keeps liveness healthy while draining', async () => {
|
||||
const service = new HealthService(
|
||||
{ health: vi.fn() } as never,
|
||||
{ health: vi.fn() } as never,
|
||||
{ getOrThrow: () => 50 } as never,
|
||||
);
|
||||
service.setDraining();
|
||||
expect(service.getLiveness()).toEqual({ status: 'ok' });
|
||||
await expect(service.getReadiness()).rejects.toThrow(
|
||||
'Service is shutting down',
|
||||
);
|
||||
});
|
||||
|
||||
it('answers not-ready during drain without probing dependencies', async () => {
|
||||
const database = { health: vi.fn().mockResolvedValue(undefined) };
|
||||
const redis = { health: vi.fn().mockResolvedValue(undefined) };
|
||||
const service = new HealthService(
|
||||
database as never,
|
||||
redis as never,
|
||||
{ getOrThrow: () => 50 } as never,
|
||||
);
|
||||
service.setDraining();
|
||||
await expect(service.getReadiness()).rejects.toThrow(
|
||||
'Service is shutting down',
|
||||
);
|
||||
// Дрейн обязан отвечать сразу: проба LB не должна ждать зависимости,
|
||||
// когда реплика уже выходит из ротации.
|
||||
expect(database.health).not.toHaveBeenCalled();
|
||||
expect(redis.health).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('fails readiness within the budget when a dependency hangs', async () => {
|
||||
const database = { health: vi.fn(() => new Promise<void>(() => {})) };
|
||||
const redis = { health: vi.fn().mockResolvedValue(undefined) };
|
||||
const service = new HealthService(
|
||||
database as never,
|
||||
redis as never,
|
||||
{ getOrThrow: () => 20 } as never,
|
||||
);
|
||||
await expect(service.getReadiness()).rejects.toBeInstanceOf(
|
||||
ServiceUnavailableException,
|
||||
);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,60 @@
|
||||
import {
|
||||
Global,
|
||||
Injectable,
|
||||
Module,
|
||||
ServiceUnavailableException,
|
||||
} from '@nestjs/common';
|
||||
import { DatabaseClient } from '../database/database.client.js';
|
||||
import { RedisClient } from '../redis/redis.client.js';
|
||||
import { ConfigService } from '@nestjs/config';
|
||||
import type { AppConfig } from '../config/app-config.schema.js';
|
||||
|
||||
@Injectable()
|
||||
export class HealthService {
|
||||
private ready = true;
|
||||
constructor(
|
||||
private readonly database: DatabaseClient,
|
||||
private readonly redis: RedisClient,
|
||||
private readonly config: ConfigService<AppConfig>,
|
||||
) {}
|
||||
getLiveness() {
|
||||
return { status: 'ok' as const };
|
||||
}
|
||||
async getReadiness() {
|
||||
if (!this.ready)
|
||||
throw new ServiceUnavailableException('Service is shutting down');
|
||||
const dependencies = Promise.all([
|
||||
this.database.health(),
|
||||
this.redis.health(),
|
||||
]);
|
||||
// Поздний отказ после дедлайна не должен стать unhandled rejection.
|
||||
dependencies.catch(() => undefined);
|
||||
let timer: NodeJS.Timeout | undefined;
|
||||
const deadline = new Promise<never>((_, reject) => {
|
||||
timer = setTimeout(
|
||||
() => reject(new Error('readiness budget exceeded')),
|
||||
this.config.getOrThrow('HEALTH_TIMEOUT_MS'),
|
||||
);
|
||||
});
|
||||
try {
|
||||
await Promise.race([dependencies, deadline]);
|
||||
} catch {
|
||||
throw new ServiceUnavailableException('Dependencies unavailable');
|
||||
} finally {
|
||||
clearTimeout(timer);
|
||||
}
|
||||
return { status: 'ok' as const };
|
||||
}
|
||||
// Called on SIGTERM before the drain delay: the replica turns not-ready while
|
||||
// it finishes in-flight requests, so the load balancer stops probing it
|
||||
// successfully and removes it from rotation. The delay itself lives in
|
||||
// main.ts, because Nest destroys providers (closing the pools) before this
|
||||
// class could serve anyone.
|
||||
setDraining() {
|
||||
this.ready = false;
|
||||
}
|
||||
}
|
||||
|
||||
@Global()
|
||||
@Module({ providers: [HealthService], exports: [HealthService] })
|
||||
export class HealthServiceModule {}
|
||||
@@ -0,0 +1,19 @@
|
||||
import { describe, expect, it } from 'vitest';
|
||||
import { retryDelayMs } from './backoff.js';
|
||||
|
||||
describe('retryDelayMs', () => {
|
||||
it('uses equal jitter', () => {
|
||||
expect(retryDelayMs(1, 1000, 30000, () => 0)).toBe(500);
|
||||
expect(retryDelayMs(1, 1000, 30000, () => 1)).toBe(1000);
|
||||
expect(retryDelayMs(10, 1000, 30000, () => 1)).toBe(30000);
|
||||
});
|
||||
|
||||
it('never exceeds the cap and grows monotonically at random 0.5', () => {
|
||||
const values = Array.from({ length: 6 }, (_, index) =>
|
||||
retryDelayMs(index + 1, 1000, 30000, () => 0.5),
|
||||
);
|
||||
for (const value of values) expect(value).toBeLessThanOrEqual(30000);
|
||||
for (let index = 1; index < values.length; index += 1)
|
||||
expect(values[index]).toBeGreaterThanOrEqual(values[index - 1]);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,14 @@
|
||||
/**
|
||||
* Equal jitter: half of the exponential delay is fixed, half is random. The
|
||||
* result never exceeds `capMs`, so `max_attempts` bounds both attempts and
|
||||
* total wait time.
|
||||
*/
|
||||
export function retryDelayMs(
|
||||
attempt: number,
|
||||
baseMs: number,
|
||||
capMs: number,
|
||||
random: () => number = Math.random,
|
||||
): number {
|
||||
const exponential = Math.min(capMs, baseMs * 2 ** (attempt - 1));
|
||||
return Math.floor(exponential * (0.5 + 0.5 * random()));
|
||||
}
|
||||
@@ -0,0 +1,419 @@
|
||||
import {
|
||||
connect as amqpConnect,
|
||||
type Channel,
|
||||
type ChannelModel,
|
||||
type ConfirmChannel,
|
||||
type ConsumeMessage,
|
||||
} from 'amqplib';
|
||||
import {
|
||||
Injectable,
|
||||
Logger,
|
||||
OnModuleDestroy,
|
||||
ServiceUnavailableException,
|
||||
} from '@nestjs/common';
|
||||
import { ConfigService } from '@nestjs/config';
|
||||
import { randomUUID } from 'node:crypto';
|
||||
import type { AppConfig } from '../config/app-config.schema.js';
|
||||
import { MetricsService } from '../metrics/metrics.service.js';
|
||||
import {
|
||||
assertTopology,
|
||||
JOB_EXCHANGE,
|
||||
JOB_QUEUE,
|
||||
JOB_ROUTING_KEY,
|
||||
} from './broker.topology.js';
|
||||
import type { JobMessage } from './job-message.js';
|
||||
|
||||
export class BrokerUnavailableError extends ServiceUnavailableException {
|
||||
constructor() {
|
||||
super('Broker unavailable');
|
||||
}
|
||||
}
|
||||
|
||||
export type ConsumeHandler = (
|
||||
message: ConsumeMessage,
|
||||
channel: Channel,
|
||||
) => Promise<void>;
|
||||
|
||||
const MAX_RECONNECT_DELAY_MS = 10_000;
|
||||
|
||||
@Injectable()
|
||||
export class BrokerClient implements OnModuleDestroy {
|
||||
private readonly logger = new Logger(BrokerClient.name);
|
||||
private readonly url: string;
|
||||
private readonly deliveryLimit: number;
|
||||
private readonly publishTimeoutMs: number;
|
||||
private connection: ChannelModel | undefined;
|
||||
private confirmChannel: ConfirmChannel | undefined;
|
||||
private consumeChannel: Channel | undefined;
|
||||
private consumerTag: string | undefined;
|
||||
private prefetch = 0;
|
||||
private handler: ConsumeHandler | undefined;
|
||||
private connecting: Promise<void> | undefined;
|
||||
private subscribing: Promise<void> | undefined;
|
||||
private reconnectTimer: NodeJS.Timeout | undefined;
|
||||
private subscribeTimer: NodeJS.Timeout | undefined;
|
||||
private reconnectDelayMs = 1000;
|
||||
private subscribeDelayMs = 1000;
|
||||
private closing = false;
|
||||
// Unroutable messages are returned before their confirm; the map lets the
|
||||
// return listener reject the exact publish that produced it.
|
||||
private readonly returned = new Map<string, () => void>();
|
||||
|
||||
constructor(
|
||||
config: ConfigService<AppConfig>,
|
||||
private readonly metrics: MetricsService,
|
||||
) {
|
||||
this.url = config.getOrThrow('BROKER_URL');
|
||||
this.deliveryLimit = config.getOrThrow('BROKER_DELIVERY_LIMIT');
|
||||
this.publishTimeoutMs = config.getOrThrow(
|
||||
'BROKER_PUBLISH_TIMEOUT_MS',
|
||||
);
|
||||
}
|
||||
|
||||
async publish(message: JobMessage | Buffer): Promise<void> {
|
||||
const channel = await this.withTimeout(
|
||||
this.ensureConnected(),
|
||||
this.publishTimeoutMs,
|
||||
);
|
||||
const content = Buffer.isBuffer(message)
|
||||
? message
|
||||
: Buffer.from(JSON.stringify(message));
|
||||
const correlationId = randomUUID();
|
||||
const options = {
|
||||
persistent: true,
|
||||
contentType: 'application/json',
|
||||
mandatory: true,
|
||||
correlationId,
|
||||
...(Buffer.isBuffer(message)
|
||||
? {}
|
||||
: {
|
||||
messageId: message.job_id,
|
||||
type: message.type,
|
||||
headers: { 'x-attempt': message.attempt },
|
||||
}),
|
||||
};
|
||||
const { promise, resolve, reject } = Promise.withResolvers<void>();
|
||||
let settled = false;
|
||||
const fail = (): void => {
|
||||
if (settled) return;
|
||||
settled = true;
|
||||
clearTimeout(timer);
|
||||
this.returned.delete(correlationId);
|
||||
this.metrics.increment('broker.publish_failed');
|
||||
reject(new BrokerUnavailableError());
|
||||
};
|
||||
const timer = setTimeout(fail, this.publishTimeoutMs);
|
||||
this.returned.set(correlationId, fail);
|
||||
try {
|
||||
channel.publish(
|
||||
JOB_EXCHANGE,
|
||||
JOB_ROUTING_KEY,
|
||||
content,
|
||||
options,
|
||||
(error) => {
|
||||
if (settled) return;
|
||||
settled = true;
|
||||
clearTimeout(timer);
|
||||
this.returned.delete(correlationId);
|
||||
if (error) {
|
||||
this.metrics.increment('broker.publish_failed');
|
||||
reject(new BrokerUnavailableError());
|
||||
return;
|
||||
}
|
||||
this.metrics.increment('broker.published');
|
||||
resolve();
|
||||
},
|
||||
);
|
||||
} catch {
|
||||
fail();
|
||||
}
|
||||
return promise;
|
||||
}
|
||||
|
||||
consume(prefetch: number, handler: ConsumeHandler): void {
|
||||
this.prefetch = prefetch;
|
||||
this.handler = handler;
|
||||
this.ensureConnected()
|
||||
.then(() => this.subscribe())
|
||||
.catch(() => this.scheduleSubscribe());
|
||||
}
|
||||
|
||||
isConsuming(): boolean {
|
||||
return this.consumerTag !== undefined;
|
||||
}
|
||||
|
||||
async queueDepth(
|
||||
queue: string,
|
||||
): Promise<{ messages: number; consumers: number }> {
|
||||
const channel = await this.withTimeout(
|
||||
this.ensureConnected(),
|
||||
this.publishTimeoutMs,
|
||||
);
|
||||
const result = await channel.checkQueue(queue);
|
||||
return {
|
||||
messages: result.messageCount,
|
||||
consumers: result.consumerCount,
|
||||
};
|
||||
}
|
||||
|
||||
async stopConsuming(): Promise<void> {
|
||||
clearTimeout(this.subscribeTimer);
|
||||
this.subscribeTimer = undefined;
|
||||
const channel = this.consumeChannel;
|
||||
const tag = this.consumerTag;
|
||||
this.consumerTag = undefined;
|
||||
this.consumeChannel = undefined;
|
||||
this.handler = undefined;
|
||||
if (channel && tag) {
|
||||
try {
|
||||
await channel.cancel(tag);
|
||||
} catch {
|
||||
// The channel is already closing; there is nothing to cancel.
|
||||
}
|
||||
}
|
||||
if (channel) {
|
||||
try {
|
||||
await channel.close();
|
||||
} catch {
|
||||
// Closing an already closed channel is harmless.
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async onModuleDestroy(): Promise<void> {
|
||||
this.closing = true;
|
||||
clearTimeout(this.reconnectTimer);
|
||||
clearTimeout(this.subscribeTimer);
|
||||
this.reconnectTimer = undefined;
|
||||
this.subscribeTimer = undefined;
|
||||
for (const reject of this.returned.values()) reject();
|
||||
this.returned.clear();
|
||||
await this.stopConsuming();
|
||||
const connection = this.connection;
|
||||
this.connection = undefined;
|
||||
this.confirmChannel = undefined;
|
||||
if (connection) {
|
||||
try {
|
||||
await connection.close();
|
||||
} catch {
|
||||
// Shutdown is best effort; channels are already unusable.
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private async ensureConnected(): Promise<ConfirmChannel> {
|
||||
if (this.closing) throw new BrokerUnavailableError();
|
||||
if (this.confirmChannel) return this.confirmChannel;
|
||||
if (!this.connecting) {
|
||||
this.connecting = this.connect()
|
||||
.then(() => undefined)
|
||||
.finally(() => {
|
||||
this.connecting = undefined;
|
||||
});
|
||||
}
|
||||
await this.connecting;
|
||||
if (!this.confirmChannel) throw new BrokerUnavailableError();
|
||||
return this.confirmChannel;
|
||||
}
|
||||
|
||||
private async connect(): Promise<void> {
|
||||
const connection = await amqpConnect(this.url);
|
||||
connection.on('error', (error: Error) => {
|
||||
this.metrics.increment('broker.error');
|
||||
this.logger.warn(`Broker connection error: ${error.message}`);
|
||||
});
|
||||
connection.on('close', () => {
|
||||
if (this.connection !== connection) return;
|
||||
this.handleClose();
|
||||
});
|
||||
try {
|
||||
const confirm = await connection.createConfirmChannel();
|
||||
confirm.on('return', (message) => {
|
||||
const correlationId = message.properties?.correlationId;
|
||||
if (!correlationId) return;
|
||||
const reject = this.returned.get(correlationId);
|
||||
if (!reject) return;
|
||||
this.returned.delete(correlationId);
|
||||
this.logger.warn('Broker returned an unroutable job message');
|
||||
reject();
|
||||
});
|
||||
await assertTopology(confirm, this.deliveryLimit);
|
||||
confirm.on('error', (error: Error) => {
|
||||
this.metrics.increment('broker.confirm_error');
|
||||
this.logger.warn(`Broker confirm channel error: ${error.message}`);
|
||||
});
|
||||
confirm.on('close', () => this.handleConfirmClose(confirm));
|
||||
this.connection = connection;
|
||||
this.confirmChannel = confirm;
|
||||
this.reconnectDelayMs = 1000;
|
||||
clearTimeout(this.reconnectTimer);
|
||||
this.reconnectTimer = undefined;
|
||||
this.metrics.increment('broker.connected');
|
||||
} catch (error) {
|
||||
if (this.connection === connection) {
|
||||
this.connection = undefined;
|
||||
this.confirmChannel = undefined;
|
||||
}
|
||||
try {
|
||||
await connection.close();
|
||||
} catch {
|
||||
// The connection is already unusable.
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
if (this.handler)
|
||||
void this.subscribe().catch(() => this.scheduleSubscribe());
|
||||
}
|
||||
|
||||
private handleConfirmClose(confirm: ConfirmChannel): void {
|
||||
if (this.confirmChannel !== confirm) return;
|
||||
this.metrics.increment('broker.confirm_closed');
|
||||
for (const reject of this.returned.values()) reject();
|
||||
this.returned.clear();
|
||||
this.confirmChannel = undefined;
|
||||
const connection = this.connection;
|
||||
if (connection) {
|
||||
void connection.close().catch(() => this.handleClose());
|
||||
} else {
|
||||
this.handleClose();
|
||||
}
|
||||
}
|
||||
|
||||
private subscribe(): Promise<void> {
|
||||
if (this.subscribing) return this.subscribing;
|
||||
const connection = this.connection;
|
||||
const handler = this.handler;
|
||||
if (!connection || !handler || this.consumerTag)
|
||||
return Promise.resolve();
|
||||
this.subscribing = this.doSubscribe(connection, handler).finally(
|
||||
() => {
|
||||
this.subscribing = undefined;
|
||||
},
|
||||
);
|
||||
return this.subscribing;
|
||||
}
|
||||
|
||||
private async doSubscribe(
|
||||
connection: ChannelModel,
|
||||
handler: ConsumeHandler,
|
||||
): Promise<void> {
|
||||
const channel = await connection.createChannel();
|
||||
try {
|
||||
channel.on('error', (error: Error) => {
|
||||
this.metrics.increment('broker.consumer_error');
|
||||
this.logger.warn(
|
||||
`Broker consumer channel error: ${error.message}`,
|
||||
);
|
||||
this.handleConsumerClose(channel);
|
||||
});
|
||||
channel.on('close', () => this.handleConsumerClose(channel));
|
||||
await channel.prefetch(this.prefetch);
|
||||
await assertTopology(channel, this.deliveryLimit);
|
||||
if (this.handler !== handler) {
|
||||
await channel.close();
|
||||
return;
|
||||
}
|
||||
const consumer = await channel.consume(
|
||||
JOB_QUEUE,
|
||||
(message) => {
|
||||
if (!message) {
|
||||
// basic.cancel from the broker: the worker must not stay
|
||||
// healthy while deaf.
|
||||
this.handleConsumerClose(channel);
|
||||
void channel.close().catch(() => undefined);
|
||||
return;
|
||||
}
|
||||
handler(message, channel).catch((error: unknown) => {
|
||||
this.metrics.increment('broker.handler_error');
|
||||
this.logger.error(
|
||||
`Job handler crashed: ${error instanceof Error ? error.message : 'unknown error'}`,
|
||||
);
|
||||
// Hold the message briefly instead of burning the
|
||||
// quorum delivery limit in a tight nack loop.
|
||||
setTimeout(() => {
|
||||
try {
|
||||
channel.nack(message, false, true);
|
||||
} catch {
|
||||
// The channel is gone; the broker redelivers after the lease.
|
||||
}
|
||||
}, 1000);
|
||||
});
|
||||
},
|
||||
{ noAck: false },
|
||||
);
|
||||
this.consumeChannel = channel;
|
||||
this.consumerTag = consumer.consumerTag;
|
||||
this.subscribeDelayMs = 1000;
|
||||
} catch (error) {
|
||||
try {
|
||||
await channel.close();
|
||||
} catch {
|
||||
// The channel is already unusable.
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
private handleConsumerClose(channel: Channel): void {
|
||||
if (this.consumeChannel && this.consumeChannel !== channel) return;
|
||||
this.consumeChannel = undefined;
|
||||
this.consumerTag = undefined;
|
||||
if (!this.closing && this.handler) this.scheduleSubscribe();
|
||||
}
|
||||
|
||||
private handleClose(): void {
|
||||
this.metrics.increment('broker.closed');
|
||||
clearTimeout(this.subscribeTimer);
|
||||
this.subscribeTimer = undefined;
|
||||
this.connection = undefined;
|
||||
this.confirmChannel = undefined;
|
||||
this.consumeChannel = undefined;
|
||||
this.consumerTag = undefined;
|
||||
for (const reject of this.returned.values()) reject();
|
||||
this.returned.clear();
|
||||
if (this.closing) return;
|
||||
this.scheduleReconnect();
|
||||
}
|
||||
|
||||
private scheduleReconnect(): void {
|
||||
if (this.closing || this.reconnectTimer) return;
|
||||
const delay = this.reconnectDelayMs;
|
||||
this.reconnectDelayMs = Math.min(
|
||||
this.reconnectDelayMs * 2,
|
||||
MAX_RECONNECT_DELAY_MS,
|
||||
);
|
||||
this.metrics.increment('broker.reconnect_scheduled');
|
||||
this.reconnectTimer = setTimeout(() => {
|
||||
this.reconnectTimer = undefined;
|
||||
this.ensureConnected()
|
||||
.then(() => this.subscribe())
|
||||
.catch(() => this.scheduleReconnect());
|
||||
}, delay);
|
||||
}
|
||||
|
||||
private scheduleSubscribe(): void {
|
||||
if (this.closing || !this.handler || this.subscribeTimer) return;
|
||||
const delay = this.subscribeDelayMs;
|
||||
this.subscribeDelayMs = Math.min(
|
||||
this.subscribeDelayMs * 2,
|
||||
MAX_RECONNECT_DELAY_MS,
|
||||
);
|
||||
this.metrics.increment('broker.subscribe_scheduled');
|
||||
this.subscribeTimer = setTimeout(() => {
|
||||
this.subscribeTimer = undefined;
|
||||
this.ensureConnected()
|
||||
.then(() => this.subscribe())
|
||||
.catch(() => this.scheduleSubscribe());
|
||||
}, delay);
|
||||
}
|
||||
|
||||
private async withTimeout<T>(promise: Promise<T>, ms: number): Promise<T> {
|
||||
const { promise: timeout, reject } = Promise.withResolvers<never>();
|
||||
const timer = setTimeout(() => reject(new BrokerUnavailableError()), ms);
|
||||
try {
|
||||
return await Promise.race([promise, timeout]);
|
||||
} finally {
|
||||
clearTimeout(timer);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,37 @@
|
||||
import type { Channel } from 'amqplib';
|
||||
|
||||
export const JOB_EXCHANGE = 'attachments.jobs';
|
||||
export const JOB_ROUTING_KEY = 'attachment.process.v1';
|
||||
export const JOB_QUEUE = 'attachments.process';
|
||||
export const DEAD_LETTER_EXCHANGE = 'attachments.dlx';
|
||||
export const DEAD_LETTER_QUEUE = 'attachments.process.dlq';
|
||||
|
||||
/**
|
||||
* Idempotent declaration used by both the producer and the consumer on every
|
||||
* (re)connect: a stand started from an empty volume and a reconnect after a
|
||||
* broker restart converge to the same durable topology.
|
||||
*/
|
||||
export async function assertTopology(
|
||||
channel: Channel,
|
||||
deliveryLimit: number,
|
||||
): Promise<void> {
|
||||
await channel.assertExchange(DEAD_LETTER_EXCHANGE, 'fanout', {
|
||||
durable: true,
|
||||
});
|
||||
await channel.assertQueue(DEAD_LETTER_QUEUE, {
|
||||
durable: true,
|
||||
arguments: { 'x-queue-type': 'quorum' },
|
||||
});
|
||||
await channel.bindQueue(DEAD_LETTER_QUEUE, DEAD_LETTER_EXCHANGE, '');
|
||||
|
||||
await channel.assertExchange(JOB_EXCHANGE, 'direct', { durable: true });
|
||||
await channel.assertQueue(JOB_QUEUE, {
|
||||
durable: true,
|
||||
arguments: {
|
||||
'x-queue-type': 'quorum',
|
||||
'x-dead-letter-exchange': DEAD_LETTER_EXCHANGE,
|
||||
'x-delivery-limit': deliveryLimit,
|
||||
},
|
||||
});
|
||||
await channel.bindQueue(JOB_QUEUE, JOB_EXCHANGE, JOB_ROUTING_KEY);
|
||||
}
|
||||
@@ -0,0 +1,33 @@
|
||||
import { describe, expect, it } from 'vitest';
|
||||
import { ObjectStorageUnavailableError } from '../storage/object-storage.js';
|
||||
import { classifyError, JobTimeoutError } from './errors.js';
|
||||
|
||||
describe('classifyError', () => {
|
||||
it('classifies unsupported sharp input as permanent invalid_image', () => {
|
||||
expect(
|
||||
classifyError(
|
||||
new Error('Input buffer contains unsupported image format'),
|
||||
),
|
||||
).toEqual({ kind: 'permanent', code: 'invalid_image' });
|
||||
});
|
||||
|
||||
it('classifies the pixel limit as permanent image_too_large', () => {
|
||||
expect(
|
||||
classifyError(new Error('Input image exceeds pixel limit')),
|
||||
).toEqual({ kind: 'permanent', code: 'image_too_large' });
|
||||
});
|
||||
|
||||
it('classifies object storage outages as transient', () => {
|
||||
expect(classifyError(new ObjectStorageUnavailableError())).toEqual({
|
||||
kind: 'transient',
|
||||
code: 'processing_failed',
|
||||
});
|
||||
});
|
||||
|
||||
it('classifies a job timeout as transient processing_timeout', () => {
|
||||
expect(classifyError(new JobTimeoutError(10))).toEqual({
|
||||
kind: 'transient',
|
||||
code: 'processing_timeout',
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,65 @@
|
||||
import { SERVER_FAULT } from '../attachments/image/image-decoder.js';
|
||||
import { InjectedFaultError } from './fault-injector.js';
|
||||
|
||||
export class SourceMissingError extends Error {
|
||||
constructor(message = 'Source object is missing') {
|
||||
super(message);
|
||||
this.name = 'SourceMissingError';
|
||||
}
|
||||
}
|
||||
|
||||
export class SourceMismatchError extends Error {
|
||||
constructor(message = 'Source object does not match its metadata') {
|
||||
super(message);
|
||||
this.name = 'SourceMismatchError';
|
||||
}
|
||||
}
|
||||
|
||||
export class UnsupportedPipelineError extends Error {
|
||||
constructor(version: number) {
|
||||
super(`Unsupported pipeline version: ${version}`);
|
||||
this.name = 'UnsupportedPipelineError';
|
||||
}
|
||||
}
|
||||
|
||||
export class JobTimeoutError extends Error {
|
||||
constructor(seconds: number) {
|
||||
super(`Job exceeded ${seconds}s deadline`);
|
||||
this.name = 'JobTimeoutError';
|
||||
}
|
||||
}
|
||||
|
||||
export type ErrorClass = {
|
||||
kind: 'permanent' | 'transient';
|
||||
code: string;
|
||||
};
|
||||
|
||||
const SHARP_INPUT =
|
||||
/unsupported image format|input buffer|input file|bad seek|premature end|vipsjpeg|vippng|corrupt|truncated|invalid (?:image|pixel)|not a valid image|exceeded.*limit/iu;
|
||||
|
||||
/**
|
||||
* Maps a processing failure to the contract's safe error class. Permanent
|
||||
* errors are never retried; transient ones are retried until max_attempts,
|
||||
* after which the job is failed as `retry_exhausted`.
|
||||
*/
|
||||
export function classifyError(error: unknown): ErrorClass {
|
||||
if (error instanceof InjectedFaultError)
|
||||
return { kind: error.kind, code: 'injected_fault' };
|
||||
if (error instanceof SourceMissingError)
|
||||
return { kind: 'permanent', code: 'source_missing' };
|
||||
if (error instanceof SourceMismatchError)
|
||||
return { kind: 'permanent', code: 'source_mismatch' };
|
||||
if (error instanceof UnsupportedPipelineError)
|
||||
return { kind: 'permanent', code: 'unsupported_pipeline' };
|
||||
if (error instanceof JobTimeoutError)
|
||||
return { kind: 'transient', code: 'processing_timeout' };
|
||||
|
||||
const message = error instanceof Error ? error.message : String(error);
|
||||
if (/pixel limit/iu.test(message))
|
||||
return { kind: 'permanent', code: 'image_too_large' };
|
||||
if (SERVER_FAULT.test(message))
|
||||
return { kind: 'transient', code: 'processing_failed' };
|
||||
if (SHARP_INPUT.test(message))
|
||||
return { kind: 'permanent', code: 'invalid_image' };
|
||||
return { kind: 'transient', code: 'processing_failed' };
|
||||
}
|
||||
@@ -0,0 +1,134 @@
|
||||
import { Injectable, Logger } from '@nestjs/common';
|
||||
import { ConfigService } from '@nestjs/config';
|
||||
import type { AppConfig } from '../config/app-config.schema.js';
|
||||
import { MetricsService } from '../metrics/metrics.service.js';
|
||||
import { RedisClient } from '../redis/redis.client.js';
|
||||
|
||||
export type FaultPoint =
|
||||
| 'api.after_commit_before_publish'
|
||||
| 'relay.after_publish_before_mark'
|
||||
| 'worker.before_side_effect'
|
||||
| 'worker.after_side_effect'
|
||||
| 'worker.after_commit_before_ack';
|
||||
|
||||
export type FaultTarget = {
|
||||
attachmentId: string;
|
||||
pipelineVersion: number;
|
||||
};
|
||||
|
||||
export type FaultKind = 'permanent' | 'transient';
|
||||
|
||||
// Node timers overflow above 2^31-1 ms and fire immediately.
|
||||
const MAX_TIMER_MS = 2_147_483_647;
|
||||
|
||||
export class InjectedFaultError extends Error {
|
||||
constructor(readonly kind: FaultKind) {
|
||||
super(`Injected ${kind} fault`);
|
||||
this.name = 'InjectedFaultError';
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Active only with FAULT_INJECTION_ENABLED=1. The switch is deliberately
|
||||
* checked before every Redis call so a normal stand never depends on Redis for
|
||||
* job correctness.
|
||||
*
|
||||
* `transient` stays until the key is deleted; every other value is consumed by
|
||||
* the first point that observes it. Worker points are addressed to one
|
||||
* attachment and pipeline version; the untargeted key is a fallback.
|
||||
*/
|
||||
@Injectable()
|
||||
export class FaultInjector {
|
||||
private readonly logger = new Logger(FaultInjector.name);
|
||||
private readonly enabled: boolean;
|
||||
|
||||
constructor(
|
||||
config: ConfigService<AppConfig>,
|
||||
private readonly redis: RedisClient,
|
||||
private readonly metrics: MetricsService,
|
||||
) {
|
||||
this.enabled = config.getOrThrow('FAULT_INJECTION_ENABLED') === 1;
|
||||
}
|
||||
|
||||
async point(
|
||||
name: FaultPoint,
|
||||
target?: FaultTarget,
|
||||
signal?: AbortSignal,
|
||||
): Promise<void> {
|
||||
if (!this.enabled) return;
|
||||
const keys = target
|
||||
? [
|
||||
`fault:${name}:${target.attachmentId}:${target.pipelineVersion}`,
|
||||
`fault:${name}`,
|
||||
]
|
||||
: [`fault:${name}`];
|
||||
for (const key of keys) {
|
||||
let value: string | null;
|
||||
let deleted: number;
|
||||
try {
|
||||
value = await this.redis.get(key);
|
||||
if (value === null) continue;
|
||||
if (value === 'transient') {
|
||||
this.metrics.increment('faults.triggered');
|
||||
throw new InjectedFaultError('transient');
|
||||
}
|
||||
deleted = await this.redis.del(key);
|
||||
} catch (error) {
|
||||
if (error instanceof InjectedFaultError) throw error;
|
||||
this.logger.warn(
|
||||
`Fault point ${name} could not read Redis: ${error instanceof Error ? error.message : 'unknown error'}`,
|
||||
);
|
||||
return;
|
||||
}
|
||||
if (deleted !== 1) continue;
|
||||
// The action is outside the Redis try/catch: an abort or an injected
|
||||
// fault must reach the job processor, not be logged as an outage.
|
||||
await this.act(value, signal);
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
private async act(value: string, signal?: AbortSignal): Promise<void> {
|
||||
if (value === 'crash') {
|
||||
this.metrics.increment('faults.triggered');
|
||||
// PID 1 in a container namespace may not be killable by SIGKILL;
|
||||
// exit 137 gives the same observable crash (no graceful shutdown).
|
||||
process.kill(process.pid, 'SIGKILL');
|
||||
process.exit(137);
|
||||
}
|
||||
if (value === 'permanent') {
|
||||
this.metrics.increment('faults.triggered');
|
||||
throw new InjectedFaultError('permanent');
|
||||
}
|
||||
if (value.startsWith('delay:')) {
|
||||
const milliseconds = Number(value.slice('delay:'.length));
|
||||
if (!Number.isFinite(milliseconds) || milliseconds < 0) return;
|
||||
this.metrics.increment('faults.triggered');
|
||||
await this.sleep(Math.min(milliseconds, MAX_TIMER_MS), signal);
|
||||
}
|
||||
}
|
||||
|
||||
private sleep(milliseconds: number, signal?: AbortSignal): Promise<void> {
|
||||
if (signal?.aborted)
|
||||
return Promise.reject(
|
||||
signal.reason instanceof Error
|
||||
? signal.reason
|
||||
: new Error('Aborted'),
|
||||
);
|
||||
const { promise, resolve, reject } = Promise.withResolvers<void>();
|
||||
const timer = setTimeout(() => {
|
||||
signal?.removeEventListener('abort', cancel);
|
||||
resolve();
|
||||
}, milliseconds);
|
||||
const cancel = (): void => {
|
||||
clearTimeout(timer);
|
||||
reject(
|
||||
signal?.reason instanceof Error
|
||||
? signal.reason
|
||||
: new Error('Aborted'),
|
||||
);
|
||||
};
|
||||
signal?.addEventListener('abort', cancel, { once: true });
|
||||
return promise;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,14 @@
|
||||
import { describe, expect, it } from 'vitest';
|
||||
import { fitInside } from './fit-inside.js';
|
||||
|
||||
describe('fitInside', () => {
|
||||
it.each([
|
||||
[320, 200, 256, [256, 160]],
|
||||
[1000, 335, 256, [256, 85]],
|
||||
[100, 50, 256, [100, 50]],
|
||||
[10000, 1, 256, [256, 1]],
|
||||
[1, 10000, 256, [1, 256]],
|
||||
])('fits %ix%i into %i as %j', (width, height, box, expected) => {
|
||||
expect(fitInside(width, height, box)).toEqual(expected);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,15 @@
|
||||
/**
|
||||
* Fits `width`x`height` into a `box`x`box` square without upscaling.
|
||||
* Integer arithmetic keeps the result exact for the contract's fixtures:
|
||||
* 320x200 -> 256x160, 1000x335 -> 256x85, 10000x1 -> 256x1.
|
||||
*/
|
||||
export function fitInside(
|
||||
width: number,
|
||||
height: number,
|
||||
box: number,
|
||||
): [number, number] {
|
||||
if (width <= box && height <= box) return [width, height];
|
||||
if (width >= height)
|
||||
return [box, Math.max(1, Math.floor((height * box) / width))];
|
||||
return [Math.max(1, Math.floor((width * box) / height)), box];
|
||||
}
|
||||
@@ -0,0 +1,47 @@
|
||||
import { readFileSync } from 'node:fs';
|
||||
import { describe, expect, it } from 'vitest';
|
||||
import { parseJobMessage } from './job-message.js';
|
||||
|
||||
const contract = JSON.parse(
|
||||
readFileSync(
|
||||
new URL('../../../contracts/jobs.schema.json', import.meta.url),
|
||||
'utf8',
|
||||
),
|
||||
) as { examples: Record<string, unknown>[] };
|
||||
const example = contract.examples[0];
|
||||
|
||||
describe('parseJobMessage', () => {
|
||||
it('accepts the contract example', () => {
|
||||
expect(parseJobMessage(Buffer.from(JSON.stringify(example)))).toEqual(
|
||||
example,
|
||||
);
|
||||
});
|
||||
|
||||
it('rejects an extra property', () => {
|
||||
expect(
|
||||
parseJobMessage(
|
||||
Buffer.from(JSON.stringify({ ...example, extra: true })),
|
||||
),
|
||||
).toBeUndefined();
|
||||
});
|
||||
|
||||
it('rejects attempt 0', () => {
|
||||
expect(
|
||||
parseJobMessage(
|
||||
Buffer.from(JSON.stringify({ ...example, attempt: 0 })),
|
||||
),
|
||||
).toBeUndefined();
|
||||
});
|
||||
|
||||
it('rejects a wrong type', () => {
|
||||
expect(
|
||||
parseJobMessage(
|
||||
Buffer.from(JSON.stringify({ ...example, type: 'other' })),
|
||||
),
|
||||
).toBeUndefined();
|
||||
});
|
||||
|
||||
it('rejects non-JSON', () => {
|
||||
expect(parseJobMessage(Buffer.from('not-json'))).toBeUndefined();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,53 @@
|
||||
import { z } from 'zod';
|
||||
|
||||
export const JOB_TYPE = 'attachment.process.v1';
|
||||
|
||||
/**
|
||||
* Mirrors contracts/jobs.schema.json. The broker message carries identifiers
|
||||
* and scheduling data only: no bytes, URLs, file names or credentials.
|
||||
*/
|
||||
export const jobMessageSchema = z.strictObject({
|
||||
schema_version: z.literal(1),
|
||||
job_id: z.uuid(),
|
||||
type: z.literal(JOB_TYPE),
|
||||
attachment_id: z.uuid(),
|
||||
pipeline_version: z.number().int().min(1),
|
||||
attempt: z.number().int().min(1),
|
||||
created_at: z.iso.datetime(),
|
||||
traceparent: z
|
||||
.string()
|
||||
.regex(/^00-[0-9a-f]{32}-[0-9a-f]{16}-[0-9a-f]{2}$/u)
|
||||
.optional(),
|
||||
tracestate: z.string().max(512).optional(),
|
||||
});
|
||||
|
||||
export type JobMessage = z.infer<typeof jobMessageSchema>;
|
||||
|
||||
export function buildJobMessage(job: {
|
||||
job_id: string;
|
||||
attachment_id: string;
|
||||
pipeline_version: number;
|
||||
attempt: number;
|
||||
created_at: Date;
|
||||
}): JobMessage {
|
||||
return {
|
||||
schema_version: 1,
|
||||
job_id: job.job_id,
|
||||
type: JOB_TYPE,
|
||||
attachment_id: job.attachment_id,
|
||||
pipeline_version: job.pipeline_version,
|
||||
attempt: job.attempt,
|
||||
created_at: job.created_at.toISOString(),
|
||||
};
|
||||
}
|
||||
|
||||
export function parseJobMessage(content: Buffer): JobMessage | undefined {
|
||||
let value: unknown;
|
||||
try {
|
||||
value = JSON.parse(content.toString('utf8'));
|
||||
} catch {
|
||||
return undefined;
|
||||
}
|
||||
const parsed = jobMessageSchema.safeParse(value);
|
||||
return parsed.success ? parsed.data : undefined;
|
||||
}
|
||||
@@ -0,0 +1,234 @@
|
||||
import { Module } from '@nestjs/common';
|
||||
import { ConfigService } from '@nestjs/config';
|
||||
import { NestFactory } from '@nestjs/core';
|
||||
import type { AppConfig } from '../config/app-config.schema.js';
|
||||
import { AppConfigModule } from '../config/app-config.module.js';
|
||||
import { DatabaseModule } from '../database/database.module.js';
|
||||
import { MetricsModule } from '../metrics/metrics.module.js';
|
||||
import { RedisModule } from '../redis/redis.module.js';
|
||||
import { ObjectStorage } from '../storage/object-storage.js';
|
||||
import { StorageModule } from '../storage/storage.module.js';
|
||||
import { DEAD_LETTER_QUEUE, JOB_QUEUE } from './broker.topology.js';
|
||||
import { BrokerClient } from './broker.client.js';
|
||||
import { JobsModule } from './jobs.module.js';
|
||||
import { JobsRepository } from './jobs.repository.js';
|
||||
|
||||
@Module({
|
||||
imports: [
|
||||
AppConfigModule,
|
||||
DatabaseModule,
|
||||
RedisModule,
|
||||
MetricsModule,
|
||||
StorageModule,
|
||||
JobsModule,
|
||||
],
|
||||
})
|
||||
class JobsCliModule {}
|
||||
|
||||
const SUPPORTED_PIPELINES = [1, 2];
|
||||
|
||||
function option(args: string[], name: string): string | undefined {
|
||||
const index = args.indexOf(name);
|
||||
if (index === -1) return undefined;
|
||||
return args[index + 1];
|
||||
}
|
||||
|
||||
async function readStdin(): Promise<string> {
|
||||
const chunks: Buffer[] = [];
|
||||
for await (const chunk of process.stdin)
|
||||
chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk));
|
||||
return Buffer.concat(chunks).toString('utf8');
|
||||
}
|
||||
|
||||
async function s3State(
|
||||
storage: ObjectStorage,
|
||||
key: string,
|
||||
): Promise<{
|
||||
exists: boolean;
|
||||
etag: string | null;
|
||||
last_modified: string | null;
|
||||
}> {
|
||||
const head = await storage.head(key);
|
||||
return head
|
||||
? {
|
||||
exists: true,
|
||||
etag: head.etag ?? null,
|
||||
last_modified: head.lastModified?.toISOString() ?? null,
|
||||
}
|
||||
: { exists: false, etag: null, last_modified: null };
|
||||
}
|
||||
|
||||
async function main(): Promise<number> {
|
||||
const args = process.argv.slice(2);
|
||||
const command = args[0];
|
||||
const app = await NestFactory.createApplicationContext(JobsCliModule, {
|
||||
logger: false,
|
||||
});
|
||||
try {
|
||||
const repository = app.get(JobsRepository);
|
||||
const storage = app.get(ObjectStorage);
|
||||
const broker = app.get(BrokerClient);
|
||||
const config = app.get(ConfigService<AppConfig>);
|
||||
|
||||
if (command === 'status') {
|
||||
const attachmentId = args[1];
|
||||
if (!attachmentId) return usage();
|
||||
const status = await repository.statusOf(attachmentId);
|
||||
if (!status) {
|
||||
console.log(JSON.stringify({ error: 'attachment_not_found' }));
|
||||
return 1;
|
||||
}
|
||||
const variants = [];
|
||||
for (const variant of status.variants) {
|
||||
variants.push({
|
||||
pipeline_version: variant.pipeline_version,
|
||||
variant: variant.variant,
|
||||
sha256: variant.sha256,
|
||||
size_bytes: Number(variant.size_bytes),
|
||||
width: variant.width,
|
||||
height: variant.height,
|
||||
object_key: variant.object_key,
|
||||
s3: await s3State(storage, variant.object_key),
|
||||
});
|
||||
}
|
||||
console.log(
|
||||
JSON.stringify({
|
||||
attachment: {
|
||||
id: status.attachment.id,
|
||||
status: status.attachment.status,
|
||||
error_code: status.attachment.error_code,
|
||||
ready_pipeline_version:
|
||||
status.attachment.ready_pipeline_version,
|
||||
staging_present: status.attachment.staging_key !== null,
|
||||
},
|
||||
jobs: status.jobs.map((job) => ({
|
||||
job_id: job.job_id,
|
||||
pipeline_version: job.pipeline_version,
|
||||
status: job.status,
|
||||
attempt: job.attempt,
|
||||
max_attempts: job.max_attempts,
|
||||
last_error_code: job.last_error_code,
|
||||
})),
|
||||
variants,
|
||||
original: await s3State(
|
||||
storage,
|
||||
status.attachment.object_key,
|
||||
),
|
||||
events: status.events.map((event) => ({
|
||||
job_id: event.job_id,
|
||||
event: event.event,
|
||||
attempt: event.attempt,
|
||||
detail: event.detail,
|
||||
created_at: event.created_at.toISOString(),
|
||||
})),
|
||||
outbox: status.outbox,
|
||||
}),
|
||||
);
|
||||
return 0;
|
||||
}
|
||||
|
||||
if (command === 'find-by-chat') {
|
||||
const chatId = args[1];
|
||||
if (!chatId) return usage();
|
||||
console.log(JSON.stringify(await repository.findByChat(chatId)));
|
||||
return 0;
|
||||
}
|
||||
|
||||
if (command === 'failed') {
|
||||
console.log(JSON.stringify(await repository.failedJobs()));
|
||||
return 0;
|
||||
}
|
||||
|
||||
if (command === 'replay') {
|
||||
const attachmentId = args[1];
|
||||
const reason = option(args, '--reason') ?? 'operator replay';
|
||||
const operator =
|
||||
option(args, '--operator') ?? process.env.USER ?? 'operator';
|
||||
if (!attachmentId) return usage();
|
||||
const candidate = await repository.replayCandidate(attachmentId);
|
||||
let sourceAvailable = false;
|
||||
if (candidate) {
|
||||
sourceAvailable =
|
||||
candidate.staging_key !== null ||
|
||||
(await storage.head(storage.objectKey(attachmentId))) !== null;
|
||||
}
|
||||
const result = await repository.replay(attachmentId, {
|
||||
reason,
|
||||
operator,
|
||||
maxAttempts: config.getOrThrow('JOB_MAX_ATTEMPTS'),
|
||||
sourceAvailable,
|
||||
expectedStagingKey: candidate?.staging_key ?? null,
|
||||
});
|
||||
console.log(JSON.stringify(result));
|
||||
return result.kind === 'replayed' ? 0 : 3;
|
||||
}
|
||||
|
||||
if (command === 'reprocess') {
|
||||
const attachmentId = args[1];
|
||||
const rawVersion = option(args, '--pipeline-version');
|
||||
const version = Number(rawVersion);
|
||||
if (
|
||||
!attachmentId ||
|
||||
rawVersion === undefined ||
|
||||
!SUPPORTED_PIPELINES.includes(version)
|
||||
)
|
||||
return usage();
|
||||
const result = await repository.reprocess(attachmentId, version, {
|
||||
reason: option(args, '--reason') ?? 'reprocess',
|
||||
operator:
|
||||
option(args, '--operator') ??
|
||||
process.env.USER ??
|
||||
'operator',
|
||||
maxAttempts: config.getOrThrow('JOB_MAX_ATTEMPTS'),
|
||||
});
|
||||
console.log(JSON.stringify(result));
|
||||
return result.kind === 'created' ? 0 : 3;
|
||||
}
|
||||
|
||||
if (command === 'publish-raw') {
|
||||
if (config.getOrThrow('FAULT_INJECTION_ENABLED') !== 1) {
|
||||
console.log(
|
||||
JSON.stringify({ refused: 'fault_injection_disabled' }),
|
||||
);
|
||||
return 3;
|
||||
}
|
||||
const text = await readStdin();
|
||||
const lines = text.split(/\r?\n/u).filter((line) => line !== '');
|
||||
for (const line of lines) await broker.publish(Buffer.from(line));
|
||||
console.log(JSON.stringify({ published: lines.length }));
|
||||
return 0;
|
||||
}
|
||||
|
||||
if (command === 'queues') {
|
||||
console.log(
|
||||
JSON.stringify({
|
||||
process: await broker.queueDepth(JOB_QUEUE),
|
||||
dlq: await broker.queueDepth(DEAD_LETTER_QUEUE),
|
||||
}),
|
||||
);
|
||||
return 0;
|
||||
}
|
||||
|
||||
return usage();
|
||||
} finally {
|
||||
await app.close();
|
||||
}
|
||||
}
|
||||
|
||||
function usage(): number {
|
||||
console.error(
|
||||
'usage: jobs.cli.js status <id> | find-by-chat <chat> | failed | replay <id> --reason <text> [--operator <name>] | reprocess <id> --pipeline-version <n> [--reason <text>] | publish-raw | queues',
|
||||
);
|
||||
return 2;
|
||||
}
|
||||
|
||||
main()
|
||||
.then((code) => process.exit(code))
|
||||
.catch((error: unknown) => {
|
||||
console.log(
|
||||
JSON.stringify({
|
||||
error: error instanceof Error ? error.message : 'unknown error',
|
||||
}),
|
||||
);
|
||||
process.exit(1);
|
||||
});
|
||||
@@ -0,0 +1,14 @@
|
||||
import { Module } from '@nestjs/common';
|
||||
import { DatabaseModule } from '../database/database.module.js';
|
||||
import { MetricsModule } from '../metrics/metrics.module.js';
|
||||
import { RedisModule } from '../redis/redis.module.js';
|
||||
import { BrokerClient } from './broker.client.js';
|
||||
import { FaultInjector } from './fault-injector.js';
|
||||
import { JobsRepository } from './jobs.repository.js';
|
||||
|
||||
@Module({
|
||||
imports: [DatabaseModule, RedisModule, MetricsModule],
|
||||
providers: [BrokerClient, JobsRepository, FaultInjector],
|
||||
exports: [BrokerClient, JobsRepository, FaultInjector],
|
||||
})
|
||||
export class JobsModule {}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,110 @@
|
||||
import {
|
||||
Injectable,
|
||||
Logger,
|
||||
OnApplicationBootstrap,
|
||||
OnModuleDestroy,
|
||||
} from '@nestjs/common';
|
||||
import { ConfigService } from '@nestjs/config';
|
||||
import type { AppConfig } from '../config/app-config.schema.js';
|
||||
import { MetricsService } from '../metrics/metrics.service.js';
|
||||
import { BrokerClient } from './broker.client.js';
|
||||
import { FaultInjector } from './fault-injector.js';
|
||||
import { JobsRepository } from './jobs.repository.js';
|
||||
|
||||
const BATCH_SIZE = 20;
|
||||
|
||||
/**
|
||||
* Runs in the API and worker processes. `publishPending` holds one transaction around
|
||||
* the broker confirms and the `published_at` update; a crash before the mark
|
||||
* rolls the transaction back and republishes the same message after restart.
|
||||
* That duplicate is expected: the worker fences it on job status.
|
||||
*/
|
||||
@Injectable()
|
||||
export class OutboxRelay implements OnApplicationBootstrap, OnModuleDestroy {
|
||||
private readonly logger = new Logger(OutboxRelay.name);
|
||||
private timer: NodeJS.Timeout | undefined;
|
||||
private running = false;
|
||||
private again = false;
|
||||
private current: Promise<void> | undefined;
|
||||
private failing = false;
|
||||
private retryAt = 0;
|
||||
private retryDelayMs = 1000;
|
||||
|
||||
constructor(
|
||||
private readonly config: ConfigService<AppConfig>,
|
||||
private readonly repository: JobsRepository,
|
||||
private readonly broker: BrokerClient,
|
||||
private readonly faults: FaultInjector,
|
||||
private readonly metrics: MetricsService,
|
||||
) {}
|
||||
|
||||
onApplicationBootstrap(): void {
|
||||
const interval = this.config.getOrThrow('OUTBOX_POLL_INTERVAL_MS');
|
||||
this.timer = setInterval(() => this.kick(), interval);
|
||||
this.timer.unref();
|
||||
this.kick();
|
||||
}
|
||||
|
||||
async onModuleDestroy(): Promise<void> {
|
||||
clearInterval(this.timer);
|
||||
this.timer = undefined;
|
||||
await this.current;
|
||||
}
|
||||
|
||||
kick(): void {
|
||||
if (Date.now() < this.retryAt) return;
|
||||
if (this.running) {
|
||||
this.again = true;
|
||||
return;
|
||||
}
|
||||
this.running = true;
|
||||
this.current = this.drain()
|
||||
.catch((error: unknown) => {
|
||||
this.metrics.increment('outbox.publish_failed');
|
||||
this.retryAt = Date.now() + this.retryDelayMs;
|
||||
this.retryDelayMs = Math.min(this.retryDelayMs * 2, 10_000);
|
||||
if (!this.failing) {
|
||||
this.failing = true;
|
||||
this.logger.warn(
|
||||
`Outbox publish failed: ${error instanceof Error ? error.message : 'unknown error'}`,
|
||||
);
|
||||
}
|
||||
})
|
||||
.finally(() => {
|
||||
this.running = false;
|
||||
this.current = undefined;
|
||||
if (this.again) {
|
||||
this.again = false;
|
||||
this.kick();
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
private async drain(): Promise<void> {
|
||||
const budgetMs = Math.max(
|
||||
1000,
|
||||
Math.floor(
|
||||
this.config.getOrThrow('DATABASE_IDLE_TX_TIMEOUT_MS') / 2,
|
||||
),
|
||||
);
|
||||
const started = Date.now();
|
||||
const published = await this.repository.publishPending(
|
||||
BATCH_SIZE,
|
||||
async (message) => {
|
||||
if (Date.now() - started > budgetMs)
|
||||
throw new Error('outbox batch deadline exceeded');
|
||||
await this.broker.publish(message);
|
||||
},
|
||||
() => this.faults.point('relay.after_publish_before_mark'),
|
||||
);
|
||||
if (published > 0) {
|
||||
this.metrics.add('outbox.published', published);
|
||||
this.retryAt = 0;
|
||||
this.retryDelayMs = 1000;
|
||||
if (this.failing) {
|
||||
this.failing = false;
|
||||
this.logger.log('Outbox publishing recovered');
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,96 @@
|
||||
import { NestFactory } from '@nestjs/core';
|
||||
import { HttpAdapterHost } from '@nestjs/core';
|
||||
import { ConfigService } from '@nestjs/config';
|
||||
import { ValidationPipe } from '@nestjs/common';
|
||||
import type { UploadRequest } from './attachments/upload-request.js';
|
||||
import {
|
||||
FastifyAdapter,
|
||||
NestFastifyApplication,
|
||||
} from '@nestjs/platform-fastify';
|
||||
import { AppModule } from './app.module.js';
|
||||
import { HealthService } from './health/health.service.js';
|
||||
import { MetricsService } from './metrics/metrics.service.js';
|
||||
import { RequestIdInterceptor } from './common/http/request-id.interceptor.js';
|
||||
|
||||
const UPLOAD_ROUTE = '/api/v1/chats/:chatId/attachments';
|
||||
const UPLOAD_STREAM_TYPE = 'application/x-messenger-upload-stream';
|
||||
|
||||
async function bootstrap(): Promise<void> {
|
||||
const app = await NestFactory.create<NestFastifyApplication>(
|
||||
AppModule,
|
||||
new FastifyAdapter({ logger: true }),
|
||||
);
|
||||
// The upload route receives its body as an unread stream whatever the
|
||||
// declared Content-Type: the service checks the type itself, after the
|
||||
// auth guard (401 before 415, contracts/README.md), and bounds the body by
|
||||
// its own limits, not Fastify's bodyLimit. The declared type is kept on the
|
||||
// request; all other routes keep Fastify's parsers untouched.
|
||||
const fastify = app.getHttpAdapter().getInstance();
|
||||
fastify.addHook(
|
||||
'onRequest',
|
||||
(request: UploadRequest, _reply: unknown, done: () => void) => {
|
||||
if (
|
||||
request.method === 'POST' &&
|
||||
request.routeOptions?.url === UPLOAD_ROUTE
|
||||
) {
|
||||
request.uploadContentType = request.headers['content-type'];
|
||||
request.headers['content-type'] = UPLOAD_STREAM_TYPE;
|
||||
}
|
||||
done();
|
||||
},
|
||||
);
|
||||
fastify.addContentTypeParser(
|
||||
UPLOAD_STREAM_TYPE,
|
||||
(
|
||||
_request: unknown,
|
||||
payload: unknown,
|
||||
done: (error: Error | null, body?: unknown) => void,
|
||||
) => done(null, payload),
|
||||
);
|
||||
app.useGlobalInterceptors(
|
||||
new RequestIdInterceptor(
|
||||
app.get(HttpAdapterHost),
|
||||
app.get(MetricsService),
|
||||
app.get(ConfigService),
|
||||
),
|
||||
);
|
||||
app.useGlobalPipes(
|
||||
new ValidationPipe({
|
||||
whitelist: true,
|
||||
forbidNonWhitelisted: true,
|
||||
transform: true,
|
||||
}),
|
||||
);
|
||||
|
||||
const config = app.get(ConfigService);
|
||||
const port = config.getOrThrow<number>('APP_PORT');
|
||||
const host = config.getOrThrow<string>('APP_HOST');
|
||||
await app.listen(port, host);
|
||||
|
||||
// Drain before teardown. Nest's own shutdown sequence destroys providers
|
||||
// (closing the database pool and Redis) before `beforeApplicationShutdown`
|
||||
// runs, so relying on it would keep the replica listed in the load balancer
|
||||
// while it can no longer serve: SIGTERM must first turn the replica
|
||||
// not-ready for the LB probe, then wait DRAIN_DELAY_MS, and only then close
|
||||
// the server and the pools.
|
||||
const health = app.get(HealthService);
|
||||
const drainDelayMs = config.getOrThrow<number>('DRAIN_DELAY_MS');
|
||||
let shuttingDown = false;
|
||||
const shutdown = async (): Promise<void> => {
|
||||
if (shuttingDown) return;
|
||||
shuttingDown = true;
|
||||
health.setDraining();
|
||||
if (drainDelayMs > 0)
|
||||
await new Promise<void>((resolve) =>
|
||||
setTimeout(resolve, drainDelayMs),
|
||||
);
|
||||
await app.close();
|
||||
process.exit(0);
|
||||
};
|
||||
for (const signal of ['SIGTERM', 'SIGINT', 'SIGHUP'] as const)
|
||||
process.once(signal, () => {
|
||||
void shutdown();
|
||||
});
|
||||
}
|
||||
|
||||
await bootstrap();
|
||||
@@ -0,0 +1,37 @@
|
||||
import { ConfigService } from '@nestjs/config';
|
||||
import { describe, expect, it, vi } from 'vitest';
|
||||
import { MetricsService } from '../../metrics/metrics.service.js';
|
||||
import { MessagesCacheService } from './messages-cache.service.js';
|
||||
|
||||
describe('MessagesCacheService metrics', () => {
|
||||
it('records a miss and a hit for a protected page fill', async () => {
|
||||
const metrics = new MetricsService();
|
||||
const page = { items: [], next_cursor: null };
|
||||
const redis = {
|
||||
get: vi
|
||||
.fn()
|
||||
.mockResolvedValueOnce(null)
|
||||
.mockResolvedValueOnce(null)
|
||||
.mockResolvedValue(JSON.stringify(page)),
|
||||
setNxPx: vi.fn().mockResolvedValue(true),
|
||||
set: vi.fn().mockResolvedValue('OK'),
|
||||
unlock: vi.fn().mockResolvedValue(1),
|
||||
};
|
||||
const config = new ConfigService({
|
||||
CACHE_MODE: 'protected',
|
||||
CACHE_LOCK_TTL_MS: 1000,
|
||||
CACHE_TTL_SECONDS: 30,
|
||||
});
|
||||
const service = new MessagesCacheService(
|
||||
redis as never,
|
||||
config,
|
||||
metrics,
|
||||
);
|
||||
await service.getOrLoad('chat', 1n, undefined, 50, async () => page);
|
||||
await service.getOrLoad('chat', 1n, undefined, 50, async () => page);
|
||||
expect(metrics.snapshot().counters).toMatchObject({
|
||||
cache_miss: 1,
|
||||
cache_hit: 1,
|
||||
});
|
||||
});
|
||||
});
|
||||
+11
@@ -0,0 +1,11 @@
|
||||
import { Module } from '@nestjs/common';
|
||||
import { MessagesCacheService } from './messages-cache.service.js';
|
||||
import { MetricsModule } from '../../metrics/metrics.module.js';
|
||||
import { RedisModule } from '../../redis/redis.module.js';
|
||||
|
||||
@Module({
|
||||
imports: [RedisModule, MetricsModule],
|
||||
providers: [MessagesCacheService],
|
||||
exports: [MessagesCacheService],
|
||||
})
|
||||
export class MessagesCacheModule {}
|
||||
@@ -0,0 +1,72 @@
|
||||
import { ConfigService } from '@nestjs/config';
|
||||
import { describe, expect, it, vi } from 'vitest';
|
||||
import type { AppConfig } from '../../config/app-config.schema.js';
|
||||
import type { RedisClient } from '../../redis/redis.client.js';
|
||||
import { MessagesCacheService } from './messages-cache.service.js';
|
||||
import { MetricsService } from '../../metrics/metrics.service.js';
|
||||
|
||||
function serviceWithRedis(redis: Partial<RedisClient>): MessagesCacheService {
|
||||
const config = new ConfigService<AppConfig>({
|
||||
CACHE_MODE: 'protected',
|
||||
CACHE_LOCK_TTL_MS: 1000,
|
||||
CACHE_TTL_SECONDS: 30,
|
||||
CACHE_MISS_DELAY_MS: 0,
|
||||
});
|
||||
return new MessagesCacheService(
|
||||
redis as RedisClient,
|
||||
config,
|
||||
new MetricsService(),
|
||||
);
|
||||
}
|
||||
|
||||
describe('MessagesCacheService fallback', () => {
|
||||
it('loads from the authoritative source when Redis read fails', async () => {
|
||||
const load = vi
|
||||
.fn()
|
||||
.mockResolvedValue({ items: [{ id: 'db' }], next_cursor: null });
|
||||
const service = serviceWithRedis({
|
||||
get: vi.fn().mockRejectedValue(new Error('redis down')),
|
||||
});
|
||||
|
||||
await expect(
|
||||
service.getOrLoad('chat', 1n, undefined, 50, load),
|
||||
).resolves.toEqual({
|
||||
items: [{ id: 'db' }],
|
||||
next_cursor: null,
|
||||
});
|
||||
expect(load).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
it('falls back after bounded lock contention', async () => {
|
||||
const load = vi
|
||||
.fn()
|
||||
.mockResolvedValue({ items: [{ id: 'db' }], next_cursor: null });
|
||||
const service = serviceWithRedis({
|
||||
get: vi.fn().mockResolvedValue(null),
|
||||
setNxPx: vi.fn().mockResolvedValue(false),
|
||||
});
|
||||
|
||||
await expect(
|
||||
service.getOrLoad('chat', 1n, undefined, 50, load),
|
||||
).resolves.toEqual({
|
||||
items: [{ id: 'db' }],
|
||||
next_cursor: null,
|
||||
});
|
||||
expect(load).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
it('returns the authoritative page when cache write fails', async () => {
|
||||
const page = { items: [{ id: 'db' }], next_cursor: null };
|
||||
const load = vi.fn().mockResolvedValue(page);
|
||||
const service = serviceWithRedis({
|
||||
get: vi.fn().mockResolvedValue(null),
|
||||
setNxPx: vi.fn().mockResolvedValue(true),
|
||||
set: vi.fn().mockRejectedValue(new Error('redis down')),
|
||||
unlock: vi.fn().mockResolvedValue(1),
|
||||
});
|
||||
|
||||
await expect(
|
||||
service.getOrLoad('chat', 1n, undefined, 50, load),
|
||||
).resolves.toEqual(page);
|
||||
});
|
||||
});
|
||||
+205
@@ -0,0 +1,205 @@
|
||||
import { Injectable, Logger } from '@nestjs/common';
|
||||
import { ConfigService } from '@nestjs/config';
|
||||
import { setTimeout as delay } from 'node:timers/promises';
|
||||
import type { AppConfig } from '../../config/app-config.schema.js';
|
||||
import { RedisClient } from '../../redis/redis.client.js';
|
||||
import type { Page } from '../../common/http/page.js';
|
||||
import { MetricsService } from '../../metrics/metrics.service.js';
|
||||
|
||||
const LOCK_RETRY_COUNT = 4;
|
||||
const LOCK_RETRY_DELAY_MS = 25;
|
||||
|
||||
export type CachePage = Page<unknown>;
|
||||
|
||||
@Injectable()
|
||||
export class MessagesCacheService {
|
||||
private readonly logger = new Logger(MessagesCacheService.name);
|
||||
|
||||
constructor(
|
||||
private readonly redis: RedisClient,
|
||||
private readonly config: ConfigService<AppConfig>,
|
||||
private readonly metrics: MetricsService,
|
||||
) {}
|
||||
|
||||
async getOrLoad(
|
||||
chatId: string,
|
||||
version: bigint,
|
||||
cursor: string | undefined,
|
||||
limit: number,
|
||||
loader: () => Promise<CachePage>,
|
||||
): Promise<CachePage> {
|
||||
const key = this.cacheKey(chatId, version, cursor, limit);
|
||||
if (this.config.getOrThrow('CACHE_MODE') === 'naive') {
|
||||
return this.loadNaively(key, loader);
|
||||
}
|
||||
|
||||
let cached: string | null;
|
||||
try {
|
||||
cached = await this.redis.get(key);
|
||||
} catch (error) {
|
||||
return this.fallbackToDatabase('read', error, loader);
|
||||
}
|
||||
const cachedPage = this.parsePage(cached);
|
||||
if (cachedPage) {
|
||||
this.metrics.increment('cache_hit');
|
||||
return cachedPage;
|
||||
}
|
||||
this.metrics.increment('cache_miss');
|
||||
|
||||
const lockKey = `lock:${key}`;
|
||||
const owner = `${process.pid}:${Math.random().toString(36).slice(2)}`;
|
||||
for (let attempt = 0; attempt < LOCK_RETRY_COUNT; attempt += 1) {
|
||||
let locked: boolean;
|
||||
try {
|
||||
locked = await this.redis.setNxPx(
|
||||
lockKey,
|
||||
owner,
|
||||
this.config.getOrThrow('CACHE_LOCK_TTL_MS'),
|
||||
);
|
||||
} catch (error) {
|
||||
return this.fallbackToDatabase('lock', error, loader);
|
||||
}
|
||||
if (locked) return this.fillLocked(key, lockKey, owner, loader);
|
||||
|
||||
if (attempt === 0) this.metrics.increment('cache_lock_wait');
|
||||
try {
|
||||
const waitedPage = this.parsePage(await this.redis.get(key));
|
||||
if (waitedPage) {
|
||||
this.metrics.increment('cache_hit');
|
||||
return waitedPage;
|
||||
}
|
||||
} catch (error) {
|
||||
return this.fallbackToDatabase('waiting read', error, loader);
|
||||
}
|
||||
await delay(LOCK_RETRY_DELAY_MS);
|
||||
}
|
||||
|
||||
// The owner did not publish the page within the bounded wait: serve the
|
||||
// authoritative result instead of blocking the request further.
|
||||
this.metrics.increment('cache_lock_timeout');
|
||||
return loader();
|
||||
}
|
||||
|
||||
private async fillLocked(
|
||||
key: string,
|
||||
lockKey: string,
|
||||
owner: string,
|
||||
loader: () => Promise<CachePage>,
|
||||
): Promise<CachePage> {
|
||||
try {
|
||||
let secondCheck: string | null;
|
||||
try {
|
||||
secondCheck = await this.redis.get(key);
|
||||
} catch (error) {
|
||||
return this.fallbackToDatabase('double-check', error, loader);
|
||||
}
|
||||
const secondPage = this.parsePage(secondCheck);
|
||||
if (secondPage) {
|
||||
this.metrics.increment('cache_hit');
|
||||
return secondPage;
|
||||
}
|
||||
|
||||
const page = await loader();
|
||||
try {
|
||||
await this.redis.set(key, JSON.stringify(page), {
|
||||
EX: this.config.getOrThrow('CACHE_TTL_SECONDS'),
|
||||
});
|
||||
} catch (error) {
|
||||
// Redis is derived state: the page is still valid for this response,
|
||||
// but the failure must stay visible in metrics and logs.
|
||||
this.metrics.increment('cache_write_error');
|
||||
this.logger.warn(
|
||||
`Cache fill failed: ${error instanceof Error ? error.message : 'unknown error'}`,
|
||||
);
|
||||
}
|
||||
return page;
|
||||
} finally {
|
||||
try {
|
||||
await this.redis.unlock(lockKey, owner);
|
||||
} catch (error) {
|
||||
// The lock has a TTL, so a stale lock expires on its own; a successful
|
||||
// response must not be replaced by an unlock failure.
|
||||
this.metrics.increment('cache_unlock_error');
|
||||
this.logger.warn(
|
||||
`Cache unlock failed: ${error instanceof Error ? error.message : 'unknown error'}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private cacheKey(
|
||||
chatId: string,
|
||||
version: bigint,
|
||||
cursor: string | undefined,
|
||||
limit: number,
|
||||
): string {
|
||||
return `messages:${chatId}:v${version.toString()}:c${cursor ?? 'start'}:l${limit}`;
|
||||
}
|
||||
|
||||
private async loadNaively(
|
||||
key: string,
|
||||
loader: () => Promise<CachePage>,
|
||||
): Promise<CachePage> {
|
||||
let cached: string | null;
|
||||
try {
|
||||
cached = await this.redis.get(key);
|
||||
} catch (error) {
|
||||
return this.fallbackToDatabase('naive read', error, loader);
|
||||
}
|
||||
const cachedPage = this.parsePage(cached);
|
||||
if (cachedPage) {
|
||||
this.metrics.increment('cache_hit');
|
||||
return cachedPage;
|
||||
}
|
||||
this.metrics.increment('cache_miss');
|
||||
|
||||
const cacheMissDelay = this.config.getOrThrow('CACHE_MISS_DELAY_MS');
|
||||
if (cacheMissDelay > 0) await delay(cacheMissDelay);
|
||||
const page = await loader();
|
||||
try {
|
||||
await this.redis.set(key, JSON.stringify(page), {
|
||||
EX: this.config.getOrThrow('CACHE_TTL_SECONDS'),
|
||||
});
|
||||
} catch (error) {
|
||||
this.metrics.increment('cache_write_error');
|
||||
this.logger.warn(
|
||||
`Cache fill failed: ${error instanceof Error ? error.message : 'unknown error'}`,
|
||||
);
|
||||
}
|
||||
return page;
|
||||
}
|
||||
|
||||
private async fallbackToDatabase(
|
||||
stage: string,
|
||||
error: unknown,
|
||||
loader: () => Promise<CachePage>,
|
||||
): Promise<CachePage> {
|
||||
this.metrics.increment('cache_fallback');
|
||||
this.logger.warn(
|
||||
`Cache ${stage} failed, serving PostgreSQL: ${error instanceof Error ? error.message : 'unknown error'}`,
|
||||
);
|
||||
return loader();
|
||||
}
|
||||
|
||||
private parsePage(value: string | null): CachePage | undefined {
|
||||
if (!value) return undefined;
|
||||
try {
|
||||
const parsed: unknown = JSON.parse(value);
|
||||
if (
|
||||
typeof parsed !== 'object' ||
|
||||
parsed === null ||
|
||||
!('items' in parsed) ||
|
||||
!('next_cursor' in parsed) ||
|
||||
!Array.isArray(parsed.items) ||
|
||||
(parsed.next_cursor !== null &&
|
||||
typeof parsed.next_cursor !== 'string')
|
||||
)
|
||||
return undefined;
|
||||
return { items: parsed.items, next_cursor: parsed.next_cursor };
|
||||
} catch {
|
||||
// A malformed entry is treated as a miss and overwritten by the owner fill.
|
||||
this.metrics.increment('cache_corrupt_entry');
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
import {
|
||||
ArrayMaxSize,
|
||||
ArrayUnique,
|
||||
IsArray,
|
||||
IsString,
|
||||
IsUUID,
|
||||
ValidateIf,
|
||||
} from 'class-validator';
|
||||
import { CodePointLength } from '../../common/validation/code-point-length.js';
|
||||
|
||||
export class CreateMessageDto {
|
||||
@IsString()
|
||||
@CodePointLength(0, 4000)
|
||||
text!: string;
|
||||
|
||||
// Absent means []; null is a client error.
|
||||
@ValidateIf((_, value) => value !== undefined)
|
||||
@IsArray()
|
||||
@ArrayMaxSize(10)
|
||||
@ArrayUnique()
|
||||
@IsUUID('all', { each: true })
|
||||
attachment_ids: string[] = [];
|
||||
}
|
||||
@@ -0,0 +1,33 @@
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import { parseDatabaseConfig } from '../../config/app-config.schema.js';
|
||||
import { DatabaseClient } from '../../database/database.client.js';
|
||||
import { MetricsService } from '../../metrics/metrics.service.js';
|
||||
import { IdempotencyRepository } from './idempotency.repository.js';
|
||||
|
||||
// Manual operator pass over the idempotency ledger: the API also purges expired
|
||||
// claims in the background, but an operator must be able to reclaim space
|
||||
// without waiting for traffic. It never touches unexpired claims.
|
||||
// in the stand: docker compose exec api node dist/messages/idempotency/idempotency-gc.cli.js
|
||||
// on the host: npm run idempotency:gc (needs a reachable DATABASE_URL)
|
||||
async function main(): Promise<void> {
|
||||
const config = parseDatabaseConfig(process.env as Record<string, unknown>);
|
||||
const database = new DatabaseClient(
|
||||
{
|
||||
getOrThrow: (key: keyof typeof config) => config[key],
|
||||
} as never,
|
||||
new MetricsService(),
|
||||
);
|
||||
try {
|
||||
const repository = new IdempotencyRepository(database);
|
||||
const purged = await repository.purgeExpired(
|
||||
config.IDEMPOTENCY_TTL_SECONDS,
|
||||
);
|
||||
process.stdout.write(
|
||||
`Purged ${purged} idempotency key(s) older than ${config.IDEMPOTENCY_TTL_SECONDS} seconds\n`,
|
||||
);
|
||||
} finally {
|
||||
await database.onModuleDestroy();
|
||||
}
|
||||
}
|
||||
|
||||
if (process.argv[1] === fileURLToPath(import.meta.url)) await main();
|
||||
@@ -0,0 +1,10 @@
|
||||
import { Module } from '@nestjs/common';
|
||||
import { DatabaseModule } from '../../database/database.module.js';
|
||||
import { IdempotencyRepository } from './idempotency.repository.js';
|
||||
|
||||
@Module({
|
||||
imports: [DatabaseModule],
|
||||
providers: [IdempotencyRepository],
|
||||
exports: [IdempotencyRepository],
|
||||
})
|
||||
export class IdempotencyModule {}
|
||||
@@ -0,0 +1,24 @@
|
||||
import { describe, expect, it, vi } from 'vitest';
|
||||
import { IdempotencyRepository } from './idempotency.repository.js';
|
||||
|
||||
describe('IdempotencyRepository expired-key cleanup', () => {
|
||||
it('deletes only rows older than the configured TTL', async () => {
|
||||
const query = vi.fn().mockResolvedValue({ rowCount: 0, rows: [] });
|
||||
const repository = new IdempotencyRepository({ query } as never);
|
||||
|
||||
await repository.forget(
|
||||
'user',
|
||||
'/api/v1/chats/chat/messages',
|
||||
'key',
|
||||
86400,
|
||||
);
|
||||
|
||||
expect(query).toHaveBeenCalledWith(
|
||||
'idempotency.forget',
|
||||
expect.stringContaining(
|
||||
'created_at < now() - make_interval(secs => $4::double precision)',
|
||||
),
|
||||
['user', '/api/v1/chats/chat/messages', 'key', 86400],
|
||||
);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,119 @@
|
||||
import { Injectable } from '@nestjs/common';
|
||||
import pg from 'pg';
|
||||
import { DatabaseClient } from '../../database/database.client.js';
|
||||
import type { Message } from '../message.js';
|
||||
|
||||
// A committed row always carries the response: the claim and the stored response
|
||||
// are written by one statement, so there is no poisoned intermediate state to
|
||||
// re-check at replay time.
|
||||
export type IdempotencyRecord = {
|
||||
request_fingerprint: string;
|
||||
status_code: number;
|
||||
response_body: Message;
|
||||
created_at: Date | string;
|
||||
expired: boolean;
|
||||
is_member: boolean;
|
||||
};
|
||||
|
||||
const PURGE_BATCH_LIMIT = 1000;
|
||||
const PURGE_BATCHES = 10;
|
||||
|
||||
@Injectable()
|
||||
export class IdempotencyRepository {
|
||||
constructor(private readonly database: DatabaseClient) {}
|
||||
|
||||
/**
|
||||
* Reads the stored result and re-checks access in the same round trip: a
|
||||
* replay must still prove that the caller may read this chat.
|
||||
*/
|
||||
async read(
|
||||
userId: string,
|
||||
path: string,
|
||||
key: string,
|
||||
chatId: string,
|
||||
ttlSeconds: number,
|
||||
): Promise<IdempotencyRecord | undefined> {
|
||||
const result = await this.database.query<IdempotencyRecord>(
|
||||
'idempotency.read',
|
||||
`SELECT k.request_fingerprint, k.status_code, k.response_body, k.created_at,
|
||||
k.created_at < now() - make_interval(secs => $4::double precision) AS expired,
|
||||
EXISTS (
|
||||
SELECT 1 FROM chat_members WHERE chat_id = $5 AND user_id = $1
|
||||
) AS is_member
|
||||
FROM idempotency_keys k
|
||||
WHERE k.user_id = $1 AND k.method = 'POST' AND k.path = $2 AND k.key = $3`,
|
||||
[userId, path, key, ttlSeconds, chatId],
|
||||
);
|
||||
return result.rows[0];
|
||||
}
|
||||
|
||||
async forget(
|
||||
userId: string,
|
||||
path: string,
|
||||
key: string,
|
||||
ttlSeconds: number,
|
||||
): Promise<void> {
|
||||
await this.database.query(
|
||||
'idempotency.forget',
|
||||
`DELETE FROM idempotency_keys
|
||||
WHERE user_id = $1 AND method = 'POST' AND path = $2 AND key = $3
|
||||
AND created_at < now() - make_interval(secs => $4::double precision)`,
|
||||
[userId, path, key, ttlSeconds],
|
||||
);
|
||||
}
|
||||
|
||||
async isMember(chatId: string, userId: string): Promise<boolean> {
|
||||
const result = await this.database.query<{ is_member: boolean }>(
|
||||
'messages.checkMembership',
|
||||
`SELECT EXISTS (
|
||||
SELECT 1 FROM chat_members WHERE chat_id = $1 AND user_id = $2
|
||||
) AS is_member`,
|
||||
[chatId, userId],
|
||||
);
|
||||
return result.rows[0]?.is_member === true;
|
||||
}
|
||||
|
||||
/**
|
||||
* Deletes expired claims in bounded batches. The transaction-scoped advisory
|
||||
* lock makes the pass single-flight across replicas without risking a lock
|
||||
* leaking back into the pool with a released connection. Returns the number
|
||||
* of deleted rows (0 when another replica is purging).
|
||||
*/
|
||||
async purgeExpired(ttlSeconds: number): Promise<number> {
|
||||
const client: pg.PoolClient = await this.database.pool.connect();
|
||||
try {
|
||||
await client.query('BEGIN');
|
||||
try {
|
||||
const lock = await client.query<{ locked: boolean }>(
|
||||
"SELECT pg_try_advisory_xact_lock(hashtext('idempotency_gc')) AS locked",
|
||||
);
|
||||
if (lock.rows[0]?.locked !== true) {
|
||||
await client.query('ROLLBACK');
|
||||
return 0;
|
||||
}
|
||||
let purged = 0;
|
||||
for (let batch = 0; batch < PURGE_BATCHES; batch += 1) {
|
||||
const result = await client.query(
|
||||
`DELETE FROM idempotency_keys
|
||||
WHERE ctid IN (
|
||||
SELECT ctid FROM idempotency_keys
|
||||
WHERE created_at < now() - make_interval(secs => $1::double precision)
|
||||
LIMIT ${PURGE_BATCH_LIMIT}
|
||||
)`,
|
||||
[ttlSeconds],
|
||||
);
|
||||
const deleted = result.rowCount ?? 0;
|
||||
purged += deleted;
|
||||
if (deleted < PURGE_BATCH_LIMIT) break;
|
||||
}
|
||||
await client.query('COMMIT');
|
||||
return purged;
|
||||
} catch (error) {
|
||||
await client.query('ROLLBACK').catch(() => undefined);
|
||||
throw error;
|
||||
}
|
||||
} finally {
|
||||
client.release();
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,26 @@
|
||||
import { createHash } from 'node:crypto';
|
||||
|
||||
// Contract: ASCII [A-Za-z0-9._:-]{1,128}. Anything else (including a duplicated
|
||||
// header, which Fastify joins with ", ") is rejected as a client error.
|
||||
export const IDEMPOTENCY_KEY_PATTERN = /^[A-Za-z0-9._:-]{1,128}$/;
|
||||
|
||||
export function messagesPath(chatId: string): string {
|
||||
return `/api/v1/chats/${chatId}/messages`;
|
||||
}
|
||||
|
||||
// The fingerprint is derived from the validated payload with a fixed field
|
||||
// order, so neither JSON key order nor an absent optional field can change it.
|
||||
// Text-only messages keep the lab3 fingerprint; the order of attachment ids
|
||||
// is significant because it is the order of attachments in the message.
|
||||
export function idempotencyFingerprint(
|
||||
text: string,
|
||||
attachmentIds: string[],
|
||||
): string {
|
||||
return createHash('sha256')
|
||||
.update(
|
||||
attachmentIds.length === 0
|
||||
? JSON.stringify({ text })
|
||||
: JSON.stringify({ text, attachment_ids: attachmentIds }),
|
||||
)
|
||||
.digest('hex');
|
||||
}
|
||||
@@ -0,0 +1,8 @@
|
||||
export type Message = {
|
||||
id: string;
|
||||
chat_id: string;
|
||||
sender_id: string;
|
||||
text: string;
|
||||
created_at: string;
|
||||
attachment_ids: string[];
|
||||
};
|
||||
@@ -0,0 +1,53 @@
|
||||
import {
|
||||
BadRequestException,
|
||||
Body,
|
||||
Controller,
|
||||
DefaultValuePipe,
|
||||
Get,
|
||||
Headers,
|
||||
Param,
|
||||
ParseIntPipe,
|
||||
ParseUUIDPipe,
|
||||
Post,
|
||||
Query,
|
||||
UseGuards,
|
||||
} from '@nestjs/common';
|
||||
import { CurrentUser } from '../common/auth/current-user.decorator.js';
|
||||
import { CurrentUserGuard } from '../common/auth/current-user.guard.js';
|
||||
import type { User } from '../users/user.js';
|
||||
import { CreateMessageDto } from './dto/create-message.dto.js';
|
||||
import { MessagesService } from './messages.service.js';
|
||||
|
||||
@Controller('api/v1/chats/:chatId/messages')
|
||||
@UseGuards(CurrentUserGuard)
|
||||
export class MessagesController {
|
||||
constructor(private readonly messagesService: MessagesService) {}
|
||||
|
||||
@Post()
|
||||
create(
|
||||
@Param('chatId', new ParseUUIDPipe()) chatId: string,
|
||||
@Body() body: CreateMessageDto,
|
||||
@CurrentUser() currentUser: User,
|
||||
@Headers('idempotency-key') idempotencyKey?: string | string[],
|
||||
) {
|
||||
return this.messagesService.create(
|
||||
chatId,
|
||||
body,
|
||||
currentUser,
|
||||
idempotencyKey,
|
||||
);
|
||||
}
|
||||
|
||||
@Get()
|
||||
list(
|
||||
@Param('chatId', new ParseUUIDPipe()) chatId: string,
|
||||
@CurrentUser() currentUser: User,
|
||||
@Query('limit', new DefaultValuePipe(50), new ParseIntPipe())
|
||||
limit: number,
|
||||
@Query('cursor') cursor?: string,
|
||||
) {
|
||||
if (limit < 1 || limit > 100)
|
||||
throw new BadRequestException('limit must be between 1 and 100');
|
||||
return this.messagesService.list(chatId, currentUser, limit, cursor);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,28 @@
|
||||
import { Module } from '@nestjs/common';
|
||||
import { DatabaseModule } from '../database/database.module.js';
|
||||
import { ChatsModule } from '../chats/chats.module.js';
|
||||
import { MessagesController } from './messages.controller.js';
|
||||
import { MessagesRepository } from './messages.repository.js';
|
||||
import { MessagesService } from './messages.service.js';
|
||||
import { MessagesCacheModule } from './cache/messages-cache.module.js';
|
||||
import { IdempotencyModule } from './idempotency/idempotency.module.js';
|
||||
import { UsersModule } from '../users/users.module.js';
|
||||
import { RedisModule } from '../redis/redis.module.js';
|
||||
import { AttachmentsModule } from '../attachments/attachments.module.js';
|
||||
import { MetricsModule } from '../metrics/metrics.module.js';
|
||||
|
||||
@Module({
|
||||
imports: [
|
||||
ChatsModule,
|
||||
MessagesCacheModule,
|
||||
IdempotencyModule,
|
||||
DatabaseModule,
|
||||
UsersModule,
|
||||
RedisModule,
|
||||
MetricsModule,
|
||||
AttachmentsModule,
|
||||
],
|
||||
controllers: [MessagesController],
|
||||
providers: [MessagesRepository, MessagesService],
|
||||
})
|
||||
export class MessagesModule {}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user