forked from mtusi2026/mtusi2026lab6
Compare commits
1
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
c236059a83 |
+17
-13
@@ -1,30 +1,34 @@
|
||||
# Copy to .env for your LOCAL synthetic test environment; never commit .env.
|
||||
# These are logical interface suggestions; map actual framework names in REPORT.md.
|
||||
APP_HOST=0.0.0.0
|
||||
APP_PORT=8080
|
||||
APP_ENV=lab
|
||||
SESSION_TTL_SECONDS=3600
|
||||
# Choose credentials locally. File paths contain no secret values.
|
||||
DATABASE_URL_FILE=/run/secrets/database_url
|
||||
# Optional on 3, required use on 4 in lab2; Redis OR Valkey:
|
||||
SESSION_STORE=postgres
|
||||
CACHE_URL_FILE=/run/secrets/cache_url
|
||||
API_REPLICAS=2
|
||||
IDEMPOTENCY_TTL_SECONDS=86400
|
||||
SESSION_STORE=redis
|
||||
REDIS_ADDR=redis:6379
|
||||
S3_ENDPOINT=http://s3:9000
|
||||
S3_REGION=us-east-1
|
||||
S3_BUCKET=messenger-attachments
|
||||
S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id
|
||||
S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key
|
||||
MAX_UPLOAD_BYTES=10485760
|
||||
MAX_IMAGE_PIXELS=40000000
|
||||
BROKER_URL_FILE=/run/secrets/broker_url
|
||||
WORKER_CONCURRENCY=2
|
||||
WORKER_CONCURRENCY=1
|
||||
WORKER_PREFETCH=1
|
||||
JOB_MAX_ATTEMPTS=5
|
||||
JOB_TIMEOUT_SECONDS=60
|
||||
JOB_LEASE_SECONDS=90
|
||||
JOB_RETRY_BASE_SECONDS=2
|
||||
STAGING_RETENTION_SECONDS=86400
|
||||
OTEL_SERVICE_NAME=messenger-api
|
||||
# Worker must set a distinct service.name.
|
||||
CLEANUP_GRACE_SECONDS=600
|
||||
CLEANUP_INTERVAL_SECONDS=5
|
||||
|
||||
# Lab 6 observability. Compose sets per-process service.name.
|
||||
LAB6_HTTP_PORT=8086
|
||||
LAB6_POSTGRES_PORT=15486
|
||||
LAB6_GRAFANA_PORT=3006
|
||||
LAB6_PROMETHEUS_PORT=9096
|
||||
LAB6_LOKI_PORT=3106
|
||||
LAB6_TEMPO_PORT=3206
|
||||
OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4318
|
||||
OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf
|
||||
OTEL_RESOURCE_ATTRIBUTES=deployment.environment.name=lab
|
||||
OTEL_SERVICE_VERSION=lab6
|
||||
|
||||
@@ -1,18 +1,27 @@
|
||||
## Лабораторная 6
|
||||
|
||||
Автор / группа: TODO
|
||||
Целевая оценка (3/4/5): TODO
|
||||
Трек 5, если заявлен: TODO
|
||||
Олег Ремез, БПИ2304. Целевая оценка — 4.
|
||||
|
||||
Что реализовано и что перенесено: TODO
|
||||
Команды запуска из чистого clone: TODO
|
||||
Результаты проверок и ссылка на REPORT.md: TODO
|
||||
Ограничения / известные ошибки: TODO
|
||||
Источники и использование ИИ: TODO
|
||||
Перенесён мессенджер ЛР 5 с двумя API-репликами, outbox и обработкой
|
||||
вложений. Добавлены OpenTelemetry, Collector, Prometheus, Loki, Tempo и
|
||||
Grafana. W3C-контекст связывает загрузку с обработкой и повторной попыткой.
|
||||
|
||||
- [ ] Весь обязательный минимум текущей и предыдущих работ сохранён с объявленными переходами.
|
||||
- [ ] Контракт и публичные тесты не ослаблены.
|
||||
- [ ] make check и make test выполнены; приложен реальный вывод.
|
||||
- [ ] Сценарии отказов/безопасности своего уровня воспроизведены.
|
||||
- [ ] Все секреты и приватные ключи находятся вне Git; отчёт не содержит tokens.
|
||||
- [ ] README задания сохранён, REPORT.md содержит инструкцию именно моего решения.
|
||||
### Запуск
|
||||
|
||||
```sh
|
||||
sh scripts/init-local.sh
|
||||
docker compose --env-file .env up -d --build --wait --wait-timeout 180
|
||||
make check test BASE_URL=http://127.0.0.1:8086
|
||||
```
|
||||
|
||||
### Проверки
|
||||
|
||||
Все девять проверок приёмки прошли. При остановке worker и блокировке БД
|
||||
оба локальных алерта сработали и вернулись в inactive. За 34 с недоступности
|
||||
Loki сохранились восемь подтверждённых сообщений, потери логов составили
|
||||
96 записей. Копия для сдачи содержит конфигурацию и обезличенные результаты.
|
||||
Для запуска локальные секреты создаёт `init-local.sh`.
|
||||
|
||||
Отчёт содержит SLO, ограничения экспериментов, ссылки на первичные источники
|
||||
и описание участия Codex в реализации и тестировании.
|
||||
[Отчёт](REPORT.md) · [Сводка приёмки](evidence/lab6-grade4/summary.json).
|
||||
|
||||
@@ -0,0 +1,21 @@
|
||||
name: Lab 6 application
|
||||
on: [push, pull_request]
|
||||
permissions:
|
||||
contents: read
|
||||
jobs:
|
||||
compose-smoke:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 20
|
||||
env:
|
||||
COMPOSE_PROJECT_NAME: lab6-ci
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- name: Initialize disposable configuration
|
||||
run: sh scripts/init-local.sh
|
||||
- name: Build and start
|
||||
run: docker compose --env-file .env up -d --build --wait --wait-timeout 180
|
||||
- name: Run contract smoke
|
||||
run: make check test BASE_URL=http://127.0.0.1:8086
|
||||
- name: Remove the CI project
|
||||
if: always()
|
||||
run: docker compose --env-file .env down --volumes
|
||||
+45
@@ -0,0 +1,45 @@
|
||||
FROM golang:1.27.1-alpine AS builder
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
COPY go.mod go.sum ./
|
||||
RUN go mod download
|
||||
|
||||
COPY cmd/api/ ./cmd/api/
|
||||
COPY cmd/jobs/ ./cmd/jobs/
|
||||
COPY cmd/relay/ ./cmd/relay/
|
||||
COPY internal/ ./internal/
|
||||
|
||||
RUN --mount=type=cache,id=highload-go-build,target=/root/.cache/go-build,sharing=locked \
|
||||
CGO_ENABLED=0 GOOS=linux go build -p 2 -trimpath -o /app/server ./cmd/api && \
|
||||
CGO_ENABLED=0 GOOS=linux go build -p 2 -trimpath -o /app/jobs ./cmd/jobs && \
|
||||
CGO_ENABLED=0 GOOS=linux go build -p 2 -trimpath -o /app/relay ./cmd/relay
|
||||
|
||||
|
||||
FROM alpine:3.22 AS runtime
|
||||
|
||||
RUN addgroup -S app && adduser -S app -G app
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
USER app
|
||||
|
||||
FROM runtime AS cleanup
|
||||
|
||||
COPY --from=builder /app/jobs ./jobs
|
||||
ENTRYPOINT ["/app/jobs"]
|
||||
CMD ["cleanup"]
|
||||
|
||||
FROM runtime AS relay
|
||||
COPY --from=builder /app/relay ./relay
|
||||
CMD ["./relay"]
|
||||
|
||||
FROM runtime AS api
|
||||
|
||||
COPY --from=builder /app/server ./server
|
||||
EXPOSE 8080
|
||||
|
||||
HEALTHCHECK --interval=10s --timeout=3s --start-period=3s --retries=3 \
|
||||
CMD wget -q -O /dev/null http://127.0.0.1:8080/health/ready || exit 1
|
||||
|
||||
CMD ["./server"]
|
||||
@@ -0,0 +1,20 @@
|
||||
FROM golang:1.27.1-alpine AS builder
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
COPY go.mod go.sum ./
|
||||
RUN go mod download
|
||||
|
||||
COPY cmd/migrate/ ./cmd/migrate/
|
||||
COPY db/migrations/ ./db/migrations/
|
||||
RUN --mount=type=cache,id=highload-go-build,target=/root/.cache/go-build,sharing=locked \
|
||||
CGO_ENABLED=0 GOOS=linux go build -p 2 -trimpath -o /app/migrate ./cmd/migrate
|
||||
|
||||
FROM alpine:3.22
|
||||
|
||||
RUN addgroup -S app && adduser -S app -G app
|
||||
WORKDIR /app
|
||||
COPY --from=builder /app/migrate ./migrate
|
||||
|
||||
USER app
|
||||
ENTRYPOINT ["/app/migrate"]
|
||||
@@ -0,0 +1,23 @@
|
||||
FROM golang:1.27.1-alpine AS builder
|
||||
|
||||
WORKDIR /app
|
||||
COPY go.mod go.sum ./
|
||||
RUN go mod download
|
||||
COPY cmd/worker/ ./cmd/worker/
|
||||
COPY internal/ ./internal/
|
||||
RUN --mount=type=cache,id=highload-go-build,target=/root/.cache/go-build,sharing=locked \
|
||||
CGO_ENABLED=0 GOOS=linux go build -p 2 -trimpath -o /app/worker ./cmd/worker
|
||||
|
||||
FROM alpine:3.22@sha256:49340b346798875b5592e5d420241f2f9cfba5f47e62ebb1446a1b34734e898f
|
||||
|
||||
RUN apk add --no-cache imagemagick imagemagick-jpeg imagemagick-webp \
|
||||
&& addgroup -S app && adduser -S -G app app
|
||||
|
||||
COPY infra/imagemagick/policy.xml /etc/ImageMagick-7/policy.xml
|
||||
|
||||
COPY --from=builder /app/worker /usr/local/bin/worker
|
||||
|
||||
WORKDIR /tmp
|
||||
USER app
|
||||
|
||||
CMD ["/usr/local/bin/worker"]
|
||||
@@ -1,37 +1,185 @@
|
||||
# Отчёт — лабораторная 6
|
||||
# Лабораторная 6. Наблюдаемость
|
||||
|
||||
- Автор, группа: TODO
|
||||
- Целевая оценка; выбранный трек 5, если нужен: TODO
|
||||
- Commit/PR: TODO после публикации
|
||||
- Использованные источники, библиотеки и помощь ИИ: TODO
|
||||
- Автор и группа: Олег Ремез, БПИ2304.
|
||||
- Целевая оценка: 4. Ветка реализации — `solution/BPI2304-remez`.
|
||||
- Реализация и отчёт подготовлены с помощью Codex. Независимые сценарии приёмки выполнил отдельный тестовый чат.
|
||||
|
||||
## Запуск из чистого clone
|
||||
## Запуск из чистого клона
|
||||
|
||||
TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных.
|
||||
Нужны Docker с Compose v2, OpenSSL, make и Python 3.14. Приложение
|
||||
собирается на Go 1.27.1 в Docker. Compose закрепляет версии образов. Для первого
|
||||
запуска нужно скачивание образов и модулей Go.
|
||||
|
||||
## Архитектура и решения
|
||||
```sh
|
||||
sh scripts/init-local.sh
|
||||
docker compose --env-file .env config --quiet
|
||||
docker compose --env-file .env up -d --build --wait --wait-timeout 180
|
||||
curl --fail http://127.0.0.1:8086/health/ready
|
||||
make check test BASE_URL=http://127.0.0.1:8086
|
||||
```
|
||||
|
||||
TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab5, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации.
|
||||
Миграции выполняет сервис `migrate`; API и worker ждут его успешного завершения.
|
||||
`init-local.sh` создаёт локальные `.env` и секреты в `secrets/`; при повторном
|
||||
запуске сохраняет существующие значения. Наружные порты можно изменить
|
||||
переменными `LAB6_*_PORT`, внутренний порт API и количество реплик согласованы
|
||||
с настройками HAProxy. API использует Redis по `redis:6379`. Настройки worker,
|
||||
повторов и очистки перечислены в `.env.example`. Лимиты загрузки и обработки
|
||||
изображений заданы в `internal/imageproc`: 10 МиБ и 40 млн пикселей.
|
||||
|
||||
- API через HAProxy доступен на [8086](http://127.0.0.1:8086/health/ready).
|
||||
- Dashboard [Messenger observability](http://127.0.0.1:3006/d/lab6-messenger)
|
||||
и [Prometheus alerts](http://127.0.0.1:9096/alerts) доступны по loopback.
|
||||
- Loki и Tempo отвечают на `127.0.0.1:3106` и `127.0.0.1:3206`. Grafana
|
||||
автоматически получает источники и dashboard из `infra/otel/grafana`.
|
||||
- `docker compose down` останавливает стенд и сохраняет историю в именованных
|
||||
томах. `docker compose down --volumes` удаляет данные этого Compose-проекта.
|
||||
|
||||
## Архитектура
|
||||
|
||||
API, relay и worker отправляют OTLP/HTTP на Collector 0.131.0. Он выставляет
|
||||
метрики на scrape endpoint для Prometheus 3.5.0, отправляет логи через Loki
|
||||
OTLP endpoint в Loki 3.5.0, трассы через OTLP в Tempo 2.8.1. Prometheus хранит
|
||||
временные ряды, Loki индексирует логи по служебным labels, Tempo хранит дерево
|
||||
spans. Grafana 12.3.1 читает эти данные из трёх provisioned datasources.
|
||||
Окно хранения всех трёх сигналов составляет до 24 часов. У Prometheus также задан предел TSDB 512 МБ.
|
||||
|
||||
Сохранён pipeline ЛР 5. HAProxy распределяет запросы между двумя API,
|
||||
сессии хранятся в Redis, PostgreSQL содержит сообщения, метаданные вложений,
|
||||
задания и outbox, RustFS хранит объекты, RabbitMQ доставляет задания. Загрузка
|
||||
пишет staging в S3 и в одной DB-транзакции сохраняет вложение, задание и
|
||||
outbox. Relay публикует сообщение c broker confirmation. Worker захватывает
|
||||
задание по lease, пишет оригинал и варианты в S3, завершает состояние БД и
|
||||
только потом подтверждает доставку. Повторная выдача сохраняет прежнюю
|
||||
семантику lease, retry и готовности вложения.
|
||||
|
||||
`traceparent` и необязательный `tracestate` попадают в первоначальный outbox
|
||||
вместе с заданием. Relay извлекает W3C context, создаёт `rabbitmq.publish`
|
||||
типа Producer и инъецирует его W3C context в публикуемый JSON. Worker
|
||||
извлекает этот контекст для `attachment.process` типа Consumer. Повторный
|
||||
outbox сохраняет контекст текущего consumer. Следующий producer и consumer
|
||||
продолжают то же дерево. Дочерние операции S3 и PostgreSQL видны до
|
||||
`CompleteJob` и `commit`. Лог запроса содержит сгенерированный сервером
|
||||
`X-Request-Id`, trace ID и span ID, лог worker содержит технический job ID
|
||||
и IDs активного span. Код экспортирует только явно заданные поля, включая
|
||||
имя операции sqlc без SQL и значений параметров. Секреты, тела запросов,
|
||||
файлы и бизнес-идентификаторы в labels исключены.
|
||||
|
||||
Использованы Go OpenTelemetry SDK 1.38.0 и SDK логов 0.14.0. В resource
|
||||
указаны `service.name` (`messenger-api`, `messenger-relay`,
|
||||
`messenger-worker`), `service.version`, `service.instance.id` и
|
||||
`deployment.environment.name=lab`. HTTP и DB атрибуты следуют именованию
|
||||
OTel semantic conventions (`http.request.method`, `http.route`,
|
||||
`http.response.status_code`, `db.system.name`, `db.operation.name`);
|
||||
инструменты `messenger_*` заданы приложением. Ссылки на первичные материалы
|
||||
доступны в [задании](README.md#первичные-материалы). Использованы также
|
||||
[Go SDK](https://opentelemetry.io/docs/languages/go/instrumentation/) и
|
||||
[Loki OTLP](https://grafana.com/docs/loki/latest/send-data/otel/).
|
||||
|
||||
## Сигналы и условие SLO
|
||||
|
||||
| Сигнал | Prometheus и расчёт | Единица |
|
||||
| --- | --- | --- |
|
||||
| API RPS и ошибки | `messenger_http_requests_total`; `sum(rate(...[1m]))` по репликам, ошибки фильтруются `status_class="5xx"` | запросы/с |
|
||||
| HTTP latency | `histogram_quantile(0.50\|0.95\|0.99, sum by (le) (rate(messenger_http_request_duration_seconds_bucket[1m])))` с исключением health | с |
|
||||
| Доля ошибок | 5xx / все бизнес-ответы API за то же окно; health исключается из обеих частей | доля |
|
||||
| Исход и длительность работы | `messenger_jobs_total` и `messenger_job_duration_seconds_bucket` с `job_type`, `outcome` | попытки/с, с |
|
||||
| Глубина и возраст очереди | `messenger_queue_depth` и `messenger_queue_oldest_age_seconds` по `queue="image.jobs"` | задания, с |
|
||||
| Пул БД | `messenger_db_pool_connections{state="used"\|"capacity"}`, `messenger_db_pool_waiting` | соединения, активные ожидания acquire |
|
||||
| Занятость worker | `sum(messenger_worker_busy) / sum(messenger_worker_capacity)` | доля |
|
||||
| CPU и память процесса | `rate(messenger_process_cpu_time_seconds_total[1m])`, `messenger_process_memory_bytes` (RSS) | ядра, байты |
|
||||
| Время до ready | `messenger_attachment_time_to_ready_seconds_bucket` от записи задания перед durable commit до commit результата | с |
|
||||
|
||||
Очередь измеряется SQL-снимком текущих jobs со статусом `queued` раз в две
|
||||
секунды. В глубину входят задания, ещё ожидающие публикации через outbox, и
|
||||
отложенные retry. Возраст считается с первого принятия соответствующей
|
||||
версии задания. Задания, которые worker уже захватил, учитываются в
|
||||
`messenger_worker_busy`. `messenger_queue_sample_age_seconds` защищает
|
||||
алерт от устаревшего снимка при недоступной БД.
|
||||
|
||||
**SLI доступности API** — доля ответов без 5xx среди всех завершённых
|
||||
бизнес-запросов к API. Считаются ответы всех реплик, кроме `/health/live`
|
||||
и `/health/ready`. Возвращённые приложением 4xx входят в знаменатель.
|
||||
**SLO** — не менее 99% в скользящем окне 5 минут при наличии запросов.
|
||||
В окне без трафика SLI не вычисляется. PromQL для проверки:
|
||||
|
||||
```promql
|
||||
1 - (
|
||||
sum(increase(messenger_http_requests_total{route!~"/health/(live|ready)",status_class="5xx"}[5m]))
|
||||
or 0 * sum(increase(messenger_http_requests_total{route!~"/health/(live|ready)"}[5m]))
|
||||
) / (sum(increase(messenger_http_requests_total{route!~"/health/(live|ready)"}[5m])) > 0)
|
||||
```
|
||||
|
||||
Prometheus проверяет правила каждые 5 секунд. `HighAPIErrorRate` срабатывает
|
||||
при 5xx >10% в окне 30 секунд и достаточном трафике (оценка ≥5 запросов),
|
||||
если условие держится 15 секунд. `GrowingQueueLag` требует непустой
|
||||
очереди и свежего снимка с возрастом старейшей задачи >15 секунд в течение
|
||||
15 секунд. Оба алерта используют короткие локальные пороги для учебных
|
||||
инцидентов. Команды и последовательность диагностики описаны в
|
||||
[runbook](infra/otel/runbook.md).
|
||||
|
||||
## Сбои контура телеметрии
|
||||
|
||||
У логов и трасс отдельные async очереди в Go SDK на 512 записей/спанов.
|
||||
Экспорт имеет таймаут 1,5 с и отключённые SDK retry, период метрик 2 с.
|
||||
Collector использует memory limiter 96 МиБ (spike 24 МиБ, проверка раз в
|
||||
секунду), сборку пачек до 128 элементов и отдельные очереди Loki/Tempo по
|
||||
16 пачек с одним consumer на экспортёр. Таймаут попытки передачи 2 с, лимит
|
||||
retry 5 с. Контейнер Collector ограничен 256 МиБ. При длительном отказе
|
||||
экспортёр отбрасывает данные. Метрики send/enqueue failures показывают,
|
||||
сколько записей потеряно на стороне Collector. Пайплайн работы с
|
||||
сообщениями и вложениями не ждёт экспорта. Телеметрия не участвует в
|
||||
readiness API.
|
||||
|
||||
## Доказательства критериев
|
||||
|
||||
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
|
||||
| --- | --- | --- | --- |
|
||||
| Публичный smoke | make test | TODO | TODO |
|
||||
| Остальные пункты уровня 3 | TODO | TODO | TODO |
|
||||
| Все пункты уровня 4, если заявлен | TODO | TODO | TODO |
|
||||
| Выбранный трек 5, если заявлен | TODO | TODO | TODO |
|
||||
Проверка 16 ноября 2026 года на Apple M2 с 8 ГБ RAM. Docker Engine
|
||||
28.4.0, Compose 2.39.4, arm64, Python 3.14.6. Ограничения контейнеров API
|
||||
составляют 2 CPU и 512 МиБ на реплику, worker 2 CPU и 1536 МиБ с одним
|
||||
обработчиком. Тестовик использовал отдельный Compose-проект
|
||||
`highload-lab6-student` и порт API 28106. Запуск и полный список сценариев
|
||||
описаны в [приёмке](student-tests/lab6/README.md). Все девять проверок
|
||||
прошли. [Сводка](evidence/lab6-grade4/summary.json) указывает, в каком
|
||||
[JSONL-прогоне](evidence/lab6-grade4/README.md) наблюдался каждый результат.
|
||||
|
||||
TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены.
|
||||
### Инциденты
|
||||
|
||||
## Эксперименты и отказы
|
||||
| Воздействие | Что показали сохранённые сигналы | Восстановление |
|
||||
| --- | --- | --- |
|
||||
| Остановлен worker | Три загрузки получили 202. Prometheus наблюдал глубину 3 и возраст 18,59 с на первом замере выше порога. `GrowingQueueLag` стал `firing` через 39,82 с после начала загрузок. | Все 3 вложения стали `ready`, глубина 0, повторное чтение подтвердило SHA-256. Через 9,04 с после старта worker правило стало `inactive`. |
|
||||
| В PostgreSQL удержан `ACCESS EXCLUSIVE` на `public.users` | `/health/ready` сохранил 200. 23 запроса `/api/v1/users/me` вернули 503 за 1,007–1,065 с. В Tempo дочерний `postgres.query` завершился ошибкой за 1,001 с, Loki показал 503 с тем же ID запроса. `HighAPIErrorRate` стал `firing` через 32,73 с. | После снятия блокировки запросы вернули 200. Через 23,81 с правило стало `inactive`. |
|
||||
| Остановлен Loki | За 34 с API успешно ответил на 278 чтений и подтвердил 8 сообщений. Очередь достигла предела 16 пачек, RSS Collector за время отказа достиг 88,6 МиБ из 256 МиБ. Счётчик потерь вырос на 96 записей, первый рост обнаружен через 8,69 с. | Все восемь сообщений прочитаны без изменений. Collector сохранил тот же экземпляр без рестарта и OOM. Через 16,94 с после возобновления Loki тест нашёл новую запись по trace ID и ID запроса. |
|
||||
|
||||
TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения.
|
||||
Интервалы отсчитываются от действия теста до наблюдения через API
|
||||
Prometheus. Они включают дискретность scrape и вычисления правил. В
|
||||
первых двух сценариях поток создавали три загрузки либо последовательные
|
||||
запросы чтения. В третьем проверялись успешные записи, непрерывные
|
||||
чтения и ограничение очереди Collector. Это измерения конкретного стенда
|
||||
на синтетических данных, без оценки предельного RPS или устойчивости
|
||||
при длительной production-нагрузке. Во время холодного старта второго
|
||||
Compose-проекта в одном пробном запуске Loki-сценария появился 503 с
|
||||
длительностью около секунды. Итоговый эксперимент выполнен на прогретом
|
||||
тестовом проекте после остановки второго стенда.
|
||||
|
||||
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
|
||||
| --- | --- | --- | --- |
|
||||
| TODO | TODO | TODO | TODO |
|
||||
### Матрица проверки
|
||||
|
||||
## Ограничения и следующие шаги
|
||||
| Критерий | Проверка | Наблюдаемый результат |
|
||||
| --- | --- | --- |
|
||||
| API ЛР 5 с включённой телеметрией | `make check`, `make test` | Публичный smoke прошёл с двумя API за HAProxy, мигратором, RabbitMQ, worker и S3. |
|
||||
| Запуск и provisioning | `backend-readiness`, `provisioning` | Три backend готовы, dashboard и три datasource загружены автоматически. `promtool` принял оба правила; Collector принял конфигурацию. |
|
||||
| API → broker → worker → S3/DB, включая retry | `trace-retry` | После временного разрыва доступа worker к S3 первая попытка сохранила `s3_unavailable`, вторая дошла до `ready`. В Tempo найдено одно дерево с 1 Server, 2 Producer и 2 Consumer spans. Ошибка первого `s3.get` и успешные S3/DB операции второй попытки дочерние к соответствующим Consumer. В обоих outbox payload тот же trace ID и `tracestate`. |
|
||||
| Структурные логи и отсутствие чувствительных значений | `trace-retry`, проверки Loki и Tempo | Логи API и worker сопоставлены со spans, запись API найдена по `X-Request-Id`. Проверены синтетические пароль, Basic credential, Bearer token, текст сообщения, имя файла и фрагмент изображения. |
|
||||
| Метрики и низкая кардинальность | `metrics` | Восемь разных чатов дали общий route template. В Prometheus обнаружены HTTP/job/ready buckets, pool, worker, очередь и ресурсы процесса. Проверка сравнила `+Inf` с `_count` и вычислила все три quantile из bucket. |
|
||||
| Рост очереди и восстановление | `worker-outage` | Три ready из трёх, глубина до 3 и обратно до 0, зафиксированы `firing` и `inactive`. |
|
||||
| Ошибки и восстановление PostgreSQL | `db-outage` | Сопоставили 503, лог и дочерний DB span. Зафиксированы `firing` и `inactive`. |
|
||||
| Недоступность telemetry storage | `loki-outage` | Восемь из восьми сообщений сохранены, измерены потери в потоке логов, очередь и RSS в пределах лимитов. |
|
||||
|
||||
TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза.
|
||||
Для сравнения квантили HTTP в одном тестовом снимке составили около
|
||||
2,2 мс, 209 мс и 1,0 с (p50/p95/p99). Значения вычисляются интерполяцией
|
||||
по бакетам. Этот снимок включает тестовые запросы и время прогрева стенда.
|
||||
|
||||
После пересоздания контейнеров Prometheus, Loki, Tempo и Grafana на прежних томах повторно прочитаны исторический отсчёт метрики, запись по request ID и тот же server span. Grafana загрузила dashboard. [Результат проверки сохранности](evidence/telemetry-persistence.json).
|
||||
|
||||
18 ноября стенд повторно собран и запущен из ветки решения. Публичный
|
||||
`make check test BASE_URL=http://127.0.0.1:8086`, `go test ./...`,
|
||||
`go vet ./...` и `sqlc compile` завершились успешно. Grafana вернула
|
||||
HTTP 200 для provisioned dashboard `lab6-messenger`.
|
||||
|
||||
+157
@@ -0,0 +1,157 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"highload/internal/config"
|
||||
"highload/internal/database"
|
||||
"highload/internal/database/sqlc"
|
||||
"highload/internal/observability"
|
||||
"highload/internal/repository/postgres"
|
||||
"highload/internal/repository/redis"
|
||||
"highload/internal/service"
|
||||
"highload/internal/storage/s3"
|
||||
httpdeliver "highload/internal/transport/http"
|
||||
"highload/internal/transport/http/handler"
|
||||
"highload/internal/transport/http/middleware"
|
||||
"log/slog"
|
||||
"net"
|
||||
"net/http"
|
||||
"os"
|
||||
"os/signal"
|
||||
"syscall"
|
||||
"time"
|
||||
)
|
||||
|
||||
func main() {
|
||||
|
||||
cfg, err := config.Load()
|
||||
|
||||
if err != nil {
|
||||
slog.Error("invalid configuration", "error", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
sessionStore := cfg.Session.Store
|
||||
|
||||
if sessionStore != "redis" {
|
||||
slog.Error("session store is "+sessionStore+"must redis", "error", nil)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
defer observability.Start(context.Background(), "messenger-api")()
|
||||
databaseURL := cfg.Database.PostgresURL
|
||||
pool, err := database.NewPostgres(context.Background(), databaseURL)
|
||||
|
||||
if err != nil {
|
||||
slog.Error("postgres error", "error", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
defer pool.Close()
|
||||
if os.Getenv("PG_POOL_DIAGNOSTICS") == "1" {
|
||||
diagnosticsCtx, stopDiagnostics := context.WithCancel(context.Background())
|
||||
defer stopDiagnostics()
|
||||
go logPoolStats(diagnosticsCtx, pool)
|
||||
}
|
||||
|
||||
queries := sqlc.New(pool)
|
||||
|
||||
redisAddr := cfg.Database.RedisAddr
|
||||
|
||||
redisClient, err := database.NewRedis(context.Background(), redisAddr)
|
||||
|
||||
if err != nil {
|
||||
slog.Error("redis error", "error", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
defer redisClient.Close()
|
||||
|
||||
instanceID, err := os.Hostname()
|
||||
if err != nil || instanceID == "" {
|
||||
slog.Error("cannot get instance ID", "error", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
s3Storage, err := s3.NewStorage(context.Background(), &cfg.S3)
|
||||
|
||||
if err != nil {
|
||||
slog.Error("cannot create new storage", "error", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
userRepo := postgres.NewUserRepository(queries)
|
||||
chatRepo := postgres.NewChatRepository(queries, pool)
|
||||
messageRepo := postgres.NewMessageRepository(queries, pool)
|
||||
sessionRepo := redis.NewSessionRepository(redisClient)
|
||||
attachmentRepo := postgres.NewAttachmentRepository(queries, pool, int32(cfg.JobMaxAttempts))
|
||||
|
||||
authService := service.NewAuthService(userRepo, sessionRepo, cfg.Session.TTL)
|
||||
userService := service.NewUserService(userRepo)
|
||||
chatService := service.NewChatService(chatRepo, userRepo, userService)
|
||||
messageService := service.NewMessageService(messageRepo, chatService, userService)
|
||||
attachmentService := service.NewAttachmentService(attachmentRepo, chatService, s3Storage)
|
||||
|
||||
healthHandler := handler.NewHealthHandler(pool, redisClient, s3Storage)
|
||||
authHandler := handler.NewAuthHandler(authService)
|
||||
userHandler := handler.NewUserHandler(userService)
|
||||
chatHandler := handler.NewChatHandler(chatService)
|
||||
messageHandler := handler.NewMessageHandler(messageService)
|
||||
attachmentHandler := handler.NewAttachmentHandler(attachmentService)
|
||||
|
||||
mid := middleware.NewMiddleware(authService, instanceID)
|
||||
router := httpdeliver.NewRouter(mid, authHandler, healthHandler, chatHandler, userHandler, messageHandler, attachmentHandler)
|
||||
|
||||
server := &http.Server{
|
||||
Addr: net.JoinHostPort(cfg.HTTP.Host, cfg.HTTP.Port),
|
||||
Handler: router.Handler(),
|
||||
ReadHeaderTimeout: 5 * time.Second,
|
||||
ReadTimeout: 10 * time.Second,
|
||||
WriteTimeout: 10 * time.Second,
|
||||
IdleTimeout: 30 * time.Second,
|
||||
}
|
||||
|
||||
listener, err := net.Listen("tcp", server.Addr)
|
||||
if err != nil {
|
||||
slog.Error("server failed to start", "error", err)
|
||||
return
|
||||
}
|
||||
|
||||
slog.Info("server started", "addr", server.Addr)
|
||||
|
||||
serverErr := make(chan error, 1)
|
||||
go func() {
|
||||
err := server.Serve(listener)
|
||||
if err != nil && !errors.Is(err, http.ErrServerClosed) {
|
||||
serverErr <- err
|
||||
slog.Error("server failed", "error", err)
|
||||
}
|
||||
}()
|
||||
|
||||
ctx, stop := signal.NotifyContext(
|
||||
context.Background(),
|
||||
os.Interrupt,
|
||||
syscall.SIGTERM,
|
||||
)
|
||||
defer stop()
|
||||
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
slog.Info("shutdown signal received")
|
||||
|
||||
case err := <-serverErr:
|
||||
slog.Error("server failed", "error", err)
|
||||
return
|
||||
}
|
||||
|
||||
shutdownCtx, cancel := context.WithTimeout(
|
||||
context.Background(),
|
||||
15*time.Second,
|
||||
)
|
||||
defer cancel()
|
||||
|
||||
if err := server.Shutdown(shutdownCtx); err != nil {
|
||||
slog.Error("shutdown failed", "error", err)
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,29 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log/slog"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
)
|
||||
|
||||
// logPoolStats is opt-in diagnostics for the isolated scaling experiment.
|
||||
// It contains only aggregate counters, never connection strings or SQL values.
|
||||
func logPoolStats(ctx context.Context, pool *pgxpool.Pool) {
|
||||
ticker := time.NewTicker(time.Second)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-ticker.C:
|
||||
s := pool.Stat()
|
||||
slog.Info("pg_pool_stats", "max_conns", s.MaxConns(),
|
||||
"acquired", s.AcquiredConns(), "idle", s.IdleConns(),
|
||||
"acquire_count", s.AcquireCount(), "empty_count", s.EmptyAcquireCount(),
|
||||
"empty_wait_ns", s.EmptyAcquireWaitTime().Nanoseconds(),
|
||||
"canceled_count", s.CanceledAcquireCount())
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,164 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"highload/internal/config"
|
||||
"highload/internal/database"
|
||||
"highload/internal/database/sqlc"
|
||||
"highload/internal/repository/postgres"
|
||||
"highload/internal/service"
|
||||
"highload/internal/storage/s3"
|
||||
"log/slog"
|
||||
"os"
|
||||
"os/signal"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5/pgconn"
|
||||
)
|
||||
|
||||
type cleanupConfig struct {
|
||||
once bool
|
||||
dryRun bool
|
||||
gracePeriod time.Duration
|
||||
interval time.Duration
|
||||
passTimeout time.Duration
|
||||
reconcile bool
|
||||
restoreID string
|
||||
backupFile string
|
||||
}
|
||||
|
||||
func main() {
|
||||
var cfg cleanupConfig
|
||||
|
||||
flag.BoolVar(&cfg.once, "once", false, "Run once and exit")
|
||||
flag.BoolVar(&cfg.dryRun, "dry-run", true, "Do not actually delete files")
|
||||
flag.DurationVar(&cfg.gracePeriod, "grace-period", 10*time.Minute, "Grace period for old files")
|
||||
flag.DurationVar(&cfg.interval, "interval", 5*time.Minute, "Interval between runs")
|
||||
flag.DurationVar(&cfg.passTimeout, "pass-timeout", 2*time.Minute, "Timeout for a single pass")
|
||||
flag.BoolVar(&cfg.reconcile, "reconcile", false, "Scan both stores once and print JSON observations")
|
||||
flag.StringVar(&cfg.restoreID, "restore-id", "", "Restore the missing original for this attachment ID")
|
||||
flag.StringVar(&cfg.backupFile, "backup-file", "", "Path to the original backup used for restore")
|
||||
flag.Parse()
|
||||
|
||||
if cfg.gracePeriod <= 0 || cfg.interval <= 0 || cfg.passTimeout <= 0 {
|
||||
slog.Error("all duration flags must be positive")
|
||||
os.Exit(1)
|
||||
}
|
||||
if (cfg.restoreID == "") != (cfg.backupFile == "") || (cfg.reconcile && cfg.restoreID != "") {
|
||||
slog.Error("use -reconcile alone, or supply both -restore-id and -backup-file")
|
||||
os.Exit(1)
|
||||
}
|
||||
if err := run(cfg); err != nil {
|
||||
slog.Error("attachment cleanup failed", "error", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
func run(cfg cleanupConfig) error {
|
||||
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
|
||||
defer stop()
|
||||
|
||||
databaseURL, s3Config, err := config.LoadAttachmentCleanup()
|
||||
if err != nil {
|
||||
return fmt.Errorf("load cleanup configuration: %w", err)
|
||||
}
|
||||
|
||||
startupCtx, cancelStartup := context.WithTimeout(ctx, 10*time.Second)
|
||||
defer cancelStartup()
|
||||
|
||||
pool, err := database.NewPostgres(startupCtx, databaseURL)
|
||||
if err != nil {
|
||||
var configErr *pgconn.ParseConfigError
|
||||
if errors.As(err, &configErr) {
|
||||
return errors.New("invalid PostgreSQL connection string")
|
||||
}
|
||||
return fmt.Errorf("connect to PostgreSQL: %w", err)
|
||||
}
|
||||
defer pool.Close()
|
||||
|
||||
s3Storage, err := s3.NewStorage(startupCtx, &s3Config)
|
||||
if err != nil {
|
||||
return fmt.Errorf("create S3 client: %w", err)
|
||||
}
|
||||
cancelStartup()
|
||||
|
||||
attachmentRepo := postgres.NewAttachmentRepository(sqlc.New(pool), pool)
|
||||
options := &service.AttachmentCleanupOptions{
|
||||
GracePeriod: cfg.gracePeriod,
|
||||
DryRun: cfg.dryRun,
|
||||
ListTimeout: 5 * time.Second,
|
||||
}
|
||||
attachmentCleanupService := service.NewAttachmentCleanupService(attachmentRepo, s3Storage, options)
|
||||
|
||||
if cfg.reconcile || cfg.restoreID != "" {
|
||||
passCtx, cancel := context.WithTimeout(ctx, cfg.passTimeout)
|
||||
defer cancel()
|
||||
reconciler := service.NewAttachmentReconciler(attachmentRepo, s3Storage, *options)
|
||||
start := time.Now()
|
||||
report := struct {
|
||||
DryRun bool `json:"dry_run"`
|
||||
DurationMS int64 `json:"duration_ms"`
|
||||
Complete bool `json:"complete"`
|
||||
Orphans *service.OrphanReport `json:"orphan_scan,omitempty"`
|
||||
Missing *service.MissingReport `json:"missing_scan,omitempty"`
|
||||
Restore string `json:"restore,omitempty"`
|
||||
}{DryRun: cfg.dryRun}
|
||||
var actionErr error
|
||||
if cfg.reconcile {
|
||||
missing, err := reconciler.ScanMissing(passCtx)
|
||||
report.Missing = &missing
|
||||
actionErr = err
|
||||
if err == nil {
|
||||
orphans, err := attachmentCleanupService.ScanOrphans(passCtx)
|
||||
report.Orphans = &orphans
|
||||
actionErr = err
|
||||
}
|
||||
} else {
|
||||
file, err := os.Open(cfg.backupFile)
|
||||
if err != nil {
|
||||
return fmt.Errorf("open backup original: %w", err)
|
||||
}
|
||||
defer file.Close()
|
||||
report.Restore, actionErr = reconciler.Restore(passCtx, cfg.restoreID, file)
|
||||
}
|
||||
report.DurationMS = time.Since(start).Milliseconds()
|
||||
report.Complete = actionErr == nil
|
||||
return errors.Join(actionErr, json.NewEncoder(os.Stdout).Encode(report))
|
||||
}
|
||||
|
||||
runPass := func() error {
|
||||
passCtx, cancelPass := context.WithTimeout(ctx, cfg.passTimeout)
|
||||
defer cancelPass()
|
||||
return attachmentCleanupService.RunOnce(passCtx)
|
||||
}
|
||||
|
||||
if cfg.once {
|
||||
return runPass()
|
||||
}
|
||||
|
||||
if err := runPass(); err != nil && ctx.Err() == nil {
|
||||
slog.Error("attachment cleanup pass failed", "error", err)
|
||||
}
|
||||
|
||||
ticker := time.NewTicker(cfg.interval)
|
||||
defer ticker.Stop()
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return nil
|
||||
case <-ticker.C:
|
||||
if ctx.Err() != nil {
|
||||
return nil
|
||||
}
|
||||
if err := runPass(); err != nil && ctx.Err() == nil {
|
||||
slog.Error("attachment cleanup pass failed", "error", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,133 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"flag"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"os"
|
||||
"os/signal"
|
||||
"strings"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"highload/internal/config"
|
||||
"highload/internal/database"
|
||||
"highload/internal/database/sqlc"
|
||||
"highload/internal/pipeline"
|
||||
"highload/internal/repository/postgres"
|
||||
"highload/internal/storage/s3"
|
||||
)
|
||||
|
||||
func main() {
|
||||
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
|
||||
defer stop()
|
||||
if err := run(ctx, os.Args[1:]); err != nil {
|
||||
slog.Error("attachment jobs command failed", "error", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
func run(ctx context.Context, args []string) error {
|
||||
const usage = "usage: jobs failed | replay | upgrade | cleanup"
|
||||
if len(args) == 0 {
|
||||
return fmt.Errorf("%s", usage)
|
||||
}
|
||||
flags := flag.NewFlagSet(args[0], flag.ContinueOnError)
|
||||
job := flags.String("job-id", "", "failed job ID")
|
||||
request := flags.String("request-id", "", "unique operator request UUID")
|
||||
operator := flags.String("operator", "", "operator ID")
|
||||
reason := flags.String("reason", "", "replay reason")
|
||||
extra := flags.Int("extra-attempts", 1, "budget for replay (1–5)")
|
||||
attachment := flags.String("attachment-id", "", "attachment ID to upgrade")
|
||||
version := flags.Int("to-version", 0, "target pipeline version (2)")
|
||||
dry := flags.Bool("dry-run", false, "report eligible objects without deleting")
|
||||
once := flags.Bool("once", false, "execute one cleanup pass")
|
||||
if err := flags.Parse(args[1:]); err != nil {
|
||||
return err
|
||||
}
|
||||
command := args[0]
|
||||
if command != "failed" && command != "cleanup" && command != "replay" && command != "upgrade" {
|
||||
return fmt.Errorf("%s", usage)
|
||||
}
|
||||
if command == "replay" && (*job == "" || *request == "" || strings.TrimSpace(*operator) == "" ||
|
||||
strings.TrimSpace(*reason) == "" || *extra < 1 || *extra > 5) {
|
||||
return fmt.Errorf("replay needs job-id, request-id, operator, reason and 1–5 extra-attempts")
|
||||
}
|
||||
if command == "upgrade" && (*attachment == "" || *version != 2) {
|
||||
return fmt.Errorf("upgrade needs attachment-id and to-version=2")
|
||||
}
|
||||
attemptLimit := 5
|
||||
if command == "upgrade" {
|
||||
var err error
|
||||
attemptLimit, err = config.EnvInt("JOB_MAX_ATTEMPTS", 5, 1, 20)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
url, storageConfig, err := config.LoadAttachmentCleanup()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
pool, err := database.NewPostgres(ctx, url)
|
||||
if err != nil {
|
||||
return fmt.Errorf("PostgreSQL unavailable")
|
||||
}
|
||||
defer pool.Close()
|
||||
repo := postgres.NewAttachmentRepository(sqlc.New(pool), pool, int32(attemptLimit))
|
||||
if command == "failed" {
|
||||
dbCtx, cancel := context.WithTimeout(ctx, 5*time.Second)
|
||||
defer cancel()
|
||||
rows, err := repo.ListFailedJobs(dbCtx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
type failed struct {
|
||||
JobID string `json:"job_id"`
|
||||
AttachmentID string `json:"attachment_id"`
|
||||
ErrorCode string `json:"error_code"`
|
||||
Attempts int32 `json:"attempts"`
|
||||
UpdatedAt time.Time `json:"updated_at"`
|
||||
}
|
||||
result := make([]failed, 0, len(rows))
|
||||
for _, row := range rows {
|
||||
result = append(result, failed{row.ID.String(), row.AttachmentID.String(), row.ErrorCode.String, row.AttemptsStarted, row.UpdatedAt.Time})
|
||||
}
|
||||
return json.NewEncoder(os.Stdout).Encode(result)
|
||||
}
|
||||
objects, err := s3.NewStorage(ctx, &storageConfig)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if command == "replay" {
|
||||
changed, err := repo.ReplayFailed(ctx, *job, *request, *operator, *reason, int32(*extra), objects.Head)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return json.NewEncoder(os.Stdout).Encode(map[string]any{"job_id": *job, "replayed": changed})
|
||||
}
|
||||
if command == "upgrade" {
|
||||
jobID, queued, err := repo.UpgradePipeline(ctx, *attachment, objects.Head)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return json.NewEncoder(os.Stdout).Encode(map[string]any{
|
||||
"attachment_id": *attachment, "pipeline_version": *version, "job_id": jobID, "queued": queued,
|
||||
})
|
||||
}
|
||||
cfg, err := config.LoadPipeline()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
cleanup := pipeline.Cleanup{Repo: repo, Objects: objects, Grace: cfg.CleanupGrace, Retention: cfg.Retention, DryRun: *dry}
|
||||
if *once {
|
||||
count, err := cleanup.RunOnce(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return json.NewEncoder(os.Stdout).Encode(map[string]any{"objects": count, "dry_run": *dry})
|
||||
}
|
||||
cleanup.Serve(ctx, cfg.CleanupPeriod)
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,254 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"io"
|
||||
"log/slog"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"highload/db/migrations"
|
||||
"highload/internal/database/sqlc"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
"github.com/jackc/pgx/v5/pgtype"
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
"github.com/pressly/goose/v3"
|
||||
)
|
||||
|
||||
func TestAsyncAttachmentsMigrationAndJobTransitions(t *testing.T) {
|
||||
db, ctx := migrationPostgresFixture(t)
|
||||
provider, err := goose.NewProvider(goose.DialectPostgres, db, migrations.Files,
|
||||
goose.WithDisableGlobalRegistry(true), goose.WithSlog(slog.New(slog.NewTextHandler(io.Discard, nil))))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := provider.UpTo(ctx, 6); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
const original = "attachments/step-two-original"
|
||||
_, err = db.ExecContext(ctx, `
|
||||
INSERT INTO users (id, username, display_name, password_hash) VALUES
|
||||
('00000000-0000-4000-8000-000000000001', 'step_two', 'Step two', 'synthetic');
|
||||
INSERT INTO chats (id, title) VALUES ('00000000-0000-4000-8000-000000000002', 'Step two');
|
||||
INSERT INTO chat_members VALUES
|
||||
('00000000-0000-4000-8000-000000000002', '00000000-0000-4000-8000-000000000001');
|
||||
INSERT INTO messages (id, chat_id, sender_id, text) VALUES
|
||||
('00000000-0000-4000-8000-000000000003', '00000000-0000-4000-8000-000000000002',
|
||||
'00000000-0000-4000-8000-000000000001', 'Old message');
|
||||
INSERT INTO attachments (id, chat_id, owner_id, filename, object_key, content_type, size_bytes, sha256) VALUES
|
||||
('00000000-0000-4000-8000-000000000004', '00000000-0000-4000-8000-000000000002',
|
||||
'00000000-0000-4000-8000-000000000001', 'old.png', 'attachments/step-two-original',
|
||||
'image/png', 123, repeat('a', 64));
|
||||
INSERT INTO messages_attachments (message_id, attachment_id, chat_id, position) VALUES
|
||||
('00000000-0000-4000-8000-000000000003', '00000000-0000-4000-8000-000000000004',
|
||||
'00000000-0000-4000-8000-000000000002', 0);
|
||||
`)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := provider.UpTo(ctx, 7); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := provider.Down(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var preservedKey string
|
||||
if err := db.QueryRowContext(ctx, "SELECT object_key FROM attachments JOIN messages_attachments ON attachment_id = id").Scan(&preservedKey); err != nil || preservedKey != original {
|
||||
t.Fatalf("rollback lost the legacy attachment reference: %v", err)
|
||||
}
|
||||
if _, err := provider.UpTo(ctx, 7); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := provider.UpTo(ctx, lab5MigrationCount); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
dsn, err := os.ReadFile(os.Getenv("DATABASE_URL_FILE"))
|
||||
if err != nil {
|
||||
t.Fatal("cannot read test connection configuration")
|
||||
}
|
||||
pool, err := pgxpool.New(ctx, string(dsn))
|
||||
if err != nil {
|
||||
t.Fatal("cannot configure test pool")
|
||||
}
|
||||
defer pool.Close()
|
||||
q := sqlc.New(pool)
|
||||
|
||||
var jobID, attachmentID pgtype.UUID
|
||||
var payload []byte
|
||||
err = pool.QueryRow(ctx, `
|
||||
SELECT j.id, a.id, o.payload
|
||||
FROM messages_attachments AS m
|
||||
JOIN attachments AS a ON a.id = m.attachment_id
|
||||
JOIN attachment_jobs AS j ON j.attachment_id = a.id
|
||||
JOIN outbox AS o ON o.job_id = j.id
|
||||
WHERE a.status = 'queued' AND a.object_key = $1 AND a.staging_key = $1
|
||||
AND o.dedup_key = 'initial/' || j.id
|
||||
`, original).Scan(&jobID, &attachmentID, &payload)
|
||||
if err != nil {
|
||||
t.Fatalf("legacy attachment was not queued with its initial event: %v", err)
|
||||
}
|
||||
var event struct {
|
||||
JobID string `json:"job_id"`
|
||||
Attempt int `json:"attempt"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
}
|
||||
if err := json.Unmarshal(payload, &event); err != nil || event.JobID != jobID.String() ||
|
||||
event.Attempt != 1 || event.CreatedAt.IsZero() {
|
||||
t.Fatalf("invalid backfill payload: %v", err)
|
||||
}
|
||||
|
||||
start := make(chan struct{})
|
||||
results := make(chan error, 2)
|
||||
claim := sqlc.ClaimJobParams{
|
||||
JobID: jobID, NewLeaseUntil: pgtype.Timestamptz{Time: time.Now().Add(time.Minute), Valid: true},
|
||||
}
|
||||
for range 2 {
|
||||
go func() {
|
||||
<-start
|
||||
_, err := q.ClaimJob(ctx, claim)
|
||||
results <- err
|
||||
}()
|
||||
}
|
||||
close(start)
|
||||
winners := 0
|
||||
for range 2 {
|
||||
switch err := <-results; {
|
||||
case err == nil:
|
||||
winners++
|
||||
case errors.Is(err, pgx.ErrNoRows):
|
||||
default:
|
||||
t.Fatalf("concurrent claim failed: %v", err)
|
||||
}
|
||||
}
|
||||
if winners != 1 {
|
||||
t.Fatalf("gave the lease to %d workers", winners)
|
||||
}
|
||||
|
||||
exec := func(query string, args ...any) {
|
||||
t.Helper()
|
||||
if _, err := pool.Exec(ctx, query, args...); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
exec("UPDATE attachment_jobs SET lease_until = clock_timestamp() - interval '1 second' WHERE id = $1", jobID)
|
||||
for _, want := range []int{1, 0} {
|
||||
recovery, err := q.EnqueueExpiredLeases(ctx, 10)
|
||||
if err != nil || len(recovery) != want {
|
||||
t.Fatalf("recovery events = %d, want %d: %v", len(recovery), want, err)
|
||||
}
|
||||
if want == 1 {
|
||||
exec("UPDATE outbox SET sent_at = now() WHERE id = $1", recovery[0].ID)
|
||||
}
|
||||
}
|
||||
row, err := q.ClaimJob(ctx, claim)
|
||||
if err != nil || row.LeaseGeneration != 2 || row.AttemptsStarted != 2 {
|
||||
t.Fatalf("expired lease was not replaced correctly: %v", err)
|
||||
}
|
||||
fail := sqlc.FailJobParams{JobID: jobID, LeaseGeneration: 1, ErrorCode: "invalid_image"}
|
||||
if n, err := q.FailJob(ctx, fail); err != nil || n != 0 {
|
||||
t.Fatalf("stale worker could fail attachment: rows=%d, err=%v", n, err)
|
||||
}
|
||||
retry := sqlc.ScheduleRetryParams{
|
||||
JobID: jobID, LeaseGeneration: 2, ErrorCode: "s3_unavailable",
|
||||
AvailableAt: pgtype.Timestamptz{Time: time.Now().Add(time.Minute), Valid: true},
|
||||
}
|
||||
if n, err := q.ScheduleRetry(ctx, retry); err != nil || n != 1 {
|
||||
t.Fatalf("retry could not be scheduled: rows=%d, err=%v", n, err)
|
||||
}
|
||||
if _, err := q.ClaimJob(ctx, claim); !errors.Is(err, pgx.ErrNoRows) {
|
||||
t.Fatalf("worker bypassed the retry delay: %v", err)
|
||||
}
|
||||
exec("UPDATE attachment_jobs SET next_attempt_at = now(), max_attempts = 3 WHERE id = $1", jobID)
|
||||
row, err = q.ClaimJob(ctx, claim)
|
||||
if err != nil || row.LeaseGeneration != 3 {
|
||||
t.Fatalf("due retry could not claim job: %v", err)
|
||||
}
|
||||
exec("UPDATE attachment_jobs SET lease_until = clock_timestamp() - interval '1 second' WHERE id = $1", jobID)
|
||||
if _, err := q.ClaimJob(ctx, claim); !errors.Is(err, pgx.ErrNoRows) {
|
||||
t.Fatalf("worker bypassed the attempt limit: %v", err)
|
||||
}
|
||||
fail.LeaseGeneration = 3
|
||||
fail.ErrorCode = "retry_exhausted"
|
||||
if n, err := q.FailJob(ctx, fail); err != nil || n != 1 {
|
||||
t.Fatalf("exhausted job could not be failed: rows=%d, err=%v", n, err)
|
||||
}
|
||||
var requestID pgtype.UUID
|
||||
if err := requestID.Scan("00000000-0000-4000-8000-000000000005"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
replay := sqlc.ReplayFailedParams{
|
||||
JobID: jobID, RequestID: requestID, OperatorID: "operator", Reason: "test replay", ExtraAttempts: 1,
|
||||
}
|
||||
for _, want := range []int64{1, 0} {
|
||||
if n, err := q.ReplayFailed(ctx, replay); err != nil || n != want {
|
||||
t.Fatalf("replay changed %d rows, want %d: %v", n, want, err)
|
||||
}
|
||||
}
|
||||
row, err = q.ClaimJob(ctx, claim)
|
||||
if err != nil || row.AttemptsStarted != 4 || row.LeaseGeneration != 4 {
|
||||
t.Fatalf("replay did not preserve attempt and lease counters: %v", err)
|
||||
}
|
||||
|
||||
tx, err := pool.Begin(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer tx.Rollback(ctx)
|
||||
locked := q.WithTx(tx)
|
||||
if _, err := locked.LockActiveAttachmentJob(ctx, sqlc.LockActiveAttachmentJobParams{JobID: jobID, LeaseGeneration: 4}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
complete := sqlc.CompleteJobParams{JobID: jobID, LeaseGeneration: 4, ObjectKey: "results/original"}
|
||||
if n, err := locked.CompleteJob(ctx, complete); err != nil || n != 0 {
|
||||
t.Fatalf("published result without variants: rows=%d, err=%v", n, err)
|
||||
}
|
||||
variant := sqlc.CreateAttachmentVariantParams{
|
||||
AttachmentID: attachmentID, PipelineVersion: 1, Name: "thumbnail", ObjectKey: "results/thumb",
|
||||
ContentType: "image/jpeg", SizeBytes: 123, Width: 16, Height: 10, Sha256: strings.Repeat("a", 64),
|
||||
}
|
||||
if err := locked.CreateAttachmentVariant(ctx, variant); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if n, err := locked.CompleteJob(ctx, complete); err != nil || n != 0 {
|
||||
t.Fatalf("published result without crop and optimized: rows=%d, err=%v", n, err)
|
||||
}
|
||||
variant.Name, variant.ObjectKey = "crop", "results/crop"
|
||||
if err := locked.CreateAttachmentVariant(ctx, variant); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
variant.Name, variant.ObjectKey, variant.ContentType = "optimized", "results/webp", "image/webp"
|
||||
if err := locked.CreateAttachmentVariant(ctx, variant); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if n, err := locked.CompleteJob(ctx, complete); err != nil || n != 1 {
|
||||
t.Fatalf("current owner could not publish result: rows=%d, err=%v", n, err)
|
||||
}
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := q.ClaimJob(ctx, claim); !errors.Is(err, pgx.ErrNoRows) {
|
||||
t.Fatalf("worker claimed a ready job: %v", err)
|
||||
}
|
||||
published, err := q.GetPublishedOriginalKey(ctx, sqlc.GetPublishedOriginalKeyParams{
|
||||
AttachmentID: attachmentID, PipelineVersion: 1,
|
||||
})
|
||||
if err != nil || published.String != complete.ObjectKey {
|
||||
t.Fatalf("published original not recorded for its version: %v", err)
|
||||
}
|
||||
|
||||
var attempts, events, replays int
|
||||
err = pool.QueryRow(ctx, `
|
||||
SELECT (SELECT count(*) FROM attachment_attempts WHERE job_id = $1 AND finished_at IS NOT NULL),
|
||||
(SELECT count(*) FROM outbox WHERE job_id = $1),
|
||||
(SELECT count(*) FROM attachment_replays WHERE job_id = $1)
|
||||
`, jobID).Scan(&attempts, &events, &replays)
|
||||
if err != nil || attempts != 4 || events != 4 || replays != 1 {
|
||||
t.Fatalf("unexpected history: attempts=%d, events=%d, replays=%d: %v", attempts, events, replays, err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,152 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"errors"
|
||||
"flag"
|
||||
"log/slog"
|
||||
"os"
|
||||
"os/signal"
|
||||
"strconv"
|
||||
"strings"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"highload/db/migrations"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
"github.com/jackc/pgx/v5/pgconn"
|
||||
"github.com/jackc/pgx/v5/stdlib"
|
||||
"github.com/pressly/goose/v3"
|
||||
"github.com/pressly/goose/v3/lock"
|
||||
)
|
||||
|
||||
// Migration errors retain their cause for classification, but deliberately do
|
||||
// not print connection strings, PostgreSQL DETAIL fields or raw SQL.
|
||||
type migrationError struct {
|
||||
stage string
|
||||
cause error
|
||||
}
|
||||
|
||||
func (e *migrationError) Error() string { return e.stage }
|
||||
func (e *migrationError) Unwrap() error { return e.cause }
|
||||
|
||||
type boundedSessionLocker struct {
|
||||
lock.SessionLocker
|
||||
}
|
||||
|
||||
func (l boundedSessionLocker) SessionUnlock(ctx context.Context, conn *sql.Conn) error {
|
||||
// Goose unlocks with a detached context; bound this cleanup separately.
|
||||
ctx, cancel := context.WithTimeout(ctx, 5*time.Second)
|
||||
defer cancel()
|
||||
return l.SessionLocker.SessionUnlock(ctx, conn)
|
||||
}
|
||||
|
||||
func main() {
|
||||
command := flag.String("command", "up", "migration action: up or status")
|
||||
timeout := flag.Duration("timeout", 2*time.Minute, "total migration/connection/lock budget")
|
||||
flag.Parse()
|
||||
logger := slog.New(slog.NewJSONHandler(os.Stdout, nil))
|
||||
if flag.NArg() != 0 || (*command != "up" && *command != "status") || *timeout < time.Second {
|
||||
logger.Error("invalid migration arguments", "allowed_commands", "up,status",
|
||||
"minimum_timeout", "1s")
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
signalCtx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
|
||||
defer stop()
|
||||
ctx, cancel := context.WithTimeout(signalCtx, *timeout)
|
||||
defer cancel()
|
||||
if err := run(ctx, *command, *timeout, logger); err != nil {
|
||||
fields := []any{"stage", err.Error(), "timeout", errors.Is(err, context.DeadlineExceeded),
|
||||
"cancelled", errors.Is(err, context.Canceled)}
|
||||
var pgErr *pgconn.PgError
|
||||
if errors.As(err, &pgErr) {
|
||||
fields = append(fields, "sqlstate", pgErr.Code)
|
||||
}
|
||||
var partial *goose.PartialError
|
||||
if errors.As(err, &partial) && partial.Failed != nil && partial.Failed.Source != nil {
|
||||
fields = append(fields, "migration", partial.Failed.Source.Path,
|
||||
"version", partial.Failed.Source.Version)
|
||||
}
|
||||
logger.Error("migration failed", fields...)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
func run(ctx context.Context, command string, timeout time.Duration, logger *slog.Logger) error {
|
||||
fail := func(stage string, err error) error {
|
||||
return &migrationError{stage: stage, cause: err}
|
||||
}
|
||||
path := os.Getenv("DATABASE_URL_FILE")
|
||||
if path == "" {
|
||||
path = "/run/secrets/database_url"
|
||||
}
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return fail("read database secret file", err)
|
||||
}
|
||||
url := strings.TrimSpace(string(data))
|
||||
if url == "" {
|
||||
return fail("database secret file is empty", errors.New("empty database URL"))
|
||||
}
|
||||
cfg, err := pgx.ParseConfig(url)
|
||||
if err != nil {
|
||||
return fail("parse database secret", err)
|
||||
}
|
||||
cfg.ConnectTimeout = 5 * time.Second
|
||||
cfg.RuntimeParams["application_name"] = "highload-migrate"
|
||||
cfg.RuntimeParams["lock_timeout"] = "10000"
|
||||
cfg.RuntimeParams["statement_timeout"] = strconv.FormatInt(min(timeout.Milliseconds(), 2147483647), 10)
|
||||
db := stdlib.OpenDB(*cfg)
|
||||
db.SetMaxOpenConns(2)
|
||||
db.SetMaxIdleConns(2)
|
||||
db.SetConnMaxLifetime(5 * time.Minute)
|
||||
defer db.Close()
|
||||
if err := db.PingContext(ctx); err != nil {
|
||||
return fail("connect to PostgreSQL", err)
|
||||
}
|
||||
|
||||
locker, err := lock.NewPostgresSessionLocker(
|
||||
lock.WithLockTimeout(1, 120),
|
||||
lock.WithUnlockTimeout(1, 5),
|
||||
)
|
||||
if err != nil {
|
||||
return fail("configure migration lock", err)
|
||||
}
|
||||
provider, err := goose.NewProvider(goose.DialectPostgres, db, migrations.Files,
|
||||
goose.WithSessionLocker(boundedSessionLocker{locker}),
|
||||
goose.WithDisableGlobalRegistry(true),
|
||||
goose.WithSlog(logger),
|
||||
// Goose verbose logging includes SQL statements. Log only selected
|
||||
// migration metadata ourselves, even when a migration fails.
|
||||
goose.WithVerbose(false),
|
||||
)
|
||||
if err != nil {
|
||||
return fail("load embedded migrations", err)
|
||||
}
|
||||
if command == "status" {
|
||||
statuses, err := provider.Status(ctx)
|
||||
if err != nil {
|
||||
return fail("read migration status", err)
|
||||
}
|
||||
for _, status := range statuses {
|
||||
logger.Info("migration status", "file", status.Source.Path,
|
||||
"version", status.Source.Version, "state", status.State)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
started := time.Now()
|
||||
results, err := provider.Up(ctx)
|
||||
if err != nil {
|
||||
return fail("apply pending migrations", err)
|
||||
}
|
||||
for _, result := range results {
|
||||
logger.Info("migration applied", "file", result.Source.Path,
|
||||
"version", result.Source.Version, "duration_seconds", result.Duration.Seconds())
|
||||
}
|
||||
logger.Info("migrations completed", "applied", len(results),
|
||||
"duration_seconds", time.Since(started).Seconds())
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,111 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"errors"
|
||||
"io"
|
||||
"log/slog"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"highload/db/migrations"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
"github.com/jackc/pgx/v5/stdlib"
|
||||
"github.com/pressly/goose/v3"
|
||||
)
|
||||
|
||||
func TestConnectionSecretErrorsAreSafe(t *testing.T) {
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
content string
|
||||
missing bool
|
||||
}{
|
||||
{name: "missing", missing: true},
|
||||
{name: "empty", content: " \n\t"},
|
||||
{name: "malformed", content: "postgres://synthetic:fixture-password@localhost/%zz"},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "database_url")
|
||||
if !test.missing {
|
||||
if err := os.WriteFile(path, []byte(test.content), 0600); err != nil {
|
||||
t.Fatal("cannot create test secret")
|
||||
}
|
||||
}
|
||||
t.Setenv("DATABASE_URL_FILE", path)
|
||||
logger := slog.New(slog.NewJSONHandler(io.Discard, nil))
|
||||
err := run(context.Background(), "up", time.Second, logger)
|
||||
if err == nil {
|
||||
t.Fatal("invalid connection secret was accepted")
|
||||
}
|
||||
if strings.Contains(err.Error(), "fixture-password") ||
|
||||
strings.Contains(err.Error(), "postgres://") {
|
||||
t.Fatal("migration error exposed a connection secret")
|
||||
}
|
||||
var classified *migrationError
|
||||
if !errors.As(err, &classified) || classified.Unwrap() == nil {
|
||||
t.Fatal("migration error lost its classification/cause")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestEmbeddedMigrationsAreVersionedAndCollectable(t *testing.T) {
|
||||
cfg, err := pgx.ParseConfig("postgres://synthetic@localhost/synthetic?sslmode=disable")
|
||||
if err != nil {
|
||||
t.Fatal("cannot configure test driver")
|
||||
}
|
||||
db := stdlib.OpenDB(*cfg)
|
||||
defer db.Close()
|
||||
// NewProvider collects sources without connecting to PostgreSQL.
|
||||
provider, err := goose.NewProvider(goose.DialectPostgres, db, migrations.Files)
|
||||
if err != nil {
|
||||
t.Fatal("embedded migration names/versions cannot be collected by Goose")
|
||||
}
|
||||
sources := provider.ListSources()
|
||||
if len(sources) == 0 {
|
||||
t.Fatal("migrator build contains no SQL")
|
||||
}
|
||||
var previous int64
|
||||
for _, source := range sources {
|
||||
if source.Version <= previous {
|
||||
t.Fatal("embedded versions are not unique and increasing")
|
||||
}
|
||||
previous = source.Version
|
||||
data, err := migrations.Files.ReadFile(source.Path)
|
||||
if err != nil || !strings.Contains(string(data), "-- +goose Up") {
|
||||
t.Fatal("embedded migration is unavailable or has no Up section")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
type deadlineProbe struct {
|
||||
deadline time.Time
|
||||
cancel error
|
||||
}
|
||||
|
||||
func (*deadlineProbe) SessionLock(context.Context, *sql.Conn) error { return nil }
|
||||
func (p *deadlineProbe) SessionUnlock(ctx context.Context, _ *sql.Conn) error {
|
||||
p.deadline, _ = ctx.Deadline()
|
||||
p.cancel = ctx.Err()
|
||||
return nil
|
||||
}
|
||||
|
||||
func TestUnlockHasBudgetEvenWithDetachedContext(t *testing.T) {
|
||||
original, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
probe := &deadlineProbe{}
|
||||
locker := boundedSessionLocker{probe}
|
||||
started := time.Now()
|
||||
if err := locker.SessionUnlock(context.WithoutCancel(original), nil); err != nil {
|
||||
t.Fatal("unlock delegation failed")
|
||||
}
|
||||
if probe.cancel != nil || probe.deadline.IsZero() ||
|
||||
probe.deadline.Before(started) || probe.deadline.After(started.Add(6*time.Second)) {
|
||||
t.Fatal("detached cleanup did not receive a bounded, live context")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,212 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"log/slog"
|
||||
"net/url"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/jackc/pgx/v5"
|
||||
"github.com/jackc/pgx/v5/pgconn"
|
||||
"github.com/jackc/pgx/v5/stdlib"
|
||||
)
|
||||
|
||||
// These tests need an explicitly supplied disposable PostgreSQL database.
|
||||
// Each test creates and drops only its own randomly named schema.
|
||||
func migrationPostgresFixture(t *testing.T) (*sql.DB, context.Context) {
|
||||
t.Helper()
|
||||
connectionURL := os.Getenv("MIGRATE_TEST_DATABASE_URL")
|
||||
if connectionURL == "" {
|
||||
t.Skip("set MIGRATE_TEST_DATABASE_URL to run PostgreSQL integration tests")
|
||||
}
|
||||
cfg, err := pgx.ParseConfig(connectionURL)
|
||||
if err != nil {
|
||||
t.Fatal("invalid test database configuration")
|
||||
}
|
||||
cfg.ConnectTimeout = 5 * time.Second
|
||||
admin := stdlib.OpenDB(*cfg)
|
||||
t.Cleanup(func() { admin.Close() })
|
||||
ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
|
||||
t.Cleanup(cancel)
|
||||
schema := "migration_test_" + strings.ReplaceAll(uuid.NewString(), "-", "")
|
||||
quoted := pgx.Identifier{schema}.Sanitize()
|
||||
if _, err := admin.ExecContext(ctx, "CREATE SCHEMA "+quoted); err != nil {
|
||||
t.Fatal("cannot create isolated migration test schema")
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
cleanupCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
if _, err := admin.ExecContext(cleanupCtx, "DROP SCHEMA "+quoted+" CASCADE"); err != nil {
|
||||
t.Error("cannot remove isolated migration test schema")
|
||||
}
|
||||
})
|
||||
cfg.RuntimeParams["search_path"] = schema
|
||||
// ConnString retains the original parsed URL; changing RuntimeParams alone
|
||||
// would not pass search_path to the separate migrator connection.
|
||||
scopedURL, err := url.Parse(connectionURL)
|
||||
if err != nil || (scopedURL.Scheme != "postgres" && scopedURL.Scheme != "postgresql") {
|
||||
t.Fatal("MIGRATE_TEST_DATABASE_URL must be a PostgreSQL URL")
|
||||
}
|
||||
query := scopedURL.Query()
|
||||
query.Set("search_path", schema)
|
||||
scopedURL.RawQuery = query.Encode()
|
||||
path := filepath.Join(t.TempDir(), "database_url")
|
||||
if err := os.WriteFile(path, []byte(scopedURL.String()), 0600); err != nil {
|
||||
t.Fatal("cannot write test connection secret")
|
||||
}
|
||||
t.Setenv("DATABASE_URL_FILE", path)
|
||||
db := stdlib.OpenDB(*cfg)
|
||||
t.Cleanup(func() { db.Close() })
|
||||
return db, ctx
|
||||
}
|
||||
|
||||
func migrationLogRecords(t *testing.T, output *bytes.Buffer) []map[string]any {
|
||||
t.Helper()
|
||||
var records []map[string]any
|
||||
for _, line := range bytes.Split(bytes.TrimSpace(output.Bytes()), []byte("\n")) {
|
||||
if len(line) == 0 {
|
||||
continue
|
||||
}
|
||||
var record map[string]any
|
||||
if err := json.Unmarshal(line, &record); err != nil {
|
||||
t.Fatal("migrator emitted a non-JSON log record")
|
||||
}
|
||||
for key := range record {
|
||||
switch key {
|
||||
case "time", "level", "msg", "file", "version", "state", "applied", "duration_seconds":
|
||||
default:
|
||||
t.Fatalf("unexpected log field %q; SQL or connection details must not be logged", key)
|
||||
}
|
||||
}
|
||||
if strings.Contains(string(line), "CREATE TABLE") || strings.Contains(string(line), "postgres://") {
|
||||
t.Fatal("migration log contains SQL or a connection URL")
|
||||
}
|
||||
records = append(records, record)
|
||||
}
|
||||
return records
|
||||
}
|
||||
|
||||
func completedMigrationCount(t *testing.T, output *bytes.Buffer) int {
|
||||
t.Helper()
|
||||
for _, record := range migrationLogRecords(t, output) {
|
||||
if record["msg"] == "migrations completed" {
|
||||
count, ok := record["applied"].(float64)
|
||||
if !ok {
|
||||
t.Fatal("migration completion log has no numeric applied count")
|
||||
}
|
||||
return int(count)
|
||||
}
|
||||
}
|
||||
t.Fatal("successful migration run has no completion log")
|
||||
return 0
|
||||
}
|
||||
|
||||
const lab5MigrationCount = 8
|
||||
|
||||
func TestPostgresMigrationsRepeatWithoutChangingDataOrLoggingSQL(t *testing.T) {
|
||||
db, ctx := migrationPostgresFixture(t)
|
||||
var output bytes.Buffer
|
||||
logger := slog.New(slog.NewJSONHandler(&output, nil))
|
||||
if err := run(ctx, "up", time.Minute, logger); err != nil {
|
||||
t.Fatal("migration of empty schema failed")
|
||||
}
|
||||
if count := completedMigrationCount(t, &output); count != lab5MigrationCount {
|
||||
t.Fatalf("empty schema: applied %d migrations, want %d", count, lab5MigrationCount)
|
||||
}
|
||||
if _, err := db.ExecContext(ctx, "INSERT INTO users (username, display_name, password_hash) VALUES ($1, $2, $3)",
|
||||
"migration_fixture", "Migration fixture", "synthetic-test-only-hash"); err != nil {
|
||||
t.Fatal("cannot insert persistent migration fixture")
|
||||
}
|
||||
output.Reset()
|
||||
if err := run(ctx, "up", time.Minute, logger); err != nil {
|
||||
t.Fatal("repeat migration failed")
|
||||
}
|
||||
if count := completedMigrationCount(t, &output); count != 0 {
|
||||
t.Fatalf("repeat run: applied %d migrations, want 0", count)
|
||||
}
|
||||
var users, versions int
|
||||
if err := db.QueryRowContext(ctx, "SELECT count(*) FROM users WHERE username = $1", "migration_fixture").Scan(&users); err != nil || users != 1 {
|
||||
t.Fatal("repeat migration lost or duplicated existing data")
|
||||
}
|
||||
if err := db.QueryRowContext(ctx, "SELECT count(*) FROM goose_db_version WHERE version_id > 0 AND is_applied").Scan(&versions); err != nil || versions != lab5MigrationCount {
|
||||
t.Fatal("repeat migration changed version history")
|
||||
}
|
||||
output.Reset()
|
||||
if err := run(ctx, "status", time.Minute, logger); err != nil {
|
||||
t.Fatal("migration status failed")
|
||||
}
|
||||
records := migrationLogRecords(t, &output)
|
||||
if len(records) != lab5MigrationCount {
|
||||
t.Fatal("status did not report all seven migrations")
|
||||
}
|
||||
for _, record := range records {
|
||||
if record["state"] != "applied" {
|
||||
t.Fatal("status did not report applied migrations")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestPostgresConcurrentMigratorsApplyEachVersionOnce(t *testing.T) {
|
||||
db, ctx := migrationPostgresFixture(t)
|
||||
var outputs [2]bytes.Buffer
|
||||
var runErrors [2]error
|
||||
var workers sync.WaitGroup
|
||||
start := make(chan struct{})
|
||||
for i := range outputs {
|
||||
workers.Add(1)
|
||||
go func(i int) {
|
||||
defer workers.Done()
|
||||
<-start
|
||||
runErrors[i] = run(ctx, "up", time.Minute, slog.New(slog.NewJSONHandler(&outputs[i], nil)))
|
||||
}(i)
|
||||
}
|
||||
close(start)
|
||||
workers.Wait()
|
||||
var applied int
|
||||
for i, err := range runErrors {
|
||||
if err != nil {
|
||||
t.Fatal("concurrent migration run failed")
|
||||
}
|
||||
applied += completedMigrationCount(t, &outputs[i])
|
||||
}
|
||||
if applied != lab5MigrationCount {
|
||||
t.Fatalf("concurrent runners applied %d migrations in total, want %d", applied, lab5MigrationCount)
|
||||
}
|
||||
var versions, distinctVersions int
|
||||
if err := db.QueryRowContext(ctx, "SELECT count(*), count(DISTINCT version_id) FROM goose_db_version WHERE version_id > 0 AND is_applied").Scan(&versions, &distinctVersions); err != nil || versions != lab5MigrationCount || distinctVersions != lab5MigrationCount {
|
||||
t.Fatal("concurrent runners duplicated or lost version history")
|
||||
}
|
||||
}
|
||||
|
||||
func TestPostgresMigrationFailureRollsBackAndPreservesCause(t *testing.T) {
|
||||
db, ctx := migrationPostgresFixture(t)
|
||||
// The first migration creates users, then conflicts with this chats table.
|
||||
if _, err := db.ExecContext(ctx, "CREATE TABLE chats (marker text)"); err != nil {
|
||||
t.Fatal("cannot create conflicting test table")
|
||||
}
|
||||
var output bytes.Buffer
|
||||
err := run(ctx, "up", time.Minute, slog.New(slog.NewJSONHandler(&output, nil)))
|
||||
var pgErr *pgconn.PgError
|
||||
if err == nil || err.Error() != "apply pending migrations" || !errors.As(err, &pgErr) || pgErr.Code != "42P07" {
|
||||
t.Fatal("migration failure did not retain a safely reportable duplicate-table error")
|
||||
}
|
||||
migrationLogRecords(t, &output)
|
||||
var usersAbsent bool
|
||||
if err := db.QueryRowContext(ctx, "SELECT to_regclass('users') IS NULL").Scan(&usersAbsent); err != nil || !usersAbsent {
|
||||
t.Fatal("failed migration left a partially applied users table")
|
||||
}
|
||||
var versions int
|
||||
if err := db.QueryRowContext(ctx, "SELECT count(*) FROM goose_db_version WHERE version_id > 0 AND is_applied").Scan(&versions); err != nil || versions != 0 {
|
||||
t.Fatal("failed migration was recorded as applied")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,167 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"io"
|
||||
"log/slog"
|
||||
"testing"
|
||||
|
||||
"github.com/pressly/goose/v3"
|
||||
"highload/db/migrations"
|
||||
)
|
||||
|
||||
func sessionSchema(t *testing.T, db *sql.DB, ctx context.Context) []string {
|
||||
t.Helper()
|
||||
rows, err := db.QueryContext(ctx, `
|
||||
SELECT 'column:'||attname||':'||format_type(atttypid,atttypmod)||':'||attnotnull||':'||
|
||||
COALESCE(pg_get_expr(d.adbin,d.adrelid),'')
|
||||
FROM pg_attribute a LEFT JOIN pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum
|
||||
WHERE a.attrelid='sessions'::regclass AND a.attnum>0 AND NOT a.attisdropped
|
||||
UNION ALL SELECT 'constraint:'||conname||':'||pg_get_constraintdef(oid)
|
||||
FROM pg_constraint WHERE conrelid='sessions'::regclass
|
||||
UNION ALL SELECT 'index:'||indexname||':'||indexdef
|
||||
FROM pg_indexes WHERE schemaname=current_schema() AND tablename='sessions'
|
||||
ORDER BY 1`)
|
||||
if err != nil {
|
||||
t.Fatal("cannot inspect restored session columns/defaults/constraints/indexes")
|
||||
}
|
||||
defer rows.Close()
|
||||
var result []string
|
||||
for rows.Next() {
|
||||
var s string
|
||||
if rows.Scan(&s) != nil {
|
||||
t.Fatal("schema scan failed")
|
||||
}
|
||||
result = append(result, s)
|
||||
}
|
||||
if rows.Err() != nil || len(result) < 10 {
|
||||
t.Fatal("incomplete session structure snapshot")
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
func businessData(t *testing.T, db *sql.DB, ctx context.Context) []string {
|
||||
t.Helper()
|
||||
result := []string{}
|
||||
for _, table := range []string{"users", "chats", "chat_members", "messages"} {
|
||||
rows, err := db.QueryContext(ctx, "SELECT row_to_json(t)::text FROM "+table+" t ORDER BY row_to_json(t)::text")
|
||||
if err != nil {
|
||||
t.Fatal("business table disappeared during migration")
|
||||
}
|
||||
count := 0
|
||||
for rows.Next() {
|
||||
var value string
|
||||
if rows.Scan(&value) != nil {
|
||||
t.Fatal("data scan failed")
|
||||
}
|
||||
result = append(result, table+value)
|
||||
count++
|
||||
}
|
||||
err = rows.Err()
|
||||
rows.Close()
|
||||
if err != nil || count == 0 {
|
||||
t.Fatal("business fixture missing during migration")
|
||||
}
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
func equalStrings(a, b []string) bool {
|
||||
if len(a) != len(b) {
|
||||
return false
|
||||
}
|
||||
for i := range a {
|
||||
if a[i] != b[i] {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func TestPostgresMigrationThreeDownTwoDownOneAndReapply(t *testing.T) {
|
||||
db, ctx := migrationPostgresFixture(t)
|
||||
provider, err := goose.NewProvider(goose.DialectPostgres, db, migrations.Files,
|
||||
goose.WithDisableGlobalRegistry(true), goose.WithSlog(slog.New(slog.NewTextHandler(io.Discard, nil))))
|
||||
if err != nil {
|
||||
t.Fatal("cannot load actual embedded migrations")
|
||||
}
|
||||
if _, err = provider.UpTo(ctx, 1); err != nil {
|
||||
t.Fatal("cannot apply version 1")
|
||||
}
|
||||
v1 := sessionSchema(t, db, ctx)
|
||||
if _, err = db.ExecContext(ctx, `
|
||||
INSERT INTO users(id,username,display_name,password_hash,created_at) VALUES
|
||||
('00000000-0000-4000-8000-000000000001','roundtrip_user','Roundtrip','synthetic-hash','2026-01-01T00:00:00Z');
|
||||
INSERT INTO chats(id,title,created_at) VALUES
|
||||
('00000000-0000-4000-8000-000000000002','Roundtrip','2026-01-01T00:00:00Z');
|
||||
INSERT INTO chat_members VALUES('00000000-0000-4000-8000-000000000002','00000000-0000-4000-8000-000000000001');
|
||||
INSERT INTO messages(id,chat_id,sender_id,text,created_at) VALUES
|
||||
('00000000-0000-4000-8000-000000000003','00000000-0000-4000-8000-000000000002','00000000-0000-4000-8000-000000000001','Keep this message','2026-01-01T00:00:00Z');
|
||||
INSERT INTO sessions VALUES(decode(repeat('ab',32),'hex'),'00000000-0000-4000-8000-000000000001','2026-01-01T00:00:00Z','2026-01-02T00:00:00Z',NULL);
|
||||
`); err != nil {
|
||||
t.Fatal("cannot create migration preservation fixtures")
|
||||
}
|
||||
original := businessData(t, db, ctx)
|
||||
if _, err = provider.UpTo(ctx, 2); err != nil {
|
||||
t.Fatal("cannot apply version 2")
|
||||
}
|
||||
v2 := sessionSchema(t, db, ctx)
|
||||
assertVersion := func(want int64) {
|
||||
t.Helper()
|
||||
got, e := provider.GetDBVersion(ctx)
|
||||
if e != nil || got != want {
|
||||
t.Fatalf("want migration version %d, got %d", want, got)
|
||||
}
|
||||
if !equalStrings(original, businessData(t, db, ctx)) {
|
||||
t.Fatal("migration changed unrelated users/chats/members/messages")
|
||||
}
|
||||
}
|
||||
assertAbsent := func() {
|
||||
t.Helper()
|
||||
var absent bool
|
||||
if db.QueryRowContext(ctx, "SELECT to_regclass('sessions') IS NULL").Scan(&absent) != nil || !absent {
|
||||
t.Fatal("version 3 must remove SQL sessions")
|
||||
}
|
||||
}
|
||||
if _, err = provider.Up(ctx); err != nil {
|
||||
t.Fatal("cannot migrate to latest version")
|
||||
}
|
||||
assertVersion(lab5MigrationCount)
|
||||
assertAbsent()
|
||||
if _, err = provider.DownTo(ctx, 2); err != nil {
|
||||
t.Fatal("latest -> 2 failed")
|
||||
}
|
||||
assertVersion(2)
|
||||
if !equalStrings(v2, sessionSchema(t, db, ctx)) {
|
||||
t.Fatal("3 -> 2 did not restore exact version-2 session columns, defaults, keys, FK, CHECK and indexes")
|
||||
}
|
||||
var count int
|
||||
if db.QueryRowContext(ctx, "SELECT count(*) FROM sessions").Scan(&count) != nil || count != 0 {
|
||||
t.Fatal("rollback must restore an empty sessions table; deleted credentials cannot be resurrected")
|
||||
}
|
||||
if _, err = db.ExecContext(ctx, `INSERT INTO sessions(token_hash,user_id,expires_at) VALUES(decode(repeat('cd',32),'hex'),'00000000-0000-4000-8000-000000000001',now()+interval '1 hour')`); err != nil {
|
||||
t.Fatal("restored table/default/FK cannot accept a valid session")
|
||||
}
|
||||
for _, statement := range []string{
|
||||
`INSERT INTO sessions(token_hash,user_id,expires_at) VALUES(decode(repeat('ef',32),'hex'),'00000000-0000-4000-8000-000000000099',now()+interval '1 hour')`,
|
||||
`INSERT INTO sessions(token_hash,user_id,expires_at) VALUES(decode(repeat('ef',32),'hex'),'00000000-0000-4000-8000-000000000001',now()-interval '1 hour')`,
|
||||
`INSERT INTO sessions(token_hash,user_id,expires_at) VALUES(decode(repeat('cd',32),'hex'),'00000000-0000-4000-8000-000000000001',now()+interval '1 hour')`,
|
||||
} {
|
||||
if _, err = db.ExecContext(ctx, statement); err == nil {
|
||||
t.Fatal("restored table lost FK, expiry CHECK or primary key")
|
||||
}
|
||||
}
|
||||
if _, err = provider.DownTo(ctx, 1); err != nil {
|
||||
t.Fatal("2 -> 1 failed")
|
||||
}
|
||||
assertVersion(1)
|
||||
if !equalStrings(v1, sessionSchema(t, db, ctx)) {
|
||||
t.Fatal("2 -> 1 did not restore original sessions structure without created_at default")
|
||||
}
|
||||
if _, err = provider.Up(ctx); err != nil {
|
||||
t.Fatal("1 -> latest reapplication failed")
|
||||
}
|
||||
assertVersion(lab5MigrationCount)
|
||||
assertAbsent()
|
||||
}
|
||||
@@ -0,0 +1,179 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"os"
|
||||
"os/signal"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"highload/internal/broker"
|
||||
"highload/internal/config"
|
||||
"highload/internal/database"
|
||||
"highload/internal/database/sqlc"
|
||||
"highload/internal/fault"
|
||||
"highload/internal/observability"
|
||||
"highload/internal/repository/postgres"
|
||||
|
||||
"go.opentelemetry.io/otel/attribute"
|
||||
"go.opentelemetry.io/otel/trace"
|
||||
)
|
||||
|
||||
func main() {
|
||||
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
|
||||
defer stop()
|
||||
if err := run(ctx); err != nil {
|
||||
slog.Error("relay startup failed", "error", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
func run(ctx context.Context) error {
|
||||
cfg, err := config.LoadPipeline()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer observability.Start(ctx, "messenger-relay")()
|
||||
if cfg.BrokerURL == "" {
|
||||
return fmt.Errorf("BROKER_URL_FILE must contain an AMQP URL")
|
||||
}
|
||||
url, err := config.LoadPostgresURL()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
pool, err := database.NewPostgres(ctx, url)
|
||||
if err != nil {
|
||||
return fmt.Errorf("PostgreSQL unavailable")
|
||||
}
|
||||
defer pool.Close()
|
||||
queries := sqlc.New(pool)
|
||||
repo := postgres.NewAttachmentRepository(queries, pool)
|
||||
sampled := make(chan struct{})
|
||||
go func() {
|
||||
defer close(sampled)
|
||||
sampleQueue(ctx, queries, observability.WatchQueue(broker.Queue))
|
||||
}()
|
||||
|
||||
recovered := make(chan struct{})
|
||||
go func() {
|
||||
defer close(recovered)
|
||||
for ctx.Err() == nil {
|
||||
if n, err := repo.EnqueueExpiredLeases(ctx, 32); err != nil && ctx.Err() == nil {
|
||||
slog.Warn("lease recovery failed", "error", err)
|
||||
} else if n > 0 {
|
||||
slog.Info("expired leases enqueued", "count", n)
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
case <-time.After(250 * time.Millisecond):
|
||||
}
|
||||
}
|
||||
}()
|
||||
for ctx.Err() == nil {
|
||||
if err := publish(ctx, repo, cfg.BrokerURL); err != nil && ctx.Err() == nil {
|
||||
slog.Warn("relay reconnecting", "error", err)
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
case <-time.After(500 * time.Millisecond):
|
||||
}
|
||||
}
|
||||
<-recovered
|
||||
<-sampled
|
||||
return nil
|
||||
}
|
||||
|
||||
func publish(ctx context.Context, repo *postgres.AttachmentRepository, url string) error {
|
||||
conn, ch, err := broker.Connect(url)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer broker.Close(conn)
|
||||
publisher, err := broker.NewPublisher(ch)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for ctx.Err() == nil {
|
||||
sent, err := repo.PublishPending(ctx, func(txCtx context.Context, event sqlc.Outbox) error {
|
||||
if err := fault.Pause(txCtx, "before_publish", event.JobID.String()); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := publishEvent(txCtx, publisher, event); err != nil {
|
||||
return err
|
||||
}
|
||||
return fault.Pause(txCtx, "after_publish", event.JobID.String())
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !sent {
|
||||
if conn.IsClosed() {
|
||||
return fmt.Errorf("RabbitMQ connection closed")
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
case <-time.After(150 * time.Millisecond):
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func sampleQueue(ctx context.Context, queries *sqlc.Queries, observe func(observability.QueueSample)) {
|
||||
ticker := time.NewTicker(2 * time.Second)
|
||||
defer ticker.Stop()
|
||||
for ctx.Err() == nil {
|
||||
sampledAt := time.Now()
|
||||
queryCtx, cancel := context.WithTimeout(ctx, 800*time.Millisecond)
|
||||
row, err := queries.GetQueueStats(queryCtx)
|
||||
cancel()
|
||||
if err == nil {
|
||||
observe(observability.QueueSample{Depth: row.Depth, OldestSeconds: row.OldestSeconds, At: sampledAt})
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
case <-ticker.C:
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func publishEvent(ctx context.Context, publisher *broker.Publisher, event sqlc.Outbox) (err error) {
|
||||
var payload map[string]json.RawMessage
|
||||
if err := json.Unmarshal(event.Payload, &payload); err != nil {
|
||||
return err
|
||||
}
|
||||
if payload == nil {
|
||||
return fmt.Errorf("outbox payload must be an object")
|
||||
}
|
||||
var traceparent, tracestate string
|
||||
_ = json.Unmarshal(payload["traceparent"], &traceparent)
|
||||
_ = json.Unmarshal(payload["tracestate"], &tracestate)
|
||||
|
||||
ctx = observability.JobContext(ctx, traceparent, tracestate)
|
||||
ctx, span := observability.Tracer.Start(ctx, "rabbitmq.publish",
|
||||
trace.WithSpanKind(trace.SpanKindProducer), trace.WithAttributes(
|
||||
attribute.String("messaging.system", "rabbitmq"),
|
||||
attribute.String("messaging.destination.name", broker.Queue),
|
||||
attribute.String("messaging.operation.name", "publish"),
|
||||
attribute.String("messaging.message.id", event.JobID.String()),
|
||||
))
|
||||
defer func() { observability.End(span, err) }()
|
||||
carrier := observability.Carrier(ctx)
|
||||
for _, key := range []string{"traceparent", "tracestate"} {
|
||||
if value := carrier.Get(key); value != "" {
|
||||
payload[key], _ = json.Marshal(value)
|
||||
} else {
|
||||
delete(payload, key)
|
||||
}
|
||||
}
|
||||
body, err := json.Marshal(payload)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
confirmCtx, cancel := context.WithTimeout(ctx, 5*time.Second)
|
||||
defer cancel()
|
||||
return publisher.Publish(confirmCtx, event.ID.String(), body)
|
||||
}
|
||||
@@ -0,0 +1,61 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"os"
|
||||
"os/signal"
|
||||
"sync"
|
||||
"syscall"
|
||||
|
||||
"highload/internal/config"
|
||||
"highload/internal/database"
|
||||
"highload/internal/database/sqlc"
|
||||
"highload/internal/observability"
|
||||
"highload/internal/pipeline"
|
||||
"highload/internal/repository/postgres"
|
||||
"highload/internal/storage/s3"
|
||||
)
|
||||
|
||||
func main() {
|
||||
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
|
||||
defer stop()
|
||||
if err := run(ctx); err != nil {
|
||||
slog.Error("worker startup failed", "error", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
func run(ctx context.Context) error {
|
||||
cfg, err := config.LoadPipeline()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer observability.Start(ctx, "messenger-worker")()
|
||||
observability.WatchWorker(cfg.Concurrency)
|
||||
if cfg.BrokerURL == "" {
|
||||
return fmt.Errorf("BROKER_URL_FILE must contain an AMQP URL")
|
||||
}
|
||||
url, storageConfig, err := config.LoadAttachmentCleanup()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
pool, err := database.NewPostgres(ctx, url)
|
||||
if err != nil {
|
||||
return fmt.Errorf("PostgreSQL unavailable")
|
||||
}
|
||||
defer pool.Close()
|
||||
objects, err := s3.NewStorage(ctx, &storageConfig)
|
||||
if err != nil {
|
||||
return fmt.Errorf("S3 configuration invalid")
|
||||
}
|
||||
worker := pipeline.Worker{Repo: postgres.NewAttachmentRepository(sqlc.New(pool), pool), Objects: objects, Config: cfg}
|
||||
slog.Info("worker started", "concurrency", cfg.Concurrency, "prefetch_per_consumer", cfg.Prefetch)
|
||||
var lanes sync.WaitGroup
|
||||
for range cfg.Concurrency {
|
||||
lanes.Go(func() { worker.Serve(ctx) })
|
||||
}
|
||||
lanes.Wait()
|
||||
return nil
|
||||
}
|
||||
+376
@@ -0,0 +1,376 @@
|
||||
name: highload-lab6
|
||||
|
||||
services:
|
||||
api:
|
||||
build:
|
||||
context: .
|
||||
dockerfile: Dockerfile
|
||||
scale: 2
|
||||
cpus: "2.0"
|
||||
mem_limit: 512m
|
||||
expose:
|
||||
- "8080"
|
||||
env_file:
|
||||
- .env
|
||||
environment:
|
||||
SESSION_STORE: redis
|
||||
REDIS_ADDR: redis:6379
|
||||
OTEL_SERVICE_NAME: messenger-api
|
||||
secrets:
|
||||
- database_url
|
||||
- s3_access_key_id
|
||||
- s3_secret_access_key
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
redis:
|
||||
condition: service_healthy
|
||||
migrate:
|
||||
condition: service_completed_successfully
|
||||
s3-init:
|
||||
condition: service_completed_successfully
|
||||
restart: unless-stopped
|
||||
stop_grace_period: 20s
|
||||
read_only: true
|
||||
tmpfs:
|
||||
- /tmp:size=32m,mode=1777
|
||||
cap_drop:
|
||||
- ALL
|
||||
security_opt:
|
||||
- no-new-privileges:true
|
||||
|
||||
attachment-cleanup:
|
||||
build:
|
||||
context: .
|
||||
dockerfile: Dockerfile
|
||||
target: cleanup
|
||||
command: ["cleanup", "-dry-run=${LAB6_CLEANUP_DRY_RUN:-false}"]
|
||||
cpus: "0.5"
|
||||
mem_limit: 128m
|
||||
env_file:
|
||||
- .env
|
||||
environment:
|
||||
BROKER_URL_FILE: ""
|
||||
secrets:
|
||||
- database_url
|
||||
- s3_access_key_id
|
||||
- s3_secret_access_key
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
migrate:
|
||||
condition: service_completed_successfully
|
||||
s3-init:
|
||||
condition: service_completed_successfully
|
||||
restart: unless-stopped
|
||||
stop_grace_period: 20s
|
||||
read_only: true
|
||||
cap_drop:
|
||||
- ALL
|
||||
security_opt:
|
||||
- no-new-privileges:true
|
||||
|
||||
relay:
|
||||
build:
|
||||
context: .
|
||||
dockerfile: Dockerfile
|
||||
target: relay
|
||||
env_file: [.env]
|
||||
environment:
|
||||
BROKER_URL_FILE: /run/secrets/broker_url
|
||||
OTEL_SERVICE_NAME: messenger-relay
|
||||
secrets: [database_url, broker_url]
|
||||
depends_on:
|
||||
migrate:
|
||||
condition: service_completed_successfully
|
||||
rabbitmq:
|
||||
condition: service_healthy
|
||||
cpus: "0.5"
|
||||
mem_limit: 128m
|
||||
read_only: true
|
||||
cap_drop: [ALL]
|
||||
security_opt: [no-new-privileges:true]
|
||||
restart: unless-stopped
|
||||
|
||||
worker:
|
||||
build:
|
||||
context: .
|
||||
dockerfile: Dockerfile.worker
|
||||
env_file: [.env]
|
||||
environment:
|
||||
BROKER_URL_FILE: /run/secrets/broker_url
|
||||
OTEL_SERVICE_NAME: messenger-worker
|
||||
secrets: [database_url, broker_url, s3_access_key_id, s3_secret_access_key]
|
||||
depends_on:
|
||||
migrate:
|
||||
condition: service_completed_successfully
|
||||
s3-init:
|
||||
condition: service_completed_successfully
|
||||
rabbitmq:
|
||||
condition: service_healthy
|
||||
cpus: "2.0"
|
||||
mem_limit: 1536m
|
||||
read_only: true
|
||||
tmpfs:
|
||||
- /tmp:size=768m,mode=1777
|
||||
cap_drop: [ALL]
|
||||
security_opt: [no-new-privileges:true]
|
||||
restart: unless-stopped
|
||||
stop_grace_period: 15s
|
||||
|
||||
rabbitmq:
|
||||
image: rabbitmq:4.2.5-alpine@sha256:d4a3806070111cc69130a6542aa4049b612789852ef11fc3862abc94631b2572
|
||||
hostname: rabbitmq
|
||||
entrypoint: ["/bin/sh", "-ec"]
|
||||
command:
|
||||
- |
|
||||
export RABBITMQ_DEFAULT_PASS="$$(cat /run/secrets/rabbitmq_password)"
|
||||
exec docker-entrypoint.sh rabbitmq-server
|
||||
environment:
|
||||
RABBITMQ_DEFAULT_USER: messenger
|
||||
RABBITMQ_DEFAULT_VHOST: messenger
|
||||
secrets: [rabbitmq_password]
|
||||
volumes:
|
||||
- rabbitmq_data:/var/lib/rabbitmq
|
||||
healthcheck:
|
||||
test: ["CMD", "rabbitmq-diagnostics", "-q", "check_running"]
|
||||
interval: 5s
|
||||
timeout: 10s
|
||||
retries: 12
|
||||
start_period: 60s
|
||||
mem_limit: 768m
|
||||
cpus: "1.0"
|
||||
restart: unless-stopped
|
||||
stop_grace_period: 30s
|
||||
|
||||
lb:
|
||||
image: haproxy:3.2.24-alpine@sha256:1aca32ee932b96d2a75f1882909325839dd23c138ba9c6fd26e4d88179f30f13
|
||||
cpus: "1.0"
|
||||
mem_limit: 256m
|
||||
ports:
|
||||
- "127.0.0.1:${LAB6_HTTP_PORT:-8086}:8080"
|
||||
volumes:
|
||||
- ./infra/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro
|
||||
depends_on:
|
||||
api:
|
||||
condition: service_healthy
|
||||
restart: unless-stopped
|
||||
read_only: true
|
||||
cap_drop:
|
||||
- ALL
|
||||
security_opt:
|
||||
- no-new-privileges:true
|
||||
|
||||
migrate:
|
||||
build:
|
||||
context: .
|
||||
dockerfile: Dockerfile.migrate
|
||||
cpus: "1.0"
|
||||
mem_limit: 256m
|
||||
environment:
|
||||
DATABASE_URL_FILE: /run/secrets/database_url
|
||||
secrets:
|
||||
- database_url
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
restart: "no"
|
||||
stop_grace_period: 20s
|
||||
read_only: true
|
||||
cap_drop:
|
||||
- ALL
|
||||
security_opt:
|
||||
- no-new-privileges:true
|
||||
|
||||
postgres:
|
||||
image: postgres:18@sha256:5a5a84b19854a9ffaa54082c166ff4ec27473a361e496e5ea167f298f2da9722
|
||||
cpus: "2.0"
|
||||
mem_limit: 512m
|
||||
environment:
|
||||
POSTGRES_DB: messenger
|
||||
POSTGRES_USER: messenger
|
||||
POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password
|
||||
secrets:
|
||||
- postgres_password
|
||||
ports:
|
||||
- "127.0.0.1:${LAB6_POSTGRES_PORT:-15486}:5432"
|
||||
volumes:
|
||||
- postgres_data:/var/lib/postgresql
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "pg_isready -h 127.0.0.1 -U messenger -d messenger"]
|
||||
interval: 5s
|
||||
timeout: 5s
|
||||
retries: 5
|
||||
restart: unless-stopped
|
||||
stop_grace_period: 60s
|
||||
|
||||
redis:
|
||||
image: redis:7.4-alpine@sha256:858f009f9709ce576febc734aa78b8f6d624b82571f9ddb6bda4377c833b3499
|
||||
cpus: "0.5"
|
||||
mem_limit: 128m
|
||||
restart: unless-stopped
|
||||
command:
|
||||
- redis-server
|
||||
- --appendonly
|
||||
- "yes"
|
||||
- --appendfsync
|
||||
- everysec
|
||||
volumes:
|
||||
- redis_data:/data
|
||||
healthcheck:
|
||||
test: ["CMD", "redis-cli", "ping"]
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 5
|
||||
|
||||
s3:
|
||||
image: rustfs/rustfs:1.0.4@sha256:ed9210d6411d77ffceb70d94b964a3cb7d225983368bc25bebe01e995293dcc2
|
||||
cpus: "1.0"
|
||||
mem_limit: 512m
|
||||
environment:
|
||||
RUSTFS_VOLUMES: /data
|
||||
RUSTFS_ACCESS_KEY_FILE: /run/secrets/rustfs_root_access_key
|
||||
RUSTFS_SECRET_KEY_FILE: /run/secrets/rustfs_root_secret_key
|
||||
RUSTFS_CONSOLE_ENABLE: "false"
|
||||
secrets:
|
||||
- rustfs_root_access_key
|
||||
- rustfs_root_secret_key
|
||||
volumes:
|
||||
- rustfs_data:/data
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "curl -fsS http://127.0.0.1:9000/health/ready >/dev/null"]
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 12
|
||||
start_period: 30s
|
||||
restart: unless-stopped
|
||||
cap_drop:
|
||||
- ALL
|
||||
security_opt:
|
||||
- no-new-privileges:true
|
||||
|
||||
s3-init:
|
||||
image: rustfs/rc:v0.1.30@sha256:1296510ec27800668d26a8bdca6251e587086187a9368c7b80134a169e8a3bb8
|
||||
cpus: "0.5"
|
||||
mem_limit: 128m
|
||||
entrypoint: ["/bin/sh", "/bootstrap.sh"]
|
||||
environment:
|
||||
HOME: /tmp
|
||||
S3_BUCKET: messenger-attachments
|
||||
S3_REGION: us-east-1
|
||||
secrets:
|
||||
- rustfs_root_access_key
|
||||
- rustfs_root_secret_key
|
||||
- s3_access_key_id
|
||||
- s3_secret_access_key
|
||||
volumes:
|
||||
- ./infra/s3/bootstrap.sh:/bootstrap.sh:ro
|
||||
depends_on:
|
||||
s3:
|
||||
condition: service_healthy
|
||||
restart: "no"
|
||||
read_only: true
|
||||
tmpfs:
|
||||
- /tmp:uid=100,gid=101,mode=0700
|
||||
cap_drop:
|
||||
- ALL
|
||||
security_opt:
|
||||
- no-new-privileges:true
|
||||
|
||||
otel-collector:
|
||||
image: otel/opentelemetry-collector-contrib:0.131.0
|
||||
command: ["--config=/etc/otelcol-contrib/config.yaml"]
|
||||
environment:
|
||||
GOMEMLIMIT: 120MiB
|
||||
volumes:
|
||||
- ./infra/otel/collector.yaml:/etc/otelcol-contrib/config.yaml:ro
|
||||
mem_limit: 256m
|
||||
cpus: "0.75"
|
||||
restart: unless-stopped
|
||||
|
||||
prometheus:
|
||||
image: prom/prometheus:v3.5.0
|
||||
command:
|
||||
- --config.file=/etc/prometheus/prometheus.yml
|
||||
- --storage.tsdb.path=/prometheus
|
||||
- --storage.tsdb.retention.time=24h
|
||||
- --storage.tsdb.retention.size=512MB
|
||||
ports: ["127.0.0.1:${LAB6_PROMETHEUS_PORT:-9096}:9090"]
|
||||
volumes:
|
||||
- ./infra/otel/prometheus.yaml:/etc/prometheus/prometheus.yml:ro
|
||||
- ./infra/otel/alerts.yaml:/etc/prometheus/alerts.yaml:ro
|
||||
- prometheus_data:/prometheus
|
||||
mem_limit: 256m
|
||||
cpus: "0.75"
|
||||
restart: unless-stopped
|
||||
|
||||
loki:
|
||||
image: grafana/loki:3.5.0
|
||||
command: ["-config.file=/etc/loki/lab.yaml"]
|
||||
ports: ["127.0.0.1:${LAB6_LOKI_PORT:-3106}:3100"]
|
||||
volumes:
|
||||
- ./infra/otel/loki.yaml:/etc/loki/lab.yaml:ro
|
||||
- loki_data:/loki
|
||||
mem_limit: 384m
|
||||
cpus: "0.75"
|
||||
restart: unless-stopped
|
||||
|
||||
tempo:
|
||||
image: grafana/tempo:2.8.1
|
||||
command: ["-config.file=/etc/tempo/lab.yaml"]
|
||||
ports: ["127.0.0.1:${LAB6_TEMPO_PORT:-3206}:3200"]
|
||||
volumes:
|
||||
- ./infra/otel/tempo.yaml:/etc/tempo/lab.yaml:ro
|
||||
- tempo_data:/var/tempo
|
||||
mem_limit: 384m
|
||||
cpus: "0.75"
|
||||
restart: unless-stopped
|
||||
|
||||
grafana:
|
||||
image: grafana/grafana:12.3.1
|
||||
environment:
|
||||
GF_AUTH_ANONYMOUS_ENABLED: "true"
|
||||
GF_AUTH_ANONYMOUS_ORG_ROLE: Viewer
|
||||
GF_AUTH_BASIC_ENABLED: "false"
|
||||
GF_AUTH_DISABLE_LOGIN_FORM: "true"
|
||||
GF_SECURITY_DISABLE_INITIAL_ADMIN_CREATION: "true"
|
||||
GF_ANALYTICS_REPORTING_ENABLED: "false"
|
||||
GF_ANALYTICS_CHECK_FOR_UPDATES: "false"
|
||||
GF_ANALYTICS_CHECK_FOR_PLUGIN_UPDATES: "false"
|
||||
GF_PLUGINS_PREINSTALL_DISABLED: "true"
|
||||
ports: ["127.0.0.1:${LAB6_GRAFANA_PORT:-3006}:3000"]
|
||||
volumes:
|
||||
- ./infra/otel/grafana/provisioning:/etc/grafana/provisioning:ro
|
||||
- ./infra/otel/grafana/dashboards:/var/lib/grafana/dashboards:ro
|
||||
- grafana_data:/var/lib/grafana
|
||||
mem_limit: 256m
|
||||
cpus: "0.75"
|
||||
restart: unless-stopped
|
||||
|
||||
secrets:
|
||||
broker_url:
|
||||
file: ./secrets/broker_url
|
||||
rabbitmq_password:
|
||||
file: ./secrets/rabbitmq_password
|
||||
database_url:
|
||||
file: ./secrets/database_url
|
||||
postgres_password:
|
||||
file: ./secrets/postgres_password
|
||||
rustfs_root_access_key:
|
||||
file: ./secrets/rustfs_root_access_key
|
||||
rustfs_root_secret_key:
|
||||
file: ./secrets/rustfs_root_secret_key
|
||||
s3_access_key_id:
|
||||
file: ./secrets/s3_access_key_id
|
||||
s3_secret_access_key:
|
||||
file: ./secrets/s3_secret_access_key
|
||||
|
||||
volumes:
|
||||
rabbitmq_data:
|
||||
postgres_data:
|
||||
redis_data:
|
||||
rustfs_data:
|
||||
prometheus_data:
|
||||
loki_data:
|
||||
tempo_data:
|
||||
grafana_data:
|
||||
@@ -0,0 +1,79 @@
|
||||
-- +goose Up
|
||||
CREATE TABLE users
|
||||
(
|
||||
id uuid PRIMARY KEY NOT NULL DEFAULT gen_random_uuid(),
|
||||
username VARCHAR(32) UNIQUE NOT NULL,
|
||||
display_name VARCHAR(100) NOT NULL,
|
||||
password_hash TEXT NOT NULL,
|
||||
created_at TIMESTAMPTZ NOT NULL,
|
||||
|
||||
CONSTRAINT users_username_format_check CHECK (username COLLATE "C" ~ '^[a-z0-9_]{3,32}$'),
|
||||
CONSTRAINT users_display_name_length_check CHECK (char_length(display_name) BETWEEN 1 AND 100),
|
||||
CONSTRAINT users_display_name_not_blank_check CHECK (char_length(btrim(display_name)) > 0),
|
||||
CONSTRAINT users_password_hash_not_empty_check CHECK (char_length(password_hash) > 0)
|
||||
|
||||
);
|
||||
|
||||
|
||||
|
||||
CREATE TABLE chats
|
||||
(
|
||||
id UUID PRIMARY KEY NOT NULL DEFAULT gen_random_uuid(),
|
||||
title VARCHAR(100) NOT NULL,
|
||||
created_at TIMESTAMPTZ NOT NULL,
|
||||
|
||||
CONSTRAINT chats_title_length_check CHECK (char_length(title) BETWEEN 1 AND 100 ),
|
||||
CONSTRAINT chats_title_not_blank_check CHECK (char_length(btrim(title)) > 0)
|
||||
|
||||
);
|
||||
|
||||
CREATE TABLE chat_members
|
||||
(
|
||||
chat_id UUID NOT NULL REFERENCES chats (id),
|
||||
user_id UUID NOT NULL REFERENCES users (id),
|
||||
|
||||
PRIMARY KEY (chat_id, user_id)
|
||||
);
|
||||
|
||||
CREATE INDEX idx_user_chat_id ON chat_members (user_id, chat_id);
|
||||
|
||||
|
||||
CREATE TABLE messages
|
||||
(
|
||||
id UUID PRIMARY KEY NOT NULL DEFAULT gen_random_uuid(),
|
||||
chat_id UUID NOT NULL,
|
||||
sender_id UUID NOT NULL,
|
||||
text TEXT NOT NULL,
|
||||
created_at TIMESTAMPTZ NOT NULL,
|
||||
|
||||
FOREIGN KEY (chat_id, sender_id)
|
||||
REFERENCES chat_members (chat_id, user_id),
|
||||
|
||||
CONSTRAINT messages_text_length_check CHECK (char_length(text) BETWEEN 1 AND 4000),
|
||||
CONSTRAINT messages_text_not_blank_check CHECK (char_length(btrim(text)) > 0)
|
||||
);
|
||||
|
||||
CREATE INDEX idx_messages_chat_created_id ON messages (chat_id, created_at, id);
|
||||
|
||||
|
||||
CREATE TABLE sessions
|
||||
(
|
||||
token_hash BYTEA PRIMARY KEY NOT NULL,
|
||||
user_id UUID NOT NULL REFERENCES users (id),
|
||||
created_at TIMESTAMPTZ NOT NULL,
|
||||
expires_at TIMESTAMPTZ NOT NULL,
|
||||
revoked_at TIMESTAMPTZ NULL,
|
||||
|
||||
CONSTRAINT sessions_expiry_after_creation_check CHECK (expires_at > created_at)
|
||||
);
|
||||
|
||||
CREATE INDEX idx_session_user_id ON sessions (user_id);
|
||||
|
||||
|
||||
-- +goose Down
|
||||
|
||||
DROP TABLE IF EXISTS sessions;
|
||||
DROP TABLE IF EXISTS messages;
|
||||
DROP TABLE IF EXISTS chat_members;
|
||||
DROP TABLE IF EXISTS chats;
|
||||
DROP TABLE IF EXISTS users;
|
||||
@@ -0,0 +1,20 @@
|
||||
-- +goose Up
|
||||
ALTER TABLE users
|
||||
ALTER COLUMN created_at SET DEFAULT now();
|
||||
ALTER TABLE chats
|
||||
ALTER COLUMN created_at SET DEFAULT now();
|
||||
ALTER TABLE messages
|
||||
ALTER COLUMN created_at SET DEFAULT now();
|
||||
ALTER TABLE sessions
|
||||
ALTER COLUMN created_at SET DEFAULT now();
|
||||
|
||||
-- +goose Down
|
||||
|
||||
ALTER TABLE sessions
|
||||
ALTER COLUMN created_at DROP DEFAULT;
|
||||
ALTER TABLE messages
|
||||
ALTER COLUMN created_at DROP DEFAULT;
|
||||
ALTER TABLE chats
|
||||
ALTER COLUMN created_at DROP DEFAULT;
|
||||
ALTER TABLE users
|
||||
ALTER COLUMN created_at DROP DEFAULT;
|
||||
@@ -0,0 +1,17 @@
|
||||
-- +goose Up
|
||||
DROP TABLE IF EXISTS sessions;
|
||||
|
||||
-- +goose Down
|
||||
-- Restore the schema at version 2; deleted session data cannot be recovered.
|
||||
CREATE TABLE sessions
|
||||
(
|
||||
token_hash BYTEA PRIMARY KEY NOT NULL,
|
||||
user_id UUID NOT NULL REFERENCES users (id),
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
expires_at TIMESTAMPTZ NOT NULL,
|
||||
revoked_at TIMESTAMPTZ NULL,
|
||||
|
||||
CONSTRAINT sessions_expiry_after_creation_check CHECK (expires_at > created_at)
|
||||
);
|
||||
|
||||
CREATE INDEX idx_session_user_id ON sessions (user_id);
|
||||
@@ -0,0 +1,23 @@
|
||||
-- +goose Up
|
||||
|
||||
CREATE TABLE message_idempotency
|
||||
(
|
||||
user_id UUID NOT NULL REFERENCES users(id),
|
||||
chat_id UUID NOT NULL REFERENCES chats(id),
|
||||
idempotency_key VARCHAR(128) NOT NULL,
|
||||
message_id UUID NOT NULL REFERENCES messages(id),
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
expires_at TIMESTAMPTZ NOT NULL DEFAULT (now() + interval '24 hours'),
|
||||
|
||||
PRIMARY KEY (user_id, chat_id, idempotency_key),
|
||||
UNIQUE (message_id),
|
||||
|
||||
CONSTRAINT message_idempotency_key_format_check
|
||||
CHECK (idempotency_key COLLATE "C" ~ '^[A-Za-z0-9._:-]{1,128}$'),
|
||||
CONSTRAINT message_idempotency_expiry_check
|
||||
CHECK (expires_at > created_at)
|
||||
);
|
||||
|
||||
-- +goose Down
|
||||
|
||||
DROP TABLE IF EXISTS message_idempotency;
|
||||
@@ -0,0 +1,54 @@
|
||||
-- +goose Up
|
||||
CREATE TABLE attachments
|
||||
(
|
||||
id UUID PRIMARY KEY NOT NULL DEFAULT gen_random_uuid(),
|
||||
chat_id UUID NOT NULL,
|
||||
owner_id UUID NOT NULL,
|
||||
filename text NOT NULL,
|
||||
object_key text UNIQUE NOT NULL,
|
||||
content_type VARCHAR(10) NOT NULL,
|
||||
size_bytes bigint NOT NULL,
|
||||
sha256 text NOT NULL,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
updated_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
|
||||
CONSTRAINT fk_attachments_chat_member FOREIGN KEY (chat_id, owner_id)
|
||||
REFERENCES chat_members(chat_id, user_id),
|
||||
CONSTRAINT attachments_sha256_format_check CHECK (sha256 COLLATE "C" ~ '^[0-9a-f]{64}$'),
|
||||
CONSTRAINT attachments_content_type_check CHECK (content_type IN ('image/png', 'image/jpeg')),
|
||||
CONSTRAINT attachments_filename_check CHECK (
|
||||
char_length(filename) BETWEEN 1 AND 255
|
||||
AND filename !~ '[/\\]'
|
||||
),
|
||||
CONSTRAINT attachments_size_bytes_check CHECK (size_bytes BETWEEN 1 AND 10485760),
|
||||
CONSTRAINT attachments_id_chat_unique UNIQUE (id, chat_id),
|
||||
CONSTRAINT attachments_object_key_length_check CHECK (char_length(object_key) > 0)
|
||||
|
||||
);
|
||||
|
||||
ALTER TABLE messages
|
||||
ADD CONSTRAINT messages_id_chat_unique UNIQUE (id, chat_id);
|
||||
|
||||
CREATE TABLE messages_attachments
|
||||
(
|
||||
message_id UUID NOT NULL ,
|
||||
attachment_id UUID NOT NULL ,
|
||||
chat_id UUID NOT NULL,
|
||||
position SMALLINT NOT NULL,
|
||||
|
||||
|
||||
CONSTRAINT message_attachments_pk PRIMARY KEY (message_id, attachment_id),
|
||||
CONSTRAINT message_attachments_position_unique UNIQUE (message_id, position),
|
||||
CONSTRAINT message_attachments_position_check CHECK (position BETWEEN 0 AND 9),
|
||||
|
||||
CONSTRAINT message_attachments_message_chat_fk
|
||||
FOREIGN KEY (message_id, chat_id) REFERENCES messages(id, chat_id),
|
||||
CONSTRAINT message_attachments_attachment_chat_fk
|
||||
FOREIGN KEY (attachment_id, chat_id) REFERENCES attachments(id, chat_id)
|
||||
);
|
||||
|
||||
|
||||
-- +goose Down
|
||||
DROP TABLE IF EXISTS messages_attachments;
|
||||
DROP TABLE IF EXISTS attachments;
|
||||
ALTER TABLE messages DROP CONSTRAINT IF EXISTS messages_id_chat_unique;
|
||||
@@ -0,0 +1,18 @@
|
||||
-- +goose Up
|
||||
ALTER TABLE messages
|
||||
DROP CONSTRAINT messages_text_length_check,
|
||||
DROP CONSTRAINT messages_text_not_blank_check;
|
||||
|
||||
ALTER TABLE messages
|
||||
ADD CONSTRAINT messages_text_length_check
|
||||
CHECK (char_length(text) <= 4000);
|
||||
|
||||
-- +goose Down
|
||||
ALTER TABLE messages
|
||||
DROP CONSTRAINT messages_text_length_check;
|
||||
|
||||
ALTER TABLE messages
|
||||
ADD CONSTRAINT messages_text_length_check
|
||||
CHECK (char_length(text) BETWEEN 1 AND 4000),
|
||||
ADD CONSTRAINT messages_text_not_blank_check
|
||||
CHECK (char_length(btrim(text)) > 0);
|
||||
@@ -0,0 +1,144 @@
|
||||
-- +goose Up
|
||||
ALTER TABLE attachments
|
||||
ADD COLUMN status TEXT NOT NULL DEFAULT 'queued',
|
||||
ADD COLUMN error_code TEXT,
|
||||
ADD COLUMN staging_key TEXT,
|
||||
ADD COLUMN pipeline_version INT NOT NULL DEFAULT 1,
|
||||
ALTER COLUMN object_key DROP NOT NULL;
|
||||
|
||||
-- Retain the original download path while the copied synchronous API is in use.
|
||||
UPDATE attachments SET staging_key = object_key;
|
||||
|
||||
ALTER TABLE attachments
|
||||
ADD CONSTRAINT attachments_status_check
|
||||
CHECK (status IN ('queued', 'processing', 'ready', 'failed')),
|
||||
ADD CONSTRAINT attachments_pipeline_check CHECK (pipeline_version > 0),
|
||||
ADD CONSTRAINT attachments_staging_key_check CHECK (length(staging_key) > 0),
|
||||
ADD CONSTRAINT attachments_object_state_check CHECK (
|
||||
(status NOT IN ('queued', 'processing') OR staging_key IS NOT NULL)
|
||||
AND (status <> 'ready' OR object_key IS NOT NULL)
|
||||
),
|
||||
ADD CONSTRAINT attachments_error_check CHECK (
|
||||
(status = 'failed') = (error_code IS NOT NULL)
|
||||
AND (error_code IS NULL OR error_code COLLATE "C" ~ '^[a-z][a-z0-9_]{0,49}$')
|
||||
);
|
||||
|
||||
CREATE UNIQUE INDEX attachments_staging_key_unique ON attachments (staging_key)
|
||||
WHERE staging_key IS NOT NULL;
|
||||
|
||||
CREATE TABLE attachment_jobs (
|
||||
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
attachment_id UUID NOT NULL REFERENCES attachments(id),
|
||||
pipeline_version INT NOT NULL CHECK (pipeline_version > 0),
|
||||
attempts_started INT NOT NULL DEFAULT 0,
|
||||
max_attempts INT NOT NULL DEFAULT 5,
|
||||
next_attempt_at TIMESTAMPTZ DEFAULT now(),
|
||||
lease_generation INT NOT NULL DEFAULT 0 CHECK (lease_generation >= 0),
|
||||
lease_until TIMESTAMPTZ,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
updated_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
UNIQUE (attachment_id, pipeline_version),
|
||||
CHECK (attempts_started >= 0 AND max_attempts > 0 AND attempts_started <= max_attempts),
|
||||
CHECK (lease_until IS NULL OR next_attempt_at IS NULL)
|
||||
);
|
||||
|
||||
CREATE INDEX attachment_jobs_expired_lease_idx ON attachment_jobs (lease_until, id)
|
||||
WHERE lease_until IS NOT NULL;
|
||||
|
||||
CREATE TABLE attachment_variants (
|
||||
attachment_id UUID NOT NULL,
|
||||
pipeline_version INT NOT NULL,
|
||||
name TEXT NOT NULL,
|
||||
object_key TEXT NOT NULL UNIQUE CHECK (length(object_key) > 0),
|
||||
content_type TEXT NOT NULL,
|
||||
size_bytes BIGINT NOT NULL CHECK (size_bytes > 0),
|
||||
width INT NOT NULL CHECK (width > 0),
|
||||
height INT NOT NULL CHECK (height > 0),
|
||||
sha256 TEXT NOT NULL CHECK (sha256 COLLATE "C" ~ '^[0-9a-f]{64}$'),
|
||||
PRIMARY KEY (attachment_id, pipeline_version, name),
|
||||
FOREIGN KEY (attachment_id, pipeline_version)
|
||||
REFERENCES attachment_jobs(attachment_id, pipeline_version),
|
||||
CHECK (
|
||||
(name IN ('thumbnail', 'crop') AND content_type = 'image/jpeg')
|
||||
OR (name = 'optimized' AND content_type = 'image/webp')
|
||||
)
|
||||
);
|
||||
|
||||
CREATE TABLE attachment_attempts (
|
||||
job_id UUID NOT NULL REFERENCES attachment_jobs(id),
|
||||
lease_generation INT NOT NULL CHECK (lease_generation > 0),
|
||||
attempt INT NOT NULL CHECK (attempt > 0),
|
||||
started_at TIMESTAMPTZ NOT NULL,
|
||||
finished_at TIMESTAMPTZ,
|
||||
outcome TEXT CHECK (outcome IN ('ready', 'retry', 'failed', 'lease_expired')),
|
||||
error_code TEXT CHECK (error_code COLLATE "C" ~ '^[a-z][a-z0-9_]{0,49}$'),
|
||||
PRIMARY KEY (job_id, lease_generation),
|
||||
UNIQUE (job_id, attempt),
|
||||
CHECK ((outcome IS NULL) = (finished_at IS NULL))
|
||||
);
|
||||
|
||||
CREATE TABLE attachment_replays (
|
||||
request_id UUID PRIMARY KEY,
|
||||
job_id UUID NOT NULL REFERENCES attachment_jobs(id),
|
||||
operator_id TEXT NOT NULL CHECK (length(btrim(operator_id)) > 0),
|
||||
reason TEXT NOT NULL CHECK (length(btrim(reason)) > 0),
|
||||
extra_attempts INT NOT NULL CHECK (extra_attempts BETWEEN 1 AND 5),
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT now()
|
||||
);
|
||||
|
||||
CREATE TABLE outbox (
|
||||
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
job_id UUID NOT NULL REFERENCES attachment_jobs(id),
|
||||
payload JSONB NOT NULL,
|
||||
available_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
sent_at TIMESTAMPTZ,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
dedup_key TEXT UNIQUE NOT NULL CHECK (length(dedup_key) > 0)
|
||||
);
|
||||
|
||||
CREATE INDEX outbox_pending_idx ON outbox (available_at, id) WHERE sent_at IS NULL;
|
||||
|
||||
-- +goose StatementBegin
|
||||
CREATE FUNCTION attachment_job_payload(uuid, uuid, integer, integer, timestamptz)
|
||||
RETURNS jsonb LANGUAGE sql STABLE
|
||||
AS $$
|
||||
SELECT jsonb_build_object(
|
||||
'schema_version', 1,
|
||||
'type', 'attachment.process.v1',
|
||||
'job_id', $1,
|
||||
'attachment_id', $2,
|
||||
'pipeline_version', $3,
|
||||
'attempt', $4,
|
||||
'created_at', to_char($5 AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS.US"Z"')
|
||||
);
|
||||
$$;
|
||||
-- +goose StatementEnd
|
||||
|
||||
WITH jobs AS (
|
||||
INSERT INTO attachment_jobs (attachment_id, pipeline_version)
|
||||
SELECT id, pipeline_version FROM attachments
|
||||
RETURNING *
|
||||
)
|
||||
INSERT INTO outbox (job_id, payload, dedup_key)
|
||||
SELECT id, attachment_job_payload(id, attachment_id, pipeline_version, 1, created_at),
|
||||
'initial/' || id
|
||||
FROM jobs;
|
||||
|
||||
-- +goose Down
|
||||
-- Restore the legacy reference when the staging original is the only copy.
|
||||
-- SET NOT NULL aborts the rollback if an original no longer exists.
|
||||
UPDATE attachments SET object_key = staging_key WHERE object_key IS NULL;
|
||||
ALTER TABLE attachments ALTER COLUMN object_key SET NOT NULL;
|
||||
|
||||
DROP FUNCTION attachment_job_payload(uuid, uuid, integer, integer, timestamptz);
|
||||
DROP TABLE outbox;
|
||||
DROP TABLE attachment_replays;
|
||||
DROP TABLE attachment_attempts;
|
||||
DROP TABLE attachment_variants;
|
||||
DROP TABLE attachment_jobs;
|
||||
|
||||
ALTER TABLE attachments
|
||||
DROP COLUMN status,
|
||||
DROP COLUMN error_code,
|
||||
DROP COLUMN staging_key,
|
||||
DROP COLUMN pipeline_version;
|
||||
@@ -0,0 +1,27 @@
|
||||
-- +goose Up
|
||||
ALTER TABLE attachment_jobs ADD COLUMN published_original_key TEXT
|
||||
CHECK (length(published_original_key) > 0);
|
||||
|
||||
UPDATE attachment_jobs AS j
|
||||
SET published_original_key = a.object_key
|
||||
FROM attachments AS a
|
||||
WHERE a.id = j.attachment_id AND a.pipeline_version = j.pipeline_version
|
||||
AND a.status = 'ready';
|
||||
|
||||
CREATE INDEX attachment_jobs_published_original_idx ON attachment_jobs (published_original_key)
|
||||
WHERE published_original_key IS NOT NULL;
|
||||
|
||||
ALTER TABLE attachment_attempts
|
||||
DROP CONSTRAINT attachment_attempts_outcome_check,
|
||||
ADD CONSTRAINT attachment_attempts_outcome_check
|
||||
CHECK (outcome IN ('ready', 'retry', 'failed', 'lease_expired', 'superseded'));
|
||||
|
||||
-- +goose Down
|
||||
UPDATE attachment_attempts SET outcome = 'failed' WHERE outcome = 'superseded';
|
||||
|
||||
ALTER TABLE attachment_attempts
|
||||
DROP CONSTRAINT attachment_attempts_outcome_check,
|
||||
ADD CONSTRAINT attachment_attempts_outcome_check
|
||||
CHECK (outcome IN ('ready', 'retry', 'failed', 'lease_expired'));
|
||||
|
||||
ALTER TABLE attachment_jobs DROP COLUMN published_original_key;
|
||||
@@ -0,0 +1,9 @@
|
||||
package migrations
|
||||
|
||||
import "embed"
|
||||
|
||||
// Files contains the SQL shipped with this exact migrator build.
|
||||
// Add schema changes as new numbered files; do not edit applied migrations.
|
||||
//
|
||||
//go:embed *.sql
|
||||
var Files embed.FS
|
||||
@@ -0,0 +1,11 @@
|
||||
-- name: CreateAttachmentVariant :exec
|
||||
INSERT INTO attachment_variants (
|
||||
attachment_id, pipeline_version, name, object_key, content_type, size_bytes, width, height, sha256
|
||||
) VALUES (
|
||||
$1, $2, $3, $4, $5, $6, $7, $8, $9
|
||||
);
|
||||
|
||||
-- name: ListAttachmentVariants :many
|
||||
SELECT * FROM attachment_variants
|
||||
WHERE attachment_id = $1 AND pipeline_version = $2
|
||||
ORDER BY name;
|
||||
@@ -0,0 +1,69 @@
|
||||
-- name: CreateAttachment :one
|
||||
-- Compatibility for the synchronous API until CreateQueuedAttachment is wired in.
|
||||
WITH legacy AS (
|
||||
INSERT INTO attachments
|
||||
(chat_id, owner_id, filename, object_key, staging_key, content_type, size_bytes, sha256)
|
||||
VALUES (sqlc.arg(chat_id), sqlc.arg(owner_id), sqlc.arg(filename),
|
||||
sqlc.arg(object_key)::text, sqlc.arg(object_key)::text,
|
||||
sqlc.arg(content_type), sqlc.arg(size_bytes), sqlc.arg(sha256))
|
||||
RETURNING *
|
||||
), job AS (
|
||||
INSERT INTO attachment_jobs (attachment_id, pipeline_version)
|
||||
SELECT id, pipeline_version FROM legacy
|
||||
RETURNING *
|
||||
), event AS (
|
||||
INSERT INTO outbox (job_id, payload, dedup_key)
|
||||
SELECT id, attachment_job_payload(id, attachment_id, pipeline_version, 1, created_at),
|
||||
'initial/' || id
|
||||
FROM job
|
||||
RETURNING job_id
|
||||
)
|
||||
SELECT legacy.* FROM legacy CROSS JOIN event;
|
||||
|
||||
-- name: GetAttachmentForUser :one
|
||||
SELECT a.*
|
||||
FROM attachments AS a
|
||||
JOIN chat_members AS cm ON cm.chat_id = a.chat_id
|
||||
WHERE a.id = sqlc.arg(attachment_id)
|
||||
AND cm.user_id = sqlc.arg(user_id);
|
||||
|
||||
|
||||
-- name: GetAttachmentsForMessages :many
|
||||
SELECT message_id, attachment_id, position
|
||||
FROM messages_attachments
|
||||
WHERE message_id = ANY(sqlc.arg(message_ids)::uuid[])
|
||||
ORDER BY message_id, position;
|
||||
|
||||
|
||||
-- name: AttachmentExistsByObjectKey :one
|
||||
SELECT EXISTS (
|
||||
SELECT 1 FROM attachments
|
||||
WHERE object_key = sqlc.arg(object_key)::text OR staging_key = sqlc.arg(object_key)::text
|
||||
UNION ALL
|
||||
SELECT 1 FROM attachment_variants WHERE object_key = sqlc.arg(object_key)::text
|
||||
UNION ALL
|
||||
SELECT 1 FROM attachment_jobs WHERE published_original_key = sqlc.arg(object_key)::text
|
||||
);
|
||||
|
||||
-- name: TryLockAttachmentObjectKey :one
|
||||
SELECT pg_try_advisory_xact_lock(
|
||||
hashtextextended(sqlc.arg(object_key)::text, 0)
|
||||
)::boolean AS locked;
|
||||
|
||||
-- name: ListAttachmentKeys :many
|
||||
SELECT id, object_key, created_at
|
||||
FROM attachments
|
||||
WHERE object_key > sqlc.arg(after_key)
|
||||
ORDER BY object_key
|
||||
LIMIT sqlc.arg(page_size);
|
||||
|
||||
-- name: GetAttachmentByID :one
|
||||
SELECT * FROM attachments WHERE id = sqlc.arg(attachment_id);
|
||||
|
||||
|
||||
-- name: CreateQueuedAttachment :one
|
||||
INSERT INTO attachments (
|
||||
id, chat_id, owner_id, filename, content_type, size_bytes, sha256, staging_key, status
|
||||
) VALUES (
|
||||
$1, $2, $3, $4, $5, $6, $7, $8, 'queued'
|
||||
) RETURNING *;
|
||||
@@ -0,0 +1,207 @@
|
||||
-- name: CreateAttachmentJob :one
|
||||
INSERT INTO attachment_jobs (attachment_id, pipeline_version, next_attempt_at, max_attempts, created_at)
|
||||
VALUES (sqlc.arg(attachment_id), sqlc.arg(pipeline_version),
|
||||
sqlc.arg(available_at)::timestamptz, sqlc.arg(max_attempts), clock_timestamp())
|
||||
RETURNING *;
|
||||
|
||||
-- name: GetQueueStats :one
|
||||
SELECT count(*)::bigint AS depth,
|
||||
COALESCE(EXTRACT(EPOCH FROM (statement_timestamp() - min(j.created_at))), 0)::double precision AS oldest_seconds
|
||||
FROM attachment_jobs AS j JOIN attachments AS a ON a.id = j.attachment_id
|
||||
WHERE a.status = 'queued' AND a.pipeline_version = j.pipeline_version;
|
||||
|
||||
-- name: ClaimJob :one
|
||||
WITH claimed AS (
|
||||
UPDATE attachment_jobs AS j
|
||||
SET lease_generation = j.lease_generation + 1,
|
||||
lease_until = sqlc.arg(new_lease_until)::timestamptz,
|
||||
attempts_started = j.attempts_started + 1,
|
||||
next_attempt_at = NULL,
|
||||
updated_at = clock_timestamp()
|
||||
FROM attachments AS a
|
||||
WHERE j.id = sqlc.arg(job_id)
|
||||
AND a.id = j.attachment_id AND a.pipeline_version = j.pipeline_version
|
||||
AND j.attempts_started < j.max_attempts
|
||||
AND sqlc.arg(new_lease_until)::timestamptz > clock_timestamp()
|
||||
AND (
|
||||
(a.status = 'queued' AND j.lease_until IS NULL AND j.next_attempt_at <= clock_timestamp())
|
||||
OR (a.status = 'processing' AND j.lease_until <= clock_timestamp())
|
||||
)
|
||||
RETURNING j.*
|
||||
), expired AS (
|
||||
UPDATE attachment_attempts AS attempt
|
||||
SET outcome = 'lease_expired', error_code = 'lease_expired', finished_at = claimed.updated_at
|
||||
FROM claimed
|
||||
WHERE attempt.job_id = claimed.id
|
||||
AND attempt.lease_generation = claimed.lease_generation - 1 AND attempt.finished_at IS NULL
|
||||
), recorded AS (
|
||||
INSERT INTO attachment_attempts (job_id, lease_generation, attempt, started_at)
|
||||
SELECT id, lease_generation, attempts_started, updated_at FROM claimed
|
||||
RETURNING job_id
|
||||
), processing AS (
|
||||
UPDATE attachments AS a
|
||||
SET status = 'processing', updated_at = claimed.updated_at
|
||||
FROM claimed
|
||||
WHERE a.id = claimed.attachment_id
|
||||
RETURNING a.id
|
||||
)
|
||||
SELECT claimed.* FROM claimed JOIN recorded ON recorded.job_id = claimed.id
|
||||
JOIN processing ON processing.id = claimed.attachment_id;
|
||||
|
||||
-- name: LockActiveAttachmentJob :one
|
||||
-- Hold this lock through variant insertion and CompleteJob. Roll back on a zero-row completion.
|
||||
SELECT j.*
|
||||
FROM attachment_jobs AS j JOIN attachments AS a ON a.id = j.attachment_id
|
||||
WHERE j.id = sqlc.arg(job_id)
|
||||
AND j.lease_generation = sqlc.arg(lease_generation)
|
||||
AND j.lease_until > clock_timestamp()
|
||||
AND a.status = 'processing' AND a.pipeline_version = j.pipeline_version
|
||||
FOR UPDATE OF j;
|
||||
|
||||
-- name: CompleteJob :execrows
|
||||
WITH completed AS (
|
||||
UPDATE attachment_jobs AS j
|
||||
SET lease_until = NULL, next_attempt_at = NULL, updated_at = clock_timestamp(),
|
||||
published_original_key = sqlc.arg(object_key)::text
|
||||
FROM attachments AS a
|
||||
WHERE j.id = sqlc.arg(job_id)
|
||||
AND j.lease_generation = sqlc.arg(lease_generation) AND j.lease_until > clock_timestamp()
|
||||
AND a.id = j.attachment_id AND a.pipeline_version = j.pipeline_version AND a.status = 'processing'
|
||||
AND (SELECT count(*) FROM attachment_variants AS v
|
||||
WHERE v.attachment_id = a.id AND v.pipeline_version = j.pipeline_version) = 3
|
||||
RETURNING j.*
|
||||
), recorded AS (
|
||||
UPDATE attachment_attempts AS attempt
|
||||
SET outcome = 'ready', finished_at = completed.updated_at
|
||||
FROM completed
|
||||
WHERE attempt.job_id = completed.id AND attempt.lease_generation = completed.lease_generation
|
||||
)
|
||||
UPDATE attachments AS a
|
||||
SET status = 'ready', object_key = sqlc.arg(object_key)::text, updated_at = completed.updated_at
|
||||
FROM completed
|
||||
WHERE a.id = completed.attachment_id;
|
||||
|
||||
-- name: ScheduleRetry :execrows
|
||||
WITH scheduled AS (
|
||||
UPDATE attachment_jobs AS j
|
||||
SET lease_until = NULL, next_attempt_at = sqlc.arg(available_at)::timestamptz,
|
||||
updated_at = clock_timestamp()
|
||||
FROM attachments AS a
|
||||
WHERE j.id = sqlc.arg(job_id)
|
||||
AND j.lease_generation = sqlc.arg(lease_generation) AND j.lease_until > clock_timestamp()
|
||||
AND a.id = j.attachment_id AND a.pipeline_version = j.pipeline_version AND a.status = 'processing'
|
||||
AND j.attempts_started < j.max_attempts
|
||||
AND sqlc.arg(available_at)::timestamptz > clock_timestamp()
|
||||
RETURNING j.*
|
||||
), recorded AS (
|
||||
UPDATE attachment_attempts AS attempt
|
||||
SET outcome = 'retry', error_code = sqlc.arg(error_code)::text, finished_at = scheduled.updated_at
|
||||
FROM scheduled
|
||||
WHERE attempt.job_id = scheduled.id AND attempt.lease_generation = scheduled.lease_generation
|
||||
), event AS (
|
||||
INSERT INTO outbox (job_id, payload, available_at, dedup_key)
|
||||
SELECT id, attachment_job_payload(id, attachment_id, pipeline_version, attempts_started + 1, updated_at)
|
||||
|| jsonb_strip_nulls(jsonb_build_object(
|
||||
'traceparent', NULLIF(sqlc.arg(traceparent)::text, ''),
|
||||
'tracestate', NULLIF(sqlc.arg(tracestate)::text, '')
|
||||
)),
|
||||
next_attempt_at, 'retry/' || id || '/' || (attempts_started + 1)
|
||||
FROM scheduled
|
||||
)
|
||||
UPDATE attachments AS a
|
||||
SET status = 'queued', updated_at = scheduled.updated_at
|
||||
FROM scheduled WHERE a.id = scheduled.attachment_id;
|
||||
|
||||
-- name: FailJob :execrows
|
||||
WITH failed AS (
|
||||
UPDATE attachment_jobs AS j
|
||||
SET lease_until = NULL, next_attempt_at = NULL, updated_at = clock_timestamp()
|
||||
FROM attachments AS a
|
||||
WHERE j.id = sqlc.arg(job_id)
|
||||
AND j.lease_generation = sqlc.arg(lease_generation)
|
||||
AND (
|
||||
j.lease_until > clock_timestamp()
|
||||
OR (j.lease_until <= clock_timestamp() AND j.attempts_started = j.max_attempts
|
||||
AND sqlc.arg(error_code)::text = 'retry_exhausted')
|
||||
)
|
||||
AND a.id = j.attachment_id AND a.pipeline_version = j.pipeline_version AND a.status = 'processing'
|
||||
RETURNING j.*
|
||||
), recorded AS (
|
||||
UPDATE attachment_attempts AS attempt
|
||||
SET outcome = 'failed', error_code = sqlc.arg(error_code)::text, finished_at = failed.updated_at
|
||||
FROM failed
|
||||
WHERE attempt.job_id = failed.id AND attempt.lease_generation = failed.lease_generation
|
||||
)
|
||||
UPDATE attachments AS a
|
||||
SET status = 'failed', error_code = sqlc.arg(error_code)::text, updated_at = failed.updated_at
|
||||
FROM failed WHERE a.id = failed.attachment_id;
|
||||
|
||||
-- name: ReplayFailed :execrows
|
||||
WITH replayed AS (
|
||||
UPDATE attachment_jobs AS j
|
||||
SET max_attempts = j.attempts_started + sqlc.arg(extra_attempts)::int,
|
||||
next_attempt_at = clock_timestamp(), updated_at = clock_timestamp()
|
||||
FROM attachments AS a
|
||||
WHERE j.id = sqlc.arg(job_id)
|
||||
AND a.id = j.attachment_id AND a.pipeline_version = j.pipeline_version
|
||||
AND a.status = 'failed' AND a.staging_key IS NOT NULL
|
||||
AND j.lease_until IS NULL AND j.next_attempt_at IS NULL
|
||||
AND NOT EXISTS (
|
||||
SELECT 1 FROM attachment_replays AS r WHERE r.request_id = sqlc.arg(request_id)::uuid
|
||||
)
|
||||
RETURNING j.*
|
||||
), recorded AS (
|
||||
INSERT INTO attachment_replays (request_id, job_id, operator_id, reason, extra_attempts)
|
||||
SELECT sqlc.arg(request_id)::uuid, id, sqlc.arg(operator_id), sqlc.arg(reason), sqlc.arg(extra_attempts)
|
||||
FROM replayed
|
||||
), event AS (
|
||||
INSERT INTO outbox (job_id, payload, available_at, dedup_key)
|
||||
SELECT id, attachment_job_payload(id, attachment_id, pipeline_version, attempts_started + 1, updated_at),
|
||||
next_attempt_at, 'replay/' || id || '/' || sqlc.arg(request_id)::uuid
|
||||
FROM replayed
|
||||
)
|
||||
UPDATE attachments AS a
|
||||
SET status = 'queued', error_code = NULL, updated_at = replayed.updated_at
|
||||
FROM replayed WHERE a.id = replayed.attachment_id;
|
||||
|
||||
-- name: GetAttachmentJobState :one
|
||||
SELECT
|
||||
j.id, j.attachment_id, j.pipeline_version,
|
||||
CASE WHEN a.pipeline_version = j.pipeline_version THEN a.status ELSE 'superseded' END::text AS status,
|
||||
j.lease_generation,
|
||||
j.lease_until,
|
||||
coalesce(j.lease_until <= clock_timestamp(), false)::boolean AS lease_expired,
|
||||
j.attempts_started,
|
||||
j.max_attempts,
|
||||
j.next_attempt_at
|
||||
FROM attachment_jobs AS j
|
||||
JOIN attachments AS a ON a.id = j.attachment_id
|
||||
WHERE j.id = sqlc.arg(job_id)::uuid;
|
||||
|
||||
-- name: ListFailedAttachmentJobs :many
|
||||
SELECT j.id, j.attachment_id, j.attempts_started, a.error_code, a.updated_at
|
||||
FROM attachment_jobs j
|
||||
JOIN attachments a ON a.id = j.attachment_id AND a.pipeline_version = j.pipeline_version
|
||||
WHERE a.status = 'failed'
|
||||
ORDER BY a.updated_at DESC, j.id
|
||||
LIMIT sqlc.arg(batch_size);
|
||||
|
||||
-- name: LockAttachmentJobForPipeline :one
|
||||
SELECT j.*
|
||||
FROM attachment_jobs j
|
||||
WHERE j.attachment_id = $1 AND j.pipeline_version = $2
|
||||
FOR UPDATE;
|
||||
|
||||
-- name: LockAttachmentJobForReplay :one
|
||||
SELECT j.*
|
||||
FROM attachment_jobs j WHERE j.id = $1
|
||||
FOR UPDATE;
|
||||
|
||||
-- name: ClearStagingReference :execrows
|
||||
UPDATE attachments
|
||||
SET object_key = CASE WHEN status = 'failed' AND object_key = staging_key THEN NULL ELSE object_key END,
|
||||
staging_key = NULL
|
||||
WHERE id = sqlc.arg(id) AND staging_key = sqlc.arg(staging_key);
|
||||
|
||||
-- name: GetAttachmentByStagingKey :one
|
||||
SELECT * FROM attachments WHERE staging_key = sqlc.arg(staging_key)::text;
|
||||
@@ -0,0 +1,3 @@
|
||||
-- name: CreateChatMember :exec
|
||||
INSERT INTO chat_members(chat_id, user_id)
|
||||
VALUES ($1, $2);
|
||||
@@ -0,0 +1,34 @@
|
||||
-- name: CreateChat :one
|
||||
INSERT INTO chats (title)
|
||||
VALUES ($1)
|
||||
RETURNING id, title, created_at;
|
||||
|
||||
|
||||
-- name: GetChatByID :one
|
||||
|
||||
SELECT c.id, c.title, c.created_at, array_agg(cm.user_id)::uuid[] AS member_ids
|
||||
FROM chats c
|
||||
JOIN chat_members cm ON cm.chat_id = c.id
|
||||
WHERE c.id = $1
|
||||
GROUP BY c.id;
|
||||
|
||||
|
||||
-- name: GetChatsByUserID :many
|
||||
SELECT c.id, c.title, c.created_at, array_agg(cm.user_id)::uuid[] AS member_ids
|
||||
FROM chats c
|
||||
JOIN chat_members cm ON cm.chat_id = c.id
|
||||
WHERE EXISTS (SELECT 1
|
||||
FROM chat_members membership
|
||||
WHERE membership.chat_id = c.id
|
||||
AND membership.user_id = sqlc.arg(user_id))
|
||||
AND (
|
||||
sqlc.narg(after_created_at)::timestamptz IS NULL
|
||||
OR (c.created_at, c.id) >
|
||||
(
|
||||
sqlc.narg(after_created_at)::timestamptz,
|
||||
sqlc.narg(after_id)::uuid
|
||||
)
|
||||
)
|
||||
GROUP BY c.id
|
||||
ORDER BY c.created_at ASC, c.id ASC
|
||||
LIMIT sqlc.arg(limit_count);
|
||||
@@ -0,0 +1,22 @@
|
||||
-- name: InsertMessageIdempotency :execrows
|
||||
INSERT INTO message_idempotency(user_id, chat_id, idempotency_key, message_id)
|
||||
VALUES ($1, $2, $3, $4)
|
||||
ON CONFLICT (user_id, chat_id, idempotency_key) DO NOTHING;
|
||||
|
||||
|
||||
-- name: GetIdempotentMessage :one
|
||||
SELECT m.id, m.chat_id, m.sender_id, m.text, m.created_at
|
||||
FROM message_idempotency AS i
|
||||
JOIN messages AS m ON m.id = i.message_id
|
||||
WHERE i.user_id = $1
|
||||
AND i.chat_id = $2
|
||||
AND i.idempotency_key = $3;
|
||||
|
||||
-- name: LockMessageRequest :exec
|
||||
SELECT pg_advisory_xact_lock(hashtextextended(
|
||||
'message/' || (sqlc.arg(user_id)::uuid)::text || '/' ||
|
||||
(sqlc.arg(chat_id)::uuid)::text || '/' || sqlc.arg(idempotency_key)::text, 0));
|
||||
|
||||
-- name: DeleteExpiredMessageIdempotency :exec
|
||||
DELETE FROM message_idempotency
|
||||
WHERE user_id = $1 AND chat_id = $2 AND idempotency_key = $3 AND expires_at <= now();
|
||||
@@ -0,0 +1,19 @@
|
||||
-- name: CreateMessage :one
|
||||
INSERT INTO messages (chat_id, sender_id, text)
|
||||
VALUES ($1, $2, $3)
|
||||
RETURNING *;
|
||||
|
||||
-- name: GetMessagesByChatID :many
|
||||
|
||||
SELECT id, chat_id, sender_id, text, created_at
|
||||
FROM messages
|
||||
WHERE chat_id = sqlc.arg(chat_id)
|
||||
AND (
|
||||
sqlc.narg(after_created_at)::timestamptz IS NULL
|
||||
OR (created_at, id) >
|
||||
(sqlc.narg(after_created_at)::timestamptz,
|
||||
sqlc.narg(after_id)::uuid
|
||||
)
|
||||
)
|
||||
ORDER BY created_at ASC, id ASC
|
||||
LIMIT sqlc.arg(limit_count);
|
||||
@@ -0,0 +1,16 @@
|
||||
-- name: CreateMessagesAttachments :execrows
|
||||
INSERT INTO messages_attachments (message_id, attachment_id, chat_id, position)
|
||||
SELECT $1, $2, $3, $4
|
||||
FROM attachments
|
||||
WHERE id = $2 AND chat_id = $3;
|
||||
|
||||
-- name: GetMessagesAttachments :many
|
||||
SELECT attachment_id, position
|
||||
FROM messages_attachments
|
||||
WHERE message_id = sqlc.arg(message_id)
|
||||
ORDER BY position ASC;
|
||||
|
||||
-- name: LockAttachmentsForMessage :many
|
||||
SELECT id, status FROM attachments
|
||||
WHERE chat_id = sqlc.arg(chat_id) AND id = ANY(sqlc.arg(attachment_ids)::uuid[])
|
||||
ORDER BY id FOR SHARE;
|
||||
@@ -0,0 +1,50 @@
|
||||
-- name: CreateInitialOutboxEvent :exec
|
||||
INSERT INTO outbox (job_id, payload, available_at, dedup_key)
|
||||
SELECT id, attachment_job_payload(id, attachment_id, pipeline_version, 1, created_at)
|
||||
|| jsonb_strip_nulls(jsonb_build_object(
|
||||
'traceparent', NULLIF(sqlc.arg(traceparent)::text, ''),
|
||||
'tracestate', NULLIF(sqlc.arg(tracestate)::text, '')
|
||||
)),
|
||||
next_attempt_at, 'initial/' || id
|
||||
FROM attachment_jobs WHERE id = sqlc.arg(job_id)::uuid;
|
||||
|
||||
|
||||
-- name: EnqueueExpiredLeases :many
|
||||
WITH expired AS (
|
||||
SELECT j.*, 'recovery/' || j.id || '/' || j.lease_generation AS recovery_key
|
||||
FROM attachment_jobs AS j
|
||||
JOIN attachments AS a ON a.id = j.attachment_id AND a.pipeline_version = j.pipeline_version
|
||||
WHERE a.status = 'processing' AND j.lease_until <= clock_timestamp()
|
||||
AND NOT EXISTS (
|
||||
SELECT 1 FROM outbox AS o WHERE o.dedup_key = 'recovery/' || j.id || '/' || j.lease_generation
|
||||
)
|
||||
ORDER BY j.lease_until, j.id
|
||||
LIMIT sqlc.arg(batch_size)
|
||||
FOR UPDATE OF j SKIP LOCKED
|
||||
)
|
||||
INSERT INTO outbox (job_id, payload, dedup_key)
|
||||
SELECT e.id, attachment_job_payload(e.id, e.attachment_id, e.pipeline_version,
|
||||
LEAST(e.attempts_started + 1, e.max_attempts), clock_timestamp())
|
||||
|| COALESCE((
|
||||
SELECT jsonb_strip_nulls(jsonb_build_object(
|
||||
'traceparent', o.payload->>'traceparent', 'tracestate', o.payload->>'tracestate'
|
||||
))
|
||||
FROM outbox AS o WHERE o.job_id = e.id
|
||||
ORDER BY o.created_at DESC, o.id DESC LIMIT 1
|
||||
), '{}'::jsonb),
|
||||
e.recovery_key
|
||||
FROM expired AS e
|
||||
ON CONFLICT (dedup_key) DO NOTHING
|
||||
RETURNING *;
|
||||
|
||||
-- name: NextPendingOutboxEvent :one
|
||||
SELECT id, job_id, payload, available_at, sent_at, created_at, dedup_key
|
||||
FROM outbox
|
||||
WHERE sent_at IS NULL AND available_at <= statement_timestamp()
|
||||
ORDER BY available_at, id
|
||||
LIMIT 1
|
||||
FOR UPDATE SKIP LOCKED;
|
||||
|
||||
-- name: MarkOutboxEventSent :execrows
|
||||
UPDATE outbox SET sent_at = clock_timestamp()
|
||||
WHERE id = sqlc.arg(id) AND sent_at IS NULL;
|
||||
@@ -0,0 +1,21 @@
|
||||
|
||||
-- name: CreateUser :one
|
||||
|
||||
INSERT INTO users (username, display_name, password_hash)
|
||||
VALUES ($1, $2, $3)
|
||||
RETURNING id, username, display_name, password_hash, created_at;
|
||||
|
||||
|
||||
|
||||
-- name: GetUserByID :one
|
||||
|
||||
SELECT id, username, display_name, password_hash, created_at
|
||||
FROM users
|
||||
WHERE id = $1;
|
||||
|
||||
|
||||
|
||||
-- name: GetUserByUsername :one
|
||||
SELECT id, username, display_name, password_hash, created_at
|
||||
FROM users
|
||||
WHERE username = $1;
|
||||
@@ -0,0 +1,30 @@
|
||||
-- name: SupersedeAttachmentJob :exec
|
||||
WITH superseded AS (
|
||||
UPDATE attachment_jobs
|
||||
SET lease_until = NULL, next_attempt_at = NULL, updated_at = clock_timestamp()
|
||||
WHERE id = sqlc.arg(job_id)::uuid
|
||||
RETURNING id, updated_at
|
||||
)
|
||||
UPDATE attachment_attempts AS attempt
|
||||
SET outcome = 'superseded', error_code = 'superseded', finished_at = superseded.updated_at
|
||||
FROM superseded
|
||||
WHERE attempt.job_id = superseded.id AND attempt.finished_at IS NULL;
|
||||
|
||||
-- name: QueueAttachmentUpgrade :one
|
||||
UPDATE attachments
|
||||
SET pipeline_version = 2, status = 'queued', staging_key = sqlc.arg(source_key)::text,
|
||||
object_key = NULL, error_code = NULL, updated_at = clock_timestamp()
|
||||
WHERE id = sqlc.arg(id)::uuid AND pipeline_version = 1
|
||||
RETURNING *;
|
||||
|
||||
-- name: GetPipelineJobID :one
|
||||
SELECT id FROM attachment_jobs
|
||||
WHERE attachment_id = sqlc.arg(attachment_id)::uuid AND pipeline_version = sqlc.arg(pipeline_version)::int;
|
||||
|
||||
-- name: GetPublishedOriginalKey :one
|
||||
SELECT published_original_key FROM attachment_jobs
|
||||
WHERE attachment_id = sqlc.arg(attachment_id)::uuid AND pipeline_version = sqlc.arg(pipeline_version)::int
|
||||
AND published_original_key IS NOT NULL;
|
||||
|
||||
-- name: IsPublishedOriginalKey :one
|
||||
SELECT EXISTS (SELECT 1 FROM attachment_jobs WHERE published_original_key = sqlc.arg(object_key)::text);
|
||||
@@ -0,0 +1,39 @@
|
||||
# Приёмка ЛР 6 на «4», 16 ноября 2026
|
||||
|
||||
Все девять проверок получили PASS по итогам отдельных прогонов на
|
||||
`highload-lab6-student`. Данные и секреты синтетические, тома выделены
|
||||
при первом запуске. API 28106, Prometheus 29196, Tempo 23206, Loki 23106,
|
||||
Grafana 23006. Docker Engine 28.4.0 на aarch64, 8 CPU и 3,83 GiB RAM.
|
||||
Использован Python 3.14.6. Команды и инварианты есть в [инструкции](../../student-tests/lab6/README.md).
|
||||
[Сводка](summary.json) указывает успешный архив для каждого сценария.
|
||||
|
||||
| Проверка | Наблюдение |
|
||||
| --- | --- |
|
||||
| Smoke и Grafana | `make check` и `make test` прошли с включённой телеметрией. У provisioning есть три datasource. Запросы опубликованных панелей успешно исполнил Prometheus. После обновления dashboard проверено также выражение с нулём при отсутствии 5xx. |
|
||||
| Сквозная трасса | 43 реальных спана в Tempo. Входящий W3C-parent ведёт к API upload, дальше к двум разным producer → consumer парам. Первая попытка дала `s3_unavailable`, вторая привела ту же job к ready. Видны дочерние S3 и PostgreSQL. Обе outbox-записи сохранили один trace ID и `tracestate`. Исходник проверен побайтно. |
|
||||
| Логи и приватность | Loki показал совпадающие trace/span IDs для API и worker и фактический request ID API. Проверено 58 записей в контрольной выборке Loki, трасса загрузки и четыре трассы регистрации, входов и сообщения. Контрольные секретные значения отсутствовали. |
|
||||
| Метрики | Согласованные buckets/count/sum для HTTP, jobs и time-to-ready. Для смешанной бизнес-выборки p50/p95/p99 HTTP составили 2,20 / 209,09 / 1000 мс. Prometheus использовал HTTP-гистограмму без health. Восемь новых чатов дали прирост счётчика на одном шаблоне пути. Присутствуют queue, DB pool, worker saturation, CPU и память. |
|
||||
| Worker остановлен | Три `202` оставили глубину 3. Первая выборка после порога показала возраст 18,59 с. `GrowingQueueLag` стал firing через 39,82 с от остановки. После запуска все три исходника сохранились и стали ready, глубина 0, resolved через 9,04 с. |
|
||||
| Чтение PostgreSQL заблокировано | Под `AccessExclusiveLock` на `public.users` API за LB ответил 503 на 23 проверочных запроса. Readiness оставался доступен. Trace 503 содержит PostgreSQL-ошибку, Loki показывает связанный API-лог. `HighAPIErrorRate` стал firing через 32,73 с от блокировки, resolved через 23,81 с после её снятия. |
|
||||
| Loki остановлен на 34 с | 278 чтений и 8 записей успешно подтверждены. Все 8 сообщений остались в истории. Счётчик неэкспортированных логов вырос на 96, впервые через 8,69 с. Очередь достигла 16 из 16, пиковый RSS 88,55 MiB при лимите 256 MiB. Collector сохранил тот же процесс. После запуска Loki видны предаварийный и новый коррелированные логи. |
|
||||
|
||||
Лимиты режима с Loki указаны в конфигурации и измерены через Prometheus.
|
||||
Очередь 16 пакетов, retry до 5 с, memory limiter 96 MiB. Подтверждён
|
||||
[сохранённый лог и точки монтирования](retention.json). Проверка истории через
|
||||
физический рестарт Prometheus и Tempo в этот прогон не входила. Короткие
|
||||
искусственные окна не подтверждают выполнение SLO за 24 часа.
|
||||
|
||||
Первые прогоны дали полезные сбои. Первую проверку smoke остановила неполная
|
||||
тестовая копия. Два сценария алертов остановил конфликт полей JSONL. В следующем
|
||||
прогоне база GrowingQueueLag ещё не успела успокоиться после retry. Тест теперь
|
||||
дожидается фактического inactive. В прогретом опыте Loki RSS подошёл к прежнему
|
||||
лимиту 192 MiB. Автор поднял лимит контейнера до 256 MiB, сохранив ограничения
|
||||
очереди и повторов. В первой попытке с новым лимитом одна API-реплика вернула
|
||||
503 за 1,09 с одновременно со стартом второго Compose. Причинность с Loki
|
||||
не установлена. [Поля API/LB для 503](loki-503-api-lb.txt) сохранены. Повтор
|
||||
с теми же строгими проверками прошёл после остановки второго стенда.
|
||||
|
||||
[Сквозной прогон и DB](final.jsonl.gz) · [Worker](worker-rerun.jsonl.gz) ·
|
||||
[Актуальный dashboard и smoke](loki-final.jsonl.gz) · [Успешный опыт Loki](loki-quiet.jsonl.gz) ·
|
||||
[Первичные результаты](acceptance-resumed.jsonl.gz) · [Образы](image-ids.txt) ·
|
||||
[Исходники и актуальные конфиги](source-sha256-latest.txt).
|
||||
Binary file not shown.
Binary file not shown.
@@ -0,0 +1,5 @@
|
||||
highload-lab6-api:latest sha256:f20080b5068ac0caf9c9ff5d495413bfe0e8230f2cfd920d2bfb080380b34447
|
||||
highload-lab6-attachment-cleanup:latest sha256:e814e8c21686a66cb87921130182c80d83f1b82414758ed421dd965793dc8159
|
||||
highload-lab6-relay:latest sha256:89bee09ca623bc36b9ba001c6beefbdbb5a2ebc5a86fc9a20101bdafdea91a40
|
||||
highload-lab6-worker:latest sha256:dad64c91ff7869ef650df2cdc312397d071a881557a25a5cf2b4101a3f89f07c
|
||||
highload-lab6-migrate:latest sha256:b424420c3d12a67c0f867393146fab6c8a193ce49c3d5e0fabf6934ff8915904
|
||||
@@ -0,0 +1,2 @@
|
||||
[16/Nov/2026:06:34:22.540] api api_replicas/api2 0/0/43/1247/1290 503 318 - - ---- 1/1/0/0/0 0/0 "GET /api/v1/users/me HTTP/1.1"
|
||||
2026-11-16T06:34:23.832489586Z 2026/11/16 06:34:23 INFO http request method=GET route=/api/v1/users/me status=503 duration=1.092129709s
|
||||
Binary file not shown.
Binary file not shown.
@@ -0,0 +1,25 @@
|
||||
{
|
||||
"loki_log_stored_before_restart_and_read_after": [
|
||||
{
|
||||
"timestamp_ns": "1794810962234573465",
|
||||
"event": "http.request",
|
||||
"trace_id": "d4ad4c8543c10b091e760dbfe866a291",
|
||||
"span_id": "5b0971b0e3ce60ae",
|
||||
"request_id": "55de0f79-ffb2-4d17-ab69-555362ed7f47"
|
||||
}
|
||||
],
|
||||
"named_volume_mounts": {
|
||||
"prometheus": [
|
||||
"/prometheus"
|
||||
],
|
||||
"loki": [
|
||||
"/loki"
|
||||
],
|
||||
"tempo": [
|
||||
"/var/tempo"
|
||||
],
|
||||
"grafana": [
|
||||
"/var/lib/grafana"
|
||||
]
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,134 @@
|
||||
2658e84d6a062552303e04e41e45bd0b5fa1f6a75e90158f616d60fedbef3b3d cmd/attachment-cleanup/main.go
|
||||
3c462c48f6373570e19d9b63b87662e7ee8a135acbb80c97d0052d5610937354 cmd/migrate/roundtrip_test.go
|
||||
f7438f00679b09125cdefa813ad5a49a8ca88004b3b1535b7112700536ab5000 cmd/migrate/async_attachments_test.go
|
||||
ac1707fdcbbfccb1a9ed678da66365c9bc9e3526ea751b7749ad512bf3a6c752 cmd/migrate/postgres_test.go
|
||||
6e9d413a86f53cf5f2e83d5ed0308af9b2c33e69fcb3d061b19bf480e352f257 cmd/migrate/main.go
|
||||
a88f2252d70caf644904331182dc3afd4df48b825a0329a82482798ec90b5d46 cmd/migrate/main_test.go
|
||||
8545c84b30af7c7e58889e1102e3e3669307294e1d99733ba153235803b06ebe cmd/api/main.go
|
||||
75c0ca950b85eaee3c0802e3001c61ba6436874b3be12cd38a5feb907b91528e cmd/api/poolstats.go
|
||||
662d72dccac5061170bea781de5e6801fc93c4c8ad54db16ee145631eed4735e cmd/jobs/main.go
|
||||
e9cf006fdce4dd646571bebc51b6ec9d96f13b70a92af463de761a0d2069772b cmd/worker/main.go
|
||||
49fdc6844992a49c33dea95dfa5f55bb9684f8f35387485bcff5db9d359464a3 cmd/relay/main.go
|
||||
ad56ecd0e8b88e0665cdc144aaa5d934ea7f48047cd702ce6f8832b6cf0bded3 db/migrations/00006_allow_attachment_only_messages.sql
|
||||
a42e52ab05bc89b557a5bc6b3f0215883d99c4a039a8b32d6c19e66b2f769508 db/migrations/embed.go
|
||||
e248c25d6af34cfed78976e349be4bee3d1987cd4cdfb8293c7aef07a9b2a92b db/migrations/00007_async_attachments.sql
|
||||
b90ec5afff3a3680440e6034bfa12b1ded9fa1c1d5a85e4374e363306905092f db/migrations/00008_versioned_pipeline.sql
|
||||
033d8884cecb50b5067453a97f2d7c51e51331a1911d9ddf8687437afa59443c db/migrations/00005_attachments.sql
|
||||
1b666cee95b9eb1210b0a303de95994bd3c24628e9d9fdb1e1f3318edb756c3d db/migrations/00003_drop_session_table.sql
|
||||
e7b3dd5a056a702dfde7acba6666430477db0c7e03ae03fb270e145d6303b9b2 db/migrations/00004_message_idempotency.sql
|
||||
e0680838294b93f916d727a3a20d83aca7c3f74077202c9696b1fc73cda14649 db/migrations/00002_add_created_at_defaults.sql
|
||||
48db014273f78be0f6eb97fc521cccc54f851dc99e3688d480d3a6ff7b9b6bbe db/migrations/00001_init_schema.sql
|
||||
c80b6e50c321ff94317545ea2f9fdee1dfd69f564d8638f7b111c8ec091ef03a db/queries/chat_members.sql
|
||||
0d73a6d46f4d4a1b0af37b750a208c07a38249f76974b41748f32c1949f990df db/queries/users.sql
|
||||
5436a4d87e9a9e8a63a8845651dedebd17729ccdec6c6739ed209bed27dc3bd3 db/queries/messages_attachments.sql
|
||||
cf38c8c9eee8d62983c05f5bf03f4804cf2a101dae19851ed6e1770e5a0ebc01 db/queries/message_idempotence.sql
|
||||
003b2cd200d6c37ed74b7f8b77a6643e301c39662b90bfc367e8534784c77afd db/queries/attachment_variants.sql
|
||||
f6f6382f27d698b1c2af2305d45699226ef10806b938b3dca78ec9087a622b7b db/queries/messages.sql
|
||||
a73568ebdc2d8f12845bdacf19a7e30754d6eede2013027e61b315b0a96e6432 db/queries/versioned_pipeline.sql
|
||||
8c52c762da78eb3a3eb7402b17ec87bd7fc2e1387f6ddb8e297b4f2d75beca7b db/queries/chats.sql
|
||||
81b9797bf5b42f53d8712d0a153edb34544c18aafb4a928603b522c76e928a68 db/queries/outbox.sql
|
||||
2e2c3cd3b5cf3a43318209d1757be412d38fbe3b3db8d0714668cde69f577471 db/queries/attachments.sql
|
||||
58d317ac9e5f2f522b88339f098728d124728e231987c6ff8d5251a37bc33b42 db/queries/attachments_jobs.sql
|
||||
566d4bfba921ddcfa55e4731eb1e00d4dd794fcf88ae07497f3fe6d557068901 internal/database/postgres.go
|
||||
5b85a60bb0d1cb156c61bcd3c518c41c955f53c1d12da265b5e3a5dd7a397602 internal/database/redis.go
|
||||
07391d01cd0480b65cbd8de0941a8ea0797d491b38e537ea9ffe878fc80b3b5d internal/database/sqlc/attachment_variants.sql.go
|
||||
493a2ba9132786a83c28fbf86295502b624aadebff4672dcffc65dad37689513 internal/database/sqlc/models.go
|
||||
3ca71a7f007a39987cae838da64303419dd804d2d1b6227676f47177971d76cf internal/database/sqlc/db.go
|
||||
3dccf4b0ffa918ec5a3b66026b3dab4027456b499486359881000d184944802c internal/database/sqlc/chats.sql.go
|
||||
92c35b6e7da73e7ed00b274e5e9acc44bdd00c9e285a2b12f74cca8fcdfab54d internal/database/sqlc/attachments.sql.go
|
||||
a73ab2511cb912fd68db0e5b3b42c2d8e2fe73a6950f080e7104f8474e8c1e36 internal/database/sqlc/message_idempotence.sql.go
|
||||
9836433d9645e6dd3bd66273ce15a53691f79093016c416fb24ffd085e62831e internal/database/sqlc/messages.sql.go
|
||||
61d538f3e664d88248fa8d77403e7e7a72cce444e839b665590fb489d59d9dbb internal/database/sqlc/outbox.sql.go
|
||||
ee1814d651bbdfe62f0685db0d25315c44d408f145f27cf4a631f023c5f8e624 internal/database/sqlc/versioned_pipeline.sql.go
|
||||
ef74459641b72c53a531e76a830a644036dfa8534a19c393ff5c6e1994b0c1cb internal/database/sqlc/messages_attachments.sql.go
|
||||
b954abd5b5964df39c20d857a60105513af84c8df68ce1ac9421da541a7800f4 internal/database/sqlc/users.sql.go
|
||||
2706d561d8dce6e30baddd9921bda9c11ae2b237653839426aba734bf993cb7b internal/database/sqlc/attachments_jobs.sql.go
|
||||
9e8cd177f6f888c72d19857fd4748fbb2020770c1af772a463fe3d1b813b37ee internal/database/sqlc/chat_members.sql.go
|
||||
f6669338f64209afcb5fa5f7f255cb6c9155a812c29f5d8fe9330906d85d1957 internal/pipeline/worker.go
|
||||
128222674808cbad6936757c9d5a413393ba13567d954787e9172f5d77bea735 internal/pipeline/cleanup.go
|
||||
bee65136137d97bea2240411c96a93cd945b1b9145452416f7a83733a1fd28e7 internal/repository/user.go
|
||||
c1654fea2ba47734a5403d6f6cbb2fe70fbe7e6e37166ae75f338d3490981de1 internal/repository/redis/session.go
|
||||
ab56571d1d0b7d13b7d7ff868916f7e506c5a54c29e6bb211b3021ad4bf601ab internal/repository/redis/errors.go
|
||||
2f92e2808317850ca5873cc26f8e6c4d83104e8fc957a89304fc74ba2f64cbf2 internal/repository/message.go
|
||||
071285ff4d8033a6512a7256583a1cb32de782d6fd9f1696fac109927f7db699 internal/repository/session.go
|
||||
eb6770e5cdd96b16a6e9d2116af89a6dd466dfbca4f4c179fe06c30c487e53c4 internal/repository/postgres/user.go
|
||||
5cc80f3837bad98bb162c20fdf0e9cc801dc14767b18a0ba69a5a1f5ff50bf51 internal/repository/postgres/versioned_pipeline.go
|
||||
b37ffbae0c4d8f60e7d237c3c7d730cae4cf554d34446eef283e6e10ceddbba8 internal/repository/postgres/pipeline_cleanup.go
|
||||
ae82a95dbd43a872cf3e32e428e699f767008d07a18ecc041f7601eae2d7ad86 internal/repository/postgres/message.go
|
||||
53ca3c2143cd98daefa924fc4fa4df1d9cf87a120332e42727a4159b87343ea8 internal/repository/postgres/pipeline.go
|
||||
04dc48cffea2f6adad7659e60649d0f6ecd724b1ccb82f7fea78eace8624d097 internal/repository/postgres/utils.go
|
||||
683b6a5a20e0e37305d178700fb6e2e5f47b1b81acaded210b5d3eaa32e29fb1 internal/repository/postgres/attachment.go
|
||||
d757c579e5b6599ab9366aef46fd59f9652247f3d947615aa46c919746b65ea5 internal/repository/postgres/chat.go
|
||||
580e5e599f7af8a403404c6850986e88efceda6393d21e772ac108be34004ba9 internal/repository/postgres/errors.go
|
||||
62e75dbb4ed1cb966abbafc74530a676409ffa0b016a7943be5281cab1dce45c internal/repository/attachment.go
|
||||
7ef8dca39ff371f16d351b073199c25f719db22c7a525ab76ea7aca948efa157 internal/repository/chat.go
|
||||
f5afee6418a23fba5066eadd2946c99eed850d76f24e331193146165d48126bb internal/repository/attachment_tx.go
|
||||
9e3f77340ff87e41edc87380f795759701980685478737e7e4e1f9737e2634fa internal/repository/errors.go
|
||||
a5f1c30e58149bfddc645ca481a074b0024e82272a9598ec80dd46001bf98c2b internal/broker/rabbitmq.go
|
||||
d181bce0f003749d813c1254df4c8ebcc1b99ff506b61efa6899da4a499e95f4 internal/transport/http/handler/chat_types.go
|
||||
86fd65ed176e9a83dc27c1112ff8384bc5aa32a92410290fb505d715ca167775 internal/transport/http/handler/user.go
|
||||
f175a570d47ca38ce627d0bc88b6f6a2adbf648345229fa961ea0144717744f1 internal/transport/http/handler/auth.go
|
||||
540065ffc9090848fc25d560f1ad9734d4f0d45cdf0b5907a0f196f9efa8ede3 internal/transport/http/handler/message.go
|
||||
1abf04652ea0b69bd87321b95e1a921d3c5ad3c04207232c2a9408bf722e550a internal/transport/http/handler/health.go
|
||||
c94b2e6550037c1ccdd4e1e505965e7b4d85d037be180afe542f38a2b4af3001 internal/transport/http/handler/attachment.go
|
||||
e5214ebf94ac9b8f37537e1a3fc844a2e8ae4e9b5aa159fe2b6e4da9ab2342f0 internal/transport/http/handler/chat.go
|
||||
5ffcd182606a77d538efb5f1a51df23b8a324a605b5d226026048d34c409ebdb internal/transport/http/handler/auth_types.go
|
||||
02ee3031e10fbb279e020f409fbd905ae2640aa306b2c558c3f8e6dcfe9efa80 internal/transport/http/handler/health_types.go
|
||||
270a9504a3001b1dc02a1f62cdaed12d69af2471340674e50022c609643dcb34 internal/transport/http/handler/message_types.go
|
||||
a87417ccc7b9ffb468e3ac1df20d5e4adc33d47f31509e37449647ea83706d11 internal/transport/http/handler/attachment_types.go
|
||||
0c4450be649afe5edf4e8d3ac9baada79deb9e1423ace56b87aa227ebd0eb998 internal/transport/http/handler/user_types.go
|
||||
413d7fb5f8573a808c2eb4e36becd29b4d7edffe5b096b26fc07ac22a0285430 internal/transport/http/middleware/auth.go
|
||||
512da6bd2133da34c22a4f1f967b05c308f1ddcdae543a7b5910d178b4a378bc internal/transport/http/middleware/logging.go
|
||||
f6db0e807659f30b05cae91f73c70d87f359760c57110e0d25b49a65bd0394a5 internal/transport/http/middleware/context.go
|
||||
59c092950b69f5390427d0c6c1f90d2c33630d87a42bef363aed704960cb3a22 internal/transport/http/httputils/helper.go
|
||||
6617ceb365903935382cb9592a96a60956d4edbd7c4132530721ddb6ad0e93af internal/transport/http/router.go
|
||||
21f5d30de769a1817b2d77fc68f9e915d512862316bbb37c7f09baff90cf9407 internal/config/config.go
|
||||
5a917a0a844d6578ba7e24ae892ae74ed55ea848759e624b94b4356dd7fda996 internal/config/pipeline.go
|
||||
8f74bef9216a2d12aa734e04fa786eac6b31f1378fe42f729fe1b97495c10487 internal/security/password.go
|
||||
5d7b5bd8e332104c057d0bb0adf6dce5a3a5bef37d2ba086b69af0cb31ed6a95 internal/observability/sdk.go
|
||||
32fc41d7bc89d084178f0deae70f1298a896f133ab46f7fb62394ac9f34741e5 internal/observability/signals.go
|
||||
283c352bfde58afa30ec26997128ac6d6ceebc5e057ef3c112bc6d32f57ab18c internal/observability/propagation.go
|
||||
c52d3f36aa750b6dcba2b720e342fc1fef69eb99130be697baf93df171c507c4 internal/observability/gauges.go
|
||||
f8ed8e5486e9278f18a2beb6abb8952aa15d946d2691ecbea5226db9c54e0e71 internal/storage/s3/client.go
|
||||
6f1f5a7dbd93f65f8101ba80b6fa17bab964744d237746b3ae4ac3650cc9b52c internal/storage/s3/errors.go
|
||||
ef50236baefe3e3aea8ca64264ae3c2e5fc89c245b9b1d7cfd207d354f0be2c9 internal/storage/object_cleanup_storage.go
|
||||
15ca1257b9c42f64a42f19a4bbb487e9d2950aed2fdf87959fe6c2d763998b64 internal/storage/object_writer.go
|
||||
21a5622cd2655663fc24ee485013c1a917279990fd128a5d406b59f5f0ee67cd internal/storage/errors.go
|
||||
c541dac69672bcd7b89439ae45d63c3e835218f700be7c48c9130a5beeeddbdd internal/imageproc/processor.go
|
||||
336bbb6ffd99659e77a4f21c0eed49d8a20dd2b79da8fcc045558fdb34acb3ef internal/model/attachment_job.go
|
||||
4bfd2d78e72f1c80b12541f2abc8500613e8f4d101575bd5d74fac22b52e3617 internal/model/user.go
|
||||
4cebc4963e2eb104e85a1df6e8a93d6745c9ae42425df2d04bf6319e1d17b12d internal/model/message.go
|
||||
b455d730e4e3efc95c6d1fe74ab43d86153e4bf863c09efd36bfe57efc669bc2 internal/model/session.go
|
||||
7ee03b8a3473600e01b83a61369c7212a7cfab4e1a3f982e297ee36dcf9fc9be internal/model/attachment.go
|
||||
e0213f8360ea106b3627aa50cc87710153ff8e353475b42c2becdb9a32592660 internal/model/chat.go
|
||||
542051ef3eeff15d340979a615a7e0da3d21ed7ed93bea2547ab52d8e303b5e2 internal/service/user.go
|
||||
71b8960640866e515e9060159af750c418d97a6373d2616c3b59b764172aa54c internal/service/auth.go
|
||||
c4e66bb48f7cd26de6fedf4ca7cf751c815dcab30e9442de61f7fe000d3c2367 internal/service/attachment_cleanup.go
|
||||
9556a18a5679d7e44b5cdcf329488185d2b90cadb3180af926df4bb6ab3f9d0d internal/service/message.go
|
||||
390320fa296d9637630e6ac0d90c1229002b578a29186cad414896d5f4d1b8a6 internal/service/helper.go
|
||||
93d786691f371a5e6ad8acb4da90141351d8f4f76df7ebdf70857d7e104a91d9 internal/service/attachment.go
|
||||
bc7c8bea0066d8bbd926f78d2c100f0bd9a48c29fb542c5cfa3f2ae415299ef5 internal/service/chat.go
|
||||
f7dbf60b14f9e82cc018bdd29482d71786a30c32bfb5d1ed1808342c9fd36cdb internal/service/attachment_reconcile.go
|
||||
fb5aa4f7be76af14ab81629eceb02e1c3a1bafd8bfeb036e7ee5ccd3940d64da internal/service/errors.go
|
||||
c607c83cfeedd8e29a678023b133299dcb5ff754362717d43667db3d84641601 internal/fault/pause.go
|
||||
baa4b0b6fa4bd2a76b194b672ef2ca8575858ef6abc7ddfbede3eec3534bc8ce infra/s3/bootstrap.sh
|
||||
895a178e7354de61a5d40a2fe18b7b7c6d1d3d5faf0e5a5faa72783e7b983cb8 infra/haproxy.cfg
|
||||
ca1f38f2f75f41d1eab7ff2200f17af9fb0999e32eb3c8fd32861966810dfd8d infra/otel/loki.yaml
|
||||
7773dc948acfde585519bee38c555a8b19710265ed93ce9ff9ad429892b89df7 infra/otel/alerts.yaml
|
||||
448a5ad110746d41d93aaeb5b2cd6b070cf765427e8ef77563a661e09233f0e0 infra/otel/prometheus.yaml
|
||||
dc6598890e91c1992c0fa536eddc9f5bc2e786cfc327665551729983e0a54b24 infra/otel/runbook.md
|
||||
f14185fb7c029a3d35388c19072e56a6881aebb1b83e3a937a44e71fa642acb6 infra/otel/grafana/provisioning/datasources/messenger.yaml
|
||||
673ac92a82c26caa47723a3c5359204417c85330983ff13f4eb9c5fcab013af1 infra/otel/grafana/provisioning/dashboards/messenger.yaml
|
||||
9937aba6adb530da8a0928d37918847bbaef100a445db07855ed5467fd1b0281 infra/otel/grafana/dashboards/messenger.json
|
||||
4a8937430ba638c32dc8d97c36e93ac19f0b3785099f2df1e22b234d109b0f95 infra/otel/README.md
|
||||
427f54f25412124eb82dfe169b3cb2ee7e018bc4256c7b839a6814722df06c12 infra/otel/collector.yaml
|
||||
3185ded54a1221207ecec0ee3c496b3d45614eebab63471e948a32a51a41d8c2 infra/otel/tempo.yaml
|
||||
0e804fa51debba4c38d3cc84abf4704897665e0335b272060363f03aadd6354f infra/README.md
|
||||
b75281f51a4c0d83904bf15c509cc79c6e8b177d05c6c11a412b6ac16d59cf85 infra/imagemagick/policy.xml
|
||||
1410205b161cbbed45dd024c37ec45b550751a54a4065cecf23a81e4a755d837 compose.yaml
|
||||
7584dc5927d3324e0f415b913ad8f756e9baa863fe02e369bffc34d777eaa9f9 Dockerfile
|
||||
fd7ac9491c0c3c663b083c41a762fb38bc2d212fa4a2ed3e6a0702af72108709 Dockerfile.example
|
||||
84b41bac810e3add9996a3a0120b3e611f2e32adbea4e447a580cbc0f88d6c4b Dockerfile.migrate
|
||||
a5b5c9aea21573f57e604428196c5ce3b7b5c0e15ecae81c134b3b6f8950ec26 Dockerfile.worker
|
||||
a59216b0859592fa49353942fa4ebd16d9e098ecdd41147f8f6b52fe1804835d go.mod
|
||||
b01795b85f5d8c4a0a48925e956e2bc747bb37a96b5e2f8531476a842ad5d7b9 go.sum
|
||||
@@ -0,0 +1,241 @@
|
||||
{
|
||||
"results": {
|
||||
"scoped-compose": {
|
||||
"status": "PASS",
|
||||
"evidence": "loki-quiet.jsonl.gz",
|
||||
"t_s": 4.056418
|
||||
},
|
||||
"backend-readiness": {
|
||||
"status": "PASS",
|
||||
"evidence": "loki-quiet.jsonl.gz",
|
||||
"t_s": 19.38088
|
||||
},
|
||||
"public-smoke": {
|
||||
"status": "PASS",
|
||||
"evidence": "loki-final.jsonl.gz",
|
||||
"t_s": 7.548447
|
||||
},
|
||||
"provisioning": {
|
||||
"status": "PASS",
|
||||
"evidence": "loki-final.jsonl.gz",
|
||||
"t_s": 7.668063
|
||||
},
|
||||
"trace-retry": {
|
||||
"status": "PASS",
|
||||
"evidence": "final.jsonl.gz",
|
||||
"t_s": 71.889912
|
||||
},
|
||||
"metrics": {
|
||||
"status": "PASS",
|
||||
"evidence": "final.jsonl.gz",
|
||||
"t_s": 79.172357
|
||||
},
|
||||
"worker-outage": {
|
||||
"status": "PASS",
|
||||
"evidence": "worker-rerun.jsonl.gz",
|
||||
"t_s": 68.043544
|
||||
},
|
||||
"db-outage": {
|
||||
"status": "PASS",
|
||||
"evidence": "final.jsonl.gz",
|
||||
"t_s": 137.340855
|
||||
},
|
||||
"loki-outage": {
|
||||
"status": "PASS",
|
||||
"evidence": "loki-quiet.jsonl.gz",
|
||||
"t_s": 74.885014
|
||||
}
|
||||
},
|
||||
"alerts": [
|
||||
{
|
||||
"event": "alert_transition",
|
||||
"phase": "db-outage",
|
||||
"t_s": 113.021074,
|
||||
"name": "HighAPIErrorRate",
|
||||
"state": "firing",
|
||||
"seconds": 32.72850495799503
|
||||
},
|
||||
{
|
||||
"event": "alert_transition",
|
||||
"phase": "db-outage",
|
||||
"t_s": 137.206116,
|
||||
"name": "HighAPIErrorRate",
|
||||
"state": "resolved",
|
||||
"seconds": 23.813540999995894
|
||||
},
|
||||
{
|
||||
"event": "alert_transition",
|
||||
"phase": "worker-outage",
|
||||
"t_s": 58.282739,
|
||||
"name": "GrowingQueueLag",
|
||||
"state": "firing",
|
||||
"seconds": 39.821040167007595
|
||||
},
|
||||
{
|
||||
"event": "alert_transition",
|
||||
"phase": "worker-outage",
|
||||
"t_s": 68.043475,
|
||||
"name": "GrowingQueueLag",
|
||||
"state": "resolved",
|
||||
"seconds": 9.0445497079927
|
||||
}
|
||||
],
|
||||
"db": {
|
||||
"event": "db_errors",
|
||||
"phase": "db-outage",
|
||||
"t_s": 113.025066,
|
||||
"count": 23,
|
||||
"elapsed_s": [
|
||||
1.0122738749923883,
|
||||
1.0136010000132956,
|
||||
1.0066660839947872,
|
||||
1.0647313329973258,
|
||||
1.0637415000092005,
|
||||
1.0244076670060167,
|
||||
1.0293545829918003,
|
||||
1.0129166250117123,
|
||||
1.0139167909946991,
|
||||
1.011540708001121,
|
||||
1.0182600000116508,
|
||||
1.0135567909892416,
|
||||
1.010816124995472,
|
||||
1.0129806670011021,
|
||||
1.0097749999986263,
|
||||
1.0129036669968627,
|
||||
1.0104306249995716,
|
||||
1.0163643749983748,
|
||||
1.0270365830074297,
|
||||
1.0105538750067353,
|
||||
1.0157577079953626,
|
||||
1.0105855829897337,
|
||||
1.023447416999261
|
||||
],
|
||||
"kind": "users-table exclusive lock with healthy SQL connectivity"
|
||||
},
|
||||
"histograms": {
|
||||
"messenger_http_request_duration_seconds": {
|
||||
"0.5": 0.002197183098591549,
|
||||
"0.95": 0.20909090909090886,
|
||||
"0.99": 0.9999999999999993
|
||||
},
|
||||
"messenger_job_duration_seconds": {
|
||||
"0.5": 0.375,
|
||||
"0.95": 98.99999999999999,
|
||||
"0.99": 115.8
|
||||
},
|
||||
"messenger_attachment_time_to_ready_seconds": {
|
||||
"0.5": 1.5,
|
||||
"0.95": 104.99999999999999,
|
||||
"0.99": 117.0
|
||||
}
|
||||
},
|
||||
"trace": {
|
||||
"trace_id": "117fb860af9afead75e086dd224dc3f3",
|
||||
"span_count": 43,
|
||||
"roles": {
|
||||
"server": [
|
||||
"033181f9ec0d505e"
|
||||
],
|
||||
"producer": [
|
||||
"92738b419e089622",
|
||||
"9e3979899a512f6b"
|
||||
],
|
||||
"consumer_error": [
|
||||
"0ce981115c578fdb"
|
||||
],
|
||||
"consumer_success": [
|
||||
"e4eaf51e02978a35"
|
||||
],
|
||||
"s3": [
|
||||
"d0fe4c64c8802873",
|
||||
"fbf208abc9acad68",
|
||||
"8fb1c65ab0adee8b",
|
||||
"6e0093008e42bfd5",
|
||||
"630ed562ef2ff0ff"
|
||||
],
|
||||
"postgres": [
|
||||
"edad82ff768e568d",
|
||||
"399d1cc7721a8d62",
|
||||
"9fcb2e4ec4fa5213",
|
||||
"591f6fa126cc2a86",
|
||||
"3318e29280f91e83",
|
||||
"cbcd5e82838679e4",
|
||||
"fc3a0681f8ba3a32",
|
||||
"a37814110380ba1a",
|
||||
"9f816e5ca0bbddc5",
|
||||
"e565df310156f969",
|
||||
"f37888f8e0cbcf08",
|
||||
"67730935285e41d5",
|
||||
"bb6218ee1ab0d63f",
|
||||
"5ca43f6e214a1096"
|
||||
],
|
||||
"failed_s3": [
|
||||
"4912ecf2e97dd446"
|
||||
]
|
||||
}
|
||||
},
|
||||
"privacy": {
|
||||
"event": "loki_correlation",
|
||||
"phase": "trace-retry",
|
||||
"t_s": 71.791617,
|
||||
"matched": {
|
||||
"messenger-api": {
|
||||
"timestamp_ns": "1794810562106030543",
|
||||
"trace_id": "117fb860af9afead75e086dd224dc3f3",
|
||||
"span_id": "033181f9ec0d505e",
|
||||
"request_id": "930ba702-f6ca-485f-83a2-eb2737b2d48c",
|
||||
"severity": "INFO",
|
||||
"event": "http.request"
|
||||
},
|
||||
"messenger-worker": {
|
||||
"timestamp_ns": "1794810625211766795",
|
||||
"trace_id": "117fb860af9afead75e086dd224dc3f3",
|
||||
"span_id": "e4eaf51e02978a35",
|
||||
"request_id": null,
|
||||
"severity": "INFO",
|
||||
"event": "job.finished"
|
||||
}
|
||||
},
|
||||
"examined": 58,
|
||||
"privacy_canaries": [
|
||||
"basic_credentials",
|
||||
"bearer_token",
|
||||
"filename",
|
||||
"image_body",
|
||||
"message_text",
|
||||
"registration_password",
|
||||
"rejected_basic_credentials"
|
||||
]
|
||||
},
|
||||
"loki": {
|
||||
"event": "loki_outage_verified",
|
||||
"phase": "loki-outage",
|
||||
"t_s": 74.884933,
|
||||
"phase_seconds": 34,
|
||||
"time_to_loss_s": 8.685361416995875,
|
||||
"initial_dropped": 24.0,
|
||||
"dropped_delta": 96.0,
|
||||
"peak_rss_bytes": 92856320.0,
|
||||
"peak_queue_size": 16.0,
|
||||
"hard_limit_bytes": 268435456,
|
||||
"successful_reads": 278,
|
||||
"preserved_messages": 8,
|
||||
"recovered_trace_id": "c69b20b4396c82e6384eee0e74426c58",
|
||||
"collector": {
|
||||
"id": "06e8839ea7ce66b2eb98cda8b183e1c31a671a634964fbdea1e7e871955c1465",
|
||||
"image": "otel/opentelemetry-collector-contrib:0.131.0",
|
||||
"status": "running",
|
||||
"started_at": "2026-11-16T06:33:52.029445876Z",
|
||||
"restarts": 0,
|
||||
"oom": false,
|
||||
"memory": 268435456,
|
||||
"cpu_nanos": 750000000,
|
||||
"ports": {
|
||||
"4317/tcp": null,
|
||||
"4318/tcp": null,
|
||||
"55678/tcp": null,
|
||||
"55679/tcp": null
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Binary file not shown.
@@ -0,0 +1,30 @@
|
||||
{
|
||||
"status": "PASS",
|
||||
"services": [
|
||||
"prometheus",
|
||||
"loki",
|
||||
"tempo",
|
||||
"grafana"
|
||||
],
|
||||
"old_container_ids": {
|
||||
"prometheus": "e063f54b27c1",
|
||||
"loki": "179ecc857b80",
|
||||
"tempo": "e4b0e7e2a3a0",
|
||||
"grafana": "a6db80031e16"
|
||||
},
|
||||
"new_container_ids": {
|
||||
"prometheus": "ab619a6a89e8",
|
||||
"loki": "f69a8b780008",
|
||||
"tempo": "1d8534706f8c",
|
||||
"grafana": "b6c8635afdbb"
|
||||
},
|
||||
"historical_sample_unix": 1794811077.7996218,
|
||||
"matched_before_and_after": {
|
||||
"trace_id": "a674a4438e9c0032c63b4f2756839598",
|
||||
"request_id": "0971f967-996b-4484-bcd5-49c6a8813a03",
|
||||
"server_span": "1801a6325cc09d20",
|
||||
"loki_matches": 1,
|
||||
"prom_completed_registration_counter": 1.0
|
||||
},
|
||||
"dashboard_uid": "lab6-messenger"
|
||||
}
|
||||
@@ -0,0 +1,66 @@
|
||||
module highload
|
||||
|
||||
go 1.27.1
|
||||
|
||||
require (
|
||||
github.com/aws/aws-sdk-go-v2 v1.48.0
|
||||
github.com/aws/aws-sdk-go-v2/config v1.33.18
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.21.0
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.114.10
|
||||
github.com/aws/smithy-go v1.28.9
|
||||
github.com/google/uuid v1.6.0
|
||||
github.com/jackc/pgx/v5 v5.11.0
|
||||
github.com/pressly/goose/v3 v3.26.0
|
||||
github.com/rabbitmq/amqp091-go v1.10.0
|
||||
github.com/redis/go-redis/v9 v9.23.0
|
||||
go.opentelemetry.io/otel v1.38.0
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.14.0
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.38.0
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.38.0
|
||||
go.opentelemetry.io/otel/log v0.14.0
|
||||
go.opentelemetry.io/otel/metric v1.38.0
|
||||
go.opentelemetry.io/otel/sdk v1.38.0
|
||||
go.opentelemetry.io/otel/sdk/log v0.14.0
|
||||
go.opentelemetry.io/otel/sdk/metric v1.38.0
|
||||
go.opentelemetry.io/otel/trace v1.38.0
|
||||
golang.org/x/crypto v0.57.0
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.27 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.9 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.12 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.12 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.12 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.25 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.13 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.12 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.12 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/signin v1.13.1 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.39.5 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.44.5 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.52.0 // indirect
|
||||
github.com/cenkalti/backoff/v5 v5.0.3 // indirect
|
||||
github.com/cespare/xxhash/v2 v2.3.0 // indirect
|
||||
github.com/go-logr/logr v1.4.3 // indirect
|
||||
github.com/go-logr/stdr v1.2.2 // indirect
|
||||
github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.2 // indirect
|
||||
github.com/jackc/pgpassfile v1.0.0 // indirect
|
||||
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect
|
||||
github.com/jackc/puddle/v2 v2.2.2 // indirect
|
||||
github.com/mfridman/interpolate v0.0.2 // indirect
|
||||
github.com/sethvargo/go-retry v0.3.0 // indirect
|
||||
go.opentelemetry.io/auto/sdk v1.1.0 // indirect
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.38.0 // indirect
|
||||
go.opentelemetry.io/proto/otlp v1.7.1 // indirect
|
||||
go.uber.org/atomic v1.12.0 // indirect
|
||||
go.uber.org/multierr v1.11.0 // indirect
|
||||
golang.org/x/net v0.58.0 // indirect
|
||||
golang.org/x/sync v0.23.0 // indirect
|
||||
golang.org/x/sys v0.48.0 // indirect
|
||||
golang.org/x/text v0.42.0 // indirect
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20250825161204-c5933d9347a5 // indirect
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20250825161204-c5933d9347a5 // indirect
|
||||
google.golang.org/grpc v1.75.0 // indirect
|
||||
google.golang.org/protobuf v1.36.8 // indirect
|
||||
)
|
||||
@@ -0,0 +1,162 @@
|
||||
github.com/aws/aws-sdk-go-v2 v1.48.0 h1:tFU7EJVm7MfyaXLsXdYTZ0Vd6vwt9OljCyqq0gqk0Mc=
|
||||
github.com/aws/aws-sdk-go-v2 v1.48.0/go.mod h1:oeKU5p+TQBnpeLmU42WZLWvzv+ORLn+74n4QVLnm/Z0=
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.27 h1:3dMP9eEmHVosFN1QiD24IJBKVY8m6iUfVYXqRCT118Q=
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.27/go.mod h1:KH4PKcqtjy1m0656eqQV2v53xE04ddJzuNloEKqlHkI=
|
||||
github.com/aws/aws-sdk-go-v2/config v1.33.18 h1:WC0C4jM/tewXG8DWXnoxXnusJPZTryPazqk2k/zgd2s=
|
||||
github.com/aws/aws-sdk-go-v2/config v1.33.18/go.mod h1:0kRpFgsnD59mC2bbbyOTVpaZ9RnXW/pFkqlfiwbgZaU=
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.21.0 h1:OViwhqeipkQzSHuO2eQDUCdLrI6r3nzGxFUyM8GqnCs=
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.21.0/go.mod h1:OffaVQod3fyZzhZJyPtw11WpHPF1uIHADc+/MYlXmbA=
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.9 h1:oH1E0tjN0g+ZRusbX1dsuat4FJ0z05l8hWaycSbDuNc=
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.9/go.mod h1:3ZaQGigIoaS45x9HgnLZ4sP475Fs7QoxU3IsZX/J3h4=
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.12 h1:W0yUUzKBA8SofGQ0yZafEUuZq9UxfvijRgdc6vcRQPA=
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.12/go.mod h1:xwKv6vwBNdpPFOxEm/ld4IODZK2s/1ksHvdBqLfH6Gs=
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.12 h1:lqSi8qvAnKC1vfosImdmOsuiJLr/jdybZX5299beB6Q=
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.12/go.mod h1:rqPQE2vqkW9YHx5+Gp55sx1wsTM4xtGQLl+TKMjxxrs=
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.12 h1:uvX2yaL+aMjkmnrLPPlR/zwjoKFRMoNhhJfeGjfk6qc=
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.12/go.mod h1:dicT+2eNhHHqGkxHO7LoXw3QghIUGuzgdmFqaSTopgI=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.25 h1:rKwkY3Z+fvt2W0msjdNs6uNewPmXzPayQ961oyuWhes=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.25/go.mod h1:dBuAadQKd1F4Q4ZOg8gnFmuSHMRE40Pe3IfFwbtC4lE=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.13 h1:U5HZ8FpTr6tyvulf1nT11Nw4KG1CAsXkuQGdUrIjZ/E=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.13/go.mod h1:dUfvCU1NaE+b3vqNENDiJncUQW7Djl5pmgUlcjesVqk=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.12 h1:wzAqMZ2O4+ij9HQqIrk+P9PBGacE0wOTuWB7Mo97//I=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.12/go.mod h1:/0TfyruByHsOlphzP26xDTRe2DKD0J1I1Uyy2TyP8EI=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.12 h1:H0IaQayT3TLiGIHiNnleQZTbndDlx7LGZ0NWZ5mDwjA=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.12/go.mod h1:YuJJNxaOjnknLIrscBmkLhIpnmg9LzHd25sQRKUboGU=
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.114.10 h1:WBJb2VxRzzKnuN94YWqZPiu5a5zc3IleFqgZnmVCyvU=
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.114.10/go.mod h1:8NhAIBVYkVHq0hIbbzujdoou0VkKwMdG+De9e5oX6qI=
|
||||
github.com/aws/aws-sdk-go-v2/service/signin v1.13.1 h1:ssrDA6AKfXEc3h0Pf4EI4vQ8sN5Zd136SrU9Tjr0XPc=
|
||||
github.com/aws/aws-sdk-go-v2/service/signin v1.13.1/go.mod h1:LdwHjKqPE9VCC7sTlWcOAizZx3ZJYjmfwC/EW2tIIAE=
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.39.5 h1:ANAoa+qt3/9kXnvQVgUyxcTzhzZf4DqaBS+M2DQrELU=
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.39.5/go.mod h1:vsMi+SSR7xGQXtV430LVi49qbv/R+U+upHHSp1W/+QI=
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.44.5 h1:uIOBLMhef4N16WguadoHKuBy9etdz+RKTeqzchXErAI=
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.44.5/go.mod h1:NwWmkFLtsTVyXQpbzPOlYLwEoANkdCFjJRtrNcbg/vY=
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.52.0 h1:IaBLmJWiIEtKkjXMgMbYGZk7WKXbYA8148wCMCXq9hU=
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.52.0/go.mod h1:MG6N8U9+lY4LqchrlHPqCOSr7hyxw70XU0Et2YR3eNA=
|
||||
github.com/aws/smithy-go v1.28.9 h1:zkZ058OpNX0iGTK24b+ZUEQSMc11CBA6oO7xtKZfzkY=
|
||||
github.com/aws/smithy-go v1.28.9/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc=
|
||||
github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs=
|
||||
github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c=
|
||||
github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA=
|
||||
github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0=
|
||||
github.com/cenkalti/backoff/v5 v5.0.3 h1:ZN+IMa753KfX5hd8vVaMixjnqRZ3y8CuJKRKj1xcsSM=
|
||||
github.com/cenkalti/backoff/v5 v5.0.3/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw=
|
||||
github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
|
||||
github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
|
||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
|
||||
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
|
||||
github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
|
||||
github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI=
|
||||
github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
|
||||
github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
|
||||
github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
|
||||
github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek=
|
||||
github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps=
|
||||
github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
|
||||
github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
|
||||
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
|
||||
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||
github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.2 h1:8Tjv8EJ+pM1xP8mK6egEbD1OgnVTyacbefKhmbLhIhU=
|
||||
github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.2/go.mod h1:pkJQ2tZHJ0aFOVEEot6oZmaVEZcRme73eIFmhiVuRWs=
|
||||
github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM=
|
||||
github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg=
|
||||
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo=
|
||||
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM=
|
||||
github.com/jackc/pgx/v5 v5.11.0 h1:IzBBtyK9AHqf98cctWFifYSci2hgQR/cd56wB4p+ogg=
|
||||
github.com/jackc/pgx/v5 v5.11.0/go.mod h1:mal1tBGAFfLHvZzaYh77YS/eC6IX9OWbRV1QIIM0Jn4=
|
||||
github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo=
|
||||
github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4=
|
||||
github.com/klauspost/cpuid/v2 v2.4.0 h1:S6Hrbc7+ywsr0r+RLapfGBHfyefhCTwEh3A0tV913Dw=
|
||||
github.com/klauspost/cpuid/v2 v2.4.0/go.mod h1:19jmZ9mjzoF//ddRSUsv0zfBTJWh3QJh9FNxZTMrGxU=
|
||||
github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
|
||||
github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
|
||||
github.com/mfridman/interpolate v0.0.2 h1:pnuTK7MQIxxFz1Gr+rjSIx9u7qVjf5VOoM/u6BbAxPY=
|
||||
github.com/mfridman/interpolate v0.0.2/go.mod h1:p+7uk6oE07mpE/Ik1b8EckO0O4ZXiGAfshKBWLUM9Xg=
|
||||
github.com/ncruces/go-strftime v0.1.9 h1:bY0MQC28UADQmHmaF5dgpLmImcShSi2kHU9XLdhx/f4=
|
||||
github.com/ncruces/go-strftime v0.1.9/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/pressly/goose/v3 v3.26.0 h1:KJakav68jdH0WDvoAcj8+n61WqOIaPGgH0bJWS6jpmM=
|
||||
github.com/pressly/goose/v3 v3.26.0/go.mod h1:4hC1KrritdCxtuFsqgs1R4AU5bWtTAf+cnWvfhf2DNY=
|
||||
github.com/rabbitmq/amqp091-go v1.10.0 h1:STpn5XsHlHGcecLmMFCtg7mqq0RnD+zFr4uzukfVhBw=
|
||||
github.com/rabbitmq/amqp091-go v1.10.0/go.mod h1:Hy4jKW5kQART1u+JkDTF9YYOQUHXqMuhrgxOEeS7G4o=
|
||||
github.com/redis/go-redis/v9 v9.23.0 h1:/TTVdJa8BRNsybwwGGnmiYktHNlUXtbOQxlWpHBPL3s=
|
||||
github.com/redis/go-redis/v9 v9.23.0/go.mod h1:EWP2UUk+XDaI1s5nQB5mablZN6vccXlmBQTyfnF87D0=
|
||||
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE=
|
||||
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
|
||||
github.com/sethvargo/go-retry v0.3.0 h1:EEt31A35QhrcRZtrYFDTBg91cqZVnFL2navjDrah2SE=
|
||||
github.com/sethvargo/go-retry v0.3.0/go.mod h1:mNX17F0C/HguQMyMyJxcnU471gOZGxCLyYaFyAZraas=
|
||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
|
||||
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
github.com/stretchr/testify v1.12.1 h1:EuwCh5fleGS7H32xRwO3wRGT7DxrDhLAT6FF8MpWDWE=
|
||||
github.com/stretchr/testify v1.12.1/go.mod h1:MDEgiDPPsNp5cuIrHPPCyornHKgEVbtFUmoNlxoYthg=
|
||||
github.com/zeebo/xxh3 v1.1.0 h1:s7DLGDK45Dyfg7++yxI0khrfwq9661w9EN78eP/UZVs=
|
||||
github.com/zeebo/xxh3 v1.1.0/go.mod h1:IisAie1LELR4xhVinxWS5+zf1lA4p0MW4T+w+W07F5s=
|
||||
go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA=
|
||||
go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A=
|
||||
go.opentelemetry.io/otel v1.38.0 h1:RkfdswUDRimDg0m2Az18RKOsnI8UDzppJAtj01/Ymk8=
|
||||
go.opentelemetry.io/otel v1.38.0/go.mod h1:zcmtmQ1+YmQM9wrNsTGV/q/uyusom3P8RxwExxkZhjM=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.14.0 h1:QQqYw3lkrzwVsoEX0w//EhH/TCnpRdEenKBOOEIMjWc=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.14.0/go.mod h1:gSVQcr17jk2ig4jqJ2DX30IdWH251JcNAecvrqTxH1s=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.38.0 h1:Oe2z/BCg5q7k4iXC3cqJxKYg0ieRiOqF0cecFYdPTwk=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.38.0/go.mod h1:ZQM5lAJpOsKnYagGg/zV2krVqTtaVdYdDkhMoX6Oalg=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.38.0 h1:GqRJVj7UmLjCVyVJ3ZFLdPRmhDUp2zFmQe3RHIOsw24=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.38.0/go.mod h1:ri3aaHSmCTVYu2AWv44YMauwAQc0aqI9gHKIcSbI1pU=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.38.0 h1:aTL7F04bJHUlztTsNGJ2l+6he8c+y/b//eR0jjjemT4=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.38.0/go.mod h1:kldtb7jDTeol0l3ewcmd8SDvx3EmIE7lyvqbasU3QC4=
|
||||
go.opentelemetry.io/otel/log v0.14.0 h1:2rzJ+pOAZ8qmZ3DDHg73NEKzSZkhkGIua9gXtxNGgrM=
|
||||
go.opentelemetry.io/otel/log v0.14.0/go.mod h1:5jRG92fEAgx0SU/vFPxmJvhIuDU9E1SUnEQrMlJpOno=
|
||||
go.opentelemetry.io/otel/metric v1.38.0 h1:Kl6lzIYGAh5M159u9NgiRkmoMKjvbsKtYRwgfrA6WpA=
|
||||
go.opentelemetry.io/otel/metric v1.38.0/go.mod h1:kB5n/QoRM8YwmUahxvI3bO34eVtQf2i4utNVLr9gEmI=
|
||||
go.opentelemetry.io/otel/sdk v1.38.0 h1:l48sr5YbNf2hpCUj/FoGhW9yDkl+Ma+LrVl8qaM5b+E=
|
||||
go.opentelemetry.io/otel/sdk v1.38.0/go.mod h1:ghmNdGlVemJI3+ZB5iDEuk4bWA3GkTpW+DOoZMYBVVg=
|
||||
go.opentelemetry.io/otel/sdk/log v0.14.0 h1:JU/U3O7N6fsAXj0+CXz21Czg532dW2V4gG1HE/e8Zrg=
|
||||
go.opentelemetry.io/otel/sdk/log v0.14.0/go.mod h1:imQvII+0ZylXfKU7/wtOND8Hn4OpT3YUoIgqJVksUkM=
|
||||
go.opentelemetry.io/otel/sdk/log/logtest v0.14.0 h1:Ijbtz+JKXl8T2MngiwqBlPaHqc4YCaP/i13Qrow6gAM=
|
||||
go.opentelemetry.io/otel/sdk/log/logtest v0.14.0/go.mod h1:dCU8aEL6q+L9cYTqcVOk8rM9Tp8WdnHOPLiBgp0SGOA=
|
||||
go.opentelemetry.io/otel/sdk/metric v1.38.0 h1:aSH66iL0aZqo//xXzQLYozmWrXxyFkBJ6qT5wthqPoM=
|
||||
go.opentelemetry.io/otel/sdk/metric v1.38.0/go.mod h1:dg9PBnW9XdQ1Hd6ZnRz689CbtrUp0wMMs9iPcgT9EZA=
|
||||
go.opentelemetry.io/otel/trace v1.38.0 h1:Fxk5bKrDZJUH+AMyyIXGcFAPah0oRcT+LuNtJrmcNLE=
|
||||
go.opentelemetry.io/otel/trace v1.38.0/go.mod h1:j1P9ivuFsTceSWe1oY+EeW3sc+Pp42sO++GHkg4wwhs=
|
||||
go.opentelemetry.io/proto/otlp v1.7.1 h1:gTOMpGDb0WTBOP8JaO72iL3auEZhVmAQg4ipjOVAtj4=
|
||||
go.opentelemetry.io/proto/otlp v1.7.1/go.mod h1:b2rVh6rfI/s2pHWNlB7ILJcRALpcNDzKhACevjI+ZnE=
|
||||
go.uber.org/atomic v1.12.0 h1:BvcXdFKuviU4fTL/f+SxdQ5qJX/Jix8pAkgdUcb3XOE=
|
||||
go.uber.org/atomic v1.12.0/go.mod h1:I6c4cg+6HCxRjfjSsYtApoFILnpc0CGUdGkXVqbYVNk=
|
||||
go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
|
||||
go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
|
||||
go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=
|
||||
go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
|
||||
go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw=
|
||||
go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg=
|
||||
golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M=
|
||||
golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA=
|
||||
golang.org/x/exp v0.0.0-20250620022241-b7579e27df2b h1:M2rDM6z3Fhozi9O7NWsxAkg/yqS/lQJ6PmkyIV3YP+o=
|
||||
golang.org/x/exp v0.0.0-20250620022241-b7579e27df2b/go.mod h1:3//PLf8L/X+8b4vuAfHzxeRUl04Adcb341+IGKfnqS8=
|
||||
golang.org/x/net v0.58.0 h1:ynWG7rqYi4ccpTEuPZ2QGWHktVEM9DMCj9yzDE0Q7To=
|
||||
golang.org/x/net v0.58.0/go.mod h1:YwCddHnFlT7eLQqVprV19OnhLGtc5xOKgE0RyqgfWAU=
|
||||
golang.org/x/sync v0.23.0 h1:KameEIfc1IkluZyXWLn39Wd4tURc6GbCiISGiZm2bQk=
|
||||
golang.org/x/sync v0.23.0/go.mod h1:sUUOizhqBxiL6pEWpqNLUiaJn1ShEbZ6BBqskPbjZm0=
|
||||
golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo=
|
||||
golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og=
|
||||
golang.org/x/text v0.42.0 h1:JbOZXgfeCPU9gacVtYliJqOhD+zhrEqK4LfdpmlUZqI=
|
||||
golang.org/x/text v0.42.0/go.mod h1:ojzP1Z+2QtioaF8DTtO8K5q7JWVVYwZKenzujK0Zd0E=
|
||||
gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk=
|
||||
gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E=
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20250825161204-c5933d9347a5 h1:BIRfGDEjiHRrk0QKZe3Xv2ieMhtgRGeLcZQ0mIVn4EY=
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20250825161204-c5933d9347a5/go.mod h1:j3QtIyytwqGr1JUDtYXwtMXWPKsEa5LtzIFN1Wn5WvE=
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20250825161204-c5933d9347a5 h1:eaY8u2EuxbRv7c3NiGK0/NedzVsCcV6hDuU5qPX5EGE=
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20250825161204-c5933d9347a5/go.mod h1:M4/wBTSeyLxupu3W3tJtOgB14jILAS/XWPSSa3TAlJc=
|
||||
google.golang.org/grpc v1.75.0 h1:+TW+dqTd2Biwe6KKfhE5JpiYIBWq865PhKGSXiivqt4=
|
||||
google.golang.org/grpc v1.75.0/go.mod h1:JtPAzKiq4v1xcAB2hydNlWI2RnF85XXcV0mhKXr2ecQ=
|
||||
google.golang.org/protobuf v1.36.8 h1:xHScyCOEuuwZEc6UtSOvPbAT4zRh0xcNRYekJwfqyMc=
|
||||
google.golang.org/protobuf v1.36.8/go.mod h1:fuxRtAxBytpl4zzqUh6/eyUujkJdNiuEkXntxiD/uRU=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
modernc.org/libc v1.66.3 h1:cfCbjTUcdsKyyZZfEUKfoHcP3S0Wkvz3jgSzByEWVCQ=
|
||||
modernc.org/libc v1.66.3/go.mod h1:XD9zO8kt59cANKvHPXpx7yS2ELPheAey0vjIuZOhOU8=
|
||||
modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU=
|
||||
modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg=
|
||||
modernc.org/memory v1.11.0 h1:o4QC8aMQzmcwCK3t3Ux/ZHmwFPzE6hf2Y5LbkRs+hbI=
|
||||
modernc.org/memory v1.11.0/go.mod h1:/JP4VbVC+K5sU2wZi9bHoq2MAkCnrt2r98UGeSK7Mjw=
|
||||
modernc.org/sqlite v1.38.2 h1:Aclu7+tgjgcQVShZqim41Bbw9Cho0y/7WzYptXqkEek=
|
||||
modernc.org/sqlite v1.38.2/go.mod h1:cPTJYSlgg3Sfg046yBShXENNtPrWrDX8bsbAQBzgQ5E=
|
||||
@@ -0,0 +1,28 @@
|
||||
global
|
||||
log stdout format raw local0
|
||||
stats socket ipv4@127.0.0.1:9999 level admin
|
||||
|
||||
defaults
|
||||
mode http
|
||||
log global
|
||||
option httplog
|
||||
timeout connect 2s
|
||||
timeout client 30s
|
||||
timeout server 30s
|
||||
timeout http-request 10s
|
||||
|
||||
resolvers docker
|
||||
nameserver docker 127.0.0.11:53
|
||||
accepted_payload_size 8192
|
||||
hold valid 1s
|
||||
|
||||
frontend api
|
||||
bind *:8080
|
||||
default_backend api_replicas
|
||||
|
||||
backend api_replicas
|
||||
balance roundrobin
|
||||
timeout check 3s
|
||||
option httpchk GET /health/ready
|
||||
http-check expect status 200
|
||||
server-template api 4 api:8080 check inter 1s fall 2 rise 2 resolvers docker init-addr none
|
||||
@@ -0,0 +1,16 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<policymap>
|
||||
<policy domain="resource" name="thread" value="1"/>
|
||||
<policy domain="resource" name="memory" value="256MiB"/>
|
||||
<policy domain="resource" name="map" value="256MiB"/>
|
||||
<policy domain="resource" name="disk" value="512MiB"/>
|
||||
<policy domain="resource" name="time" value="45"/>
|
||||
<policy domain="resource" name="list-length" value="2"/>
|
||||
<policy domain="delegate" rights="none" pattern="*"/>
|
||||
<policy domain="filter" rights="none" pattern="*"/>
|
||||
<policy domain="path" rights="none" pattern="@*"/>
|
||||
<policy domain="module" rights="none" pattern="*"/>
|
||||
<policy domain="module" rights="read | write" pattern="{JPEG,PNG,WEBP}"/>
|
||||
<policy domain="coder" rights="none" pattern="*"/>
|
||||
<policy domain="coder" rights="read | write" pattern="{JPEG,JPG,PNG,WEBP}"/>
|
||||
</policymap>
|
||||
+11
-4
@@ -1,7 +1,14 @@
|
||||
# Конфигурация наблюдаемости — реализовать
|
||||
# Конфигурация наблюдаемости
|
||||
|
||||
Добавьте `collector.yaml` с OTLP receivers, processors и exporters выбранной версии Collector. Укажите явные bind endpoints, внутренние сети, memory limits/batch и поведение при backpressure по уровню. Сопоставьте все три pipeline с реально развёрнутыми backend.
|
||||
`collector.yaml` принимает OTLP/HTTP от API, relay и worker и отправляет
|
||||
сигналы в Prometheus, Loki и Tempo. `prometheus.yaml` настраивает сбор
|
||||
метрик приложения и самого Collector. `alerts.yaml` содержит два правила
|
||||
с последовательностью диагностики в [runbook](runbook.md).
|
||||
|
||||
Добавьте конфигурации storage, Grafana provisioning/datasources, provisioning/dashboards и импортируемые dashboard JSON. На 4 — alert rules и runbooks. Пустой dashboard или debug exporter, который только печатает данные в stdout, не заменяют хранилища/доказательства из `contracts/observability.md`.
|
||||
`grafana/provisioning` подключает три источника и регистрирует dashboard
|
||||
[Messenger observability](grafana/dashboards/messenger.json). JSON можно
|
||||
импортировать в Grafana, где доступны те же UID источников. Compose
|
||||
публикует интерфейсы чтения только на loopback. Приёмник OTLP и scrape
|
||||
endpoints находятся во внутренней сети.
|
||||
|
||||
Не публикуйте OTLP/metrics/административные ports в общую сеть без необходимости. В lab7 реализуйте TLS/mTLS по trust matrix; экспорт Authorization и message bodies запрещён.
|
||||
Команды запуска и описание метрик приведены в [отчёте](../../REPORT.md).
|
||||
|
||||
@@ -0,0 +1,27 @@
|
||||
groups:
|
||||
- name: messenger
|
||||
rules:
|
||||
- alert: HighAPIErrorRate
|
||||
expr: |
|
||||
(
|
||||
sum(rate(messenger_http_requests_total{route!~"/health/(live|ready)",status_class="5xx"}[30s]))
|
||||
/ clamp_min(sum(rate(messenger_http_requests_total{route!~"/health/(live|ready)"}[30s])), 0.001)
|
||||
) > 0.10
|
||||
and sum(increase(messenger_http_requests_total{route!~"/health/(live|ready)"}[30s])) >= 5
|
||||
for: 15s
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: Business API 5xx ratio above 10 percent
|
||||
runbook: See infra/otel/runbook.md, PostgreSQL degradation.
|
||||
- alert: GrowingQueueLag
|
||||
expr: |
|
||||
max by (queue) (messenger_queue_oldest_age_seconds) > 15
|
||||
and on (queue) max by (queue) (messenger_queue_depth) >= 1
|
||||
and on (queue) min by (queue) (messenger_queue_sample_age_seconds) < 8
|
||||
for: 15s
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: A durable queued job has waited longer than 15 seconds
|
||||
runbook: See infra/otel/runbook.md, stopped worker.
|
||||
@@ -0,0 +1,68 @@
|
||||
receivers:
|
||||
otlp:
|
||||
protocols:
|
||||
http:
|
||||
endpoint: 0.0.0.0:4318
|
||||
|
||||
processors:
|
||||
memory_limiter:
|
||||
check_interval: 1s
|
||||
limit_mib: 96
|
||||
spike_limit_mib: 24
|
||||
batch:
|
||||
timeout: 1s
|
||||
send_batch_size: 64
|
||||
send_batch_max_size: 128
|
||||
|
||||
exporters:
|
||||
prometheus:
|
||||
endpoint: 0.0.0.0:9464
|
||||
metric_expiration: 1m
|
||||
resource_to_telemetry_conversion:
|
||||
enabled: true
|
||||
otlphttp/loki:
|
||||
endpoint: http://loki:3100/otlp
|
||||
timeout: 2s
|
||||
sending_queue:
|
||||
enabled: true
|
||||
num_consumers: 1
|
||||
queue_size: 16
|
||||
retry_on_failure:
|
||||
initial_interval: 1s
|
||||
max_interval: 2s
|
||||
max_elapsed_time: 5s
|
||||
otlphttp/tempo:
|
||||
endpoint: http://tempo:4318
|
||||
timeout: 2s
|
||||
sending_queue:
|
||||
enabled: true
|
||||
num_consumers: 1
|
||||
queue_size: 16
|
||||
retry_on_failure:
|
||||
initial_interval: 1s
|
||||
max_interval: 2s
|
||||
max_elapsed_time: 5s
|
||||
|
||||
service:
|
||||
telemetry:
|
||||
metrics:
|
||||
readers:
|
||||
- pull:
|
||||
exporter:
|
||||
prometheus:
|
||||
host: 0.0.0.0
|
||||
port: 8888
|
||||
without_units: true
|
||||
pipelines:
|
||||
metrics:
|
||||
receivers: [otlp]
|
||||
processors: [memory_limiter, batch]
|
||||
exporters: [prometheus]
|
||||
traces:
|
||||
receivers: [otlp]
|
||||
processors: [memory_limiter, batch]
|
||||
exporters: [otlphttp/tempo]
|
||||
logs:
|
||||
receivers: [otlp]
|
||||
processors: [memory_limiter, batch]
|
||||
exporters: [otlphttp/loki]
|
||||
@@ -0,0 +1,572 @@
|
||||
{
|
||||
"id": null,
|
||||
"uid": "lab6-messenger",
|
||||
"title": "Messenger observability",
|
||||
"tags": [
|
||||
"lab6"
|
||||
],
|
||||
"timezone": "browser",
|
||||
"schemaVersion": 39,
|
||||
"version": 1,
|
||||
"refresh": "5s",
|
||||
"time": {
|
||||
"from": "now-15m",
|
||||
"to": "now"
|
||||
},
|
||||
"panels": [
|
||||
{
|
||||
"id": 1,
|
||||
"type": "timeseries",
|
||||
"title": "API requests per second",
|
||||
"description": "Per-second rate across both API replicas, excluding health checks. Window 1 minute.",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "lab-prometheus"
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 7,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 0
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "reqps",
|
||||
"min": 0
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "sum(rate(messenger_http_requests_total{route!~\"/health/(live|ready)\"}[1m]))",
|
||||
"legendFormat": "All business requests"
|
||||
},
|
||||
{
|
||||
"refId": "B",
|
||||
"expr": "sum(rate(messenger_http_requests_total{route!~\"/health/(live|ready)\",status_class=\"5xx\"}[1m]))",
|
||||
"legendFormat": "5xx"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 2,
|
||||
"type": "timeseries",
|
||||
"title": "API error ratio",
|
||||
"description": "Server errors divided by all business responses. An empty window has no ratio.",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "lab-prometheus"
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 7,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 0
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "percentunit",
|
||||
"min": 0,
|
||||
"max": 1
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "(sum(rate(messenger_http_requests_total{route!~\"/health/(live|ready)\",status_class=\"5xx\"}[1m])) or 0 * sum(rate(messenger_http_requests_total{route!~\"/health/(live|ready)\"}[1m]))) / (sum(rate(messenger_http_requests_total{route!~\"/health/(live|ready)\"}[1m])) > 0)",
|
||||
"legendFormat": "5xx / requests"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 3,
|
||||
"type": "timeseries",
|
||||
"title": "API response duration",
|
||||
"description": "p50, p95 and p99 from cumulative histogram buckets, aggregated across replicas.",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "lab-prometheus"
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 7,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 7
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "s",
|
||||
"min": 0
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "histogram_quantile(0.5, sum by (le) (rate(messenger_http_request_duration_seconds_bucket{route!~\"/health/(live|ready)\"}[1m])))",
|
||||
"legendFormat": "p50"
|
||||
},
|
||||
{
|
||||
"refId": "B",
|
||||
"expr": "histogram_quantile(0.95, sum by (le) (rate(messenger_http_request_duration_seconds_bucket{route!~\"/health/(live|ready)\"}[1m])))",
|
||||
"legendFormat": "p95"
|
||||
},
|
||||
{
|
||||
"refId": "C",
|
||||
"expr": "histogram_quantile(0.99, sum by (le) (rate(messenger_http_request_duration_seconds_bucket{route!~\"/health/(live|ready)\"}[1m])))",
|
||||
"legendFormat": "p99"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 4,
|
||||
"type": "timeseries",
|
||||
"title": "Worker job outcomes per second",
|
||||
"description": "Rate of actual attempts, grouped by outcome. Duplicate deliveries without a claim are not attempts.",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "lab-prometheus"
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 7,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 7
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "ops",
|
||||
"min": 0
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "sum by (outcome) (rate(messenger_jobs_total[1m]))",
|
||||
"legendFormat": "{{outcome}}"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 5,
|
||||
"type": "timeseries",
|
||||
"title": "Worker processing duration",
|
||||
"description": "p50, p95 and p99 from the duration histogram for claimed attempts.",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "lab-prometheus"
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 7,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 14
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "s",
|
||||
"min": 0
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "histogram_quantile(0.5, sum by (le) (rate(messenger_job_duration_seconds_bucket[1m])))",
|
||||
"legendFormat": "p50"
|
||||
},
|
||||
{
|
||||
"refId": "B",
|
||||
"expr": "histogram_quantile(0.95, sum by (le) (rate(messenger_job_duration_seconds_bucket[1m])))",
|
||||
"legendFormat": "p95"
|
||||
},
|
||||
{
|
||||
"refId": "C",
|
||||
"expr": "histogram_quantile(0.99, sum by (le) (rate(messenger_job_duration_seconds_bucket[1m])))",
|
||||
"legendFormat": "p99"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 6,
|
||||
"type": "timeseries",
|
||||
"title": "Attachment time to ready",
|
||||
"description": "From durable job creation to the commit that exposes ready. Includes time queued and any retries.",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "lab-prometheus"
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 7,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 14
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "s",
|
||||
"min": 0
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "histogram_quantile(0.5, sum by (le) (rate(messenger_attachment_time_to_ready_seconds_bucket[1m])))",
|
||||
"legendFormat": "p50"
|
||||
},
|
||||
{
|
||||
"refId": "B",
|
||||
"expr": "histogram_quantile(0.95, sum by (le) (rate(messenger_attachment_time_to_ready_seconds_bucket[1m])))",
|
||||
"legendFormat": "p95"
|
||||
},
|
||||
{
|
||||
"refId": "C",
|
||||
"expr": "histogram_quantile(0.99, sum by (le) (rate(messenger_attachment_time_to_ready_seconds_bucket[1m])))",
|
||||
"legendFormat": "p99"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 7,
|
||||
"type": "timeseries",
|
||||
"title": "Queue depth",
|
||||
"description": "Durable queued jobs on the current pipeline version, including pending outbox and delayed retries.",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "lab-prometheus"
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 7,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 21
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "short",
|
||||
"min": 0
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "max by (queue) (messenger_queue_depth)",
|
||||
"legendFormat": "{{queue}}"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 8,
|
||||
"type": "timeseries",
|
||||
"title": "Oldest queued job age",
|
||||
"description": "Elapsed time since the oldest job still in the durable queue was accepted.",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "lab-prometheus"
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 7,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 21
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "s",
|
||||
"min": 0
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "max by (queue) (messenger_queue_oldest_age_seconds)",
|
||||
"legendFormat": "{{queue}}"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 9,
|
||||
"type": "timeseries",
|
||||
"title": "PostgreSQL pool used and capacity",
|
||||
"description": "Checked-out connections and configured pool capacity, summed per service.",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "lab-prometheus"
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 7,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 28
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "short",
|
||||
"min": 0
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "sum by (service_name,state) (messenger_db_pool_connections)",
|
||||
"legendFormat": "{{service_name}} / {{state}}"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 10,
|
||||
"type": "timeseries",
|
||||
"title": "PostgreSQL pool waiting",
|
||||
"description": "Acquisition calls in progress. Instantaneous gauge, so very short waits can fall between scrapes.",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "lab-prometheus"
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 7,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 28
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "short",
|
||||
"min": 0
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "sum by (service_name) (messenger_db_pool_waiting)",
|
||||
"legendFormat": "{{service_name}}"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 11,
|
||||
"type": "timeseries",
|
||||
"title": "Worker busy and capacity",
|
||||
"description": "Active claimed jobs and total processing lanes.",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "lab-prometheus"
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 7,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 35
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "short",
|
||||
"min": 0
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "sum(messenger_worker_busy)",
|
||||
"legendFormat": "Busy"
|
||||
},
|
||||
{
|
||||
"refId": "B",
|
||||
"expr": "sum(messenger_worker_capacity)",
|
||||
"legendFormat": "Capacity"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 12,
|
||||
"type": "timeseries",
|
||||
"title": "Worker saturation",
|
||||
"description": "Fraction of processing lanes occupied by claimed attempts.",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "lab-prometheus"
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 7,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 35
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "percentunit",
|
||||
"min": 0,
|
||||
"max": 1
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "sum(messenger_worker_busy) / (sum(messenger_worker_capacity) > 0)",
|
||||
"legendFormat": "Busy / capacity"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 13,
|
||||
"type": "timeseries",
|
||||
"title": "Process CPU",
|
||||
"description": "CPU seconds per wall second. 1.0 is one full CPU core.",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "lab-prometheus"
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 7,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 42
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "none",
|
||||
"min": 0
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "rate(messenger_process_cpu_time_seconds_total[1m])",
|
||||
"legendFormat": "{{service_name}} / {{service_instance_id}}"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 14,
|
||||
"type": "timeseries",
|
||||
"title": "Process resident memory",
|
||||
"description": "RSS per application process, read from Linux statm.",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "lab-prometheus"
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 7,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 42
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "bytes",
|
||||
"min": 0
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "messenger_process_memory_bytes",
|
||||
"legendFormat": "{{service_name}} / {{service_instance_id}}"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 15,
|
||||
"type": "timeseries",
|
||||
"title": "Firing alerts",
|
||||
"description": "Firing state reported by Prometheus. Inactive rules disappear from this series.",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "lab-prometheus"
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 7,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 49
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "short",
|
||||
"min": 0
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "ALERTS{alertname=~\"HighAPIErrorRate|GrowingQueueLag\",alertstate=\"firing\"}",
|
||||
"legendFormat": "{{alertname}} {{queue}}"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 16,
|
||||
"type": "timeseries",
|
||||
"title": "Collector exporter queue",
|
||||
"description": "Batches pending delivery to telemetry storage and the configured queue limit.",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "lab-prometheus"
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 7,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 49
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "short",
|
||||
"min": 0
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "otelcol_exporter_queue_size",
|
||||
"legendFormat": "{{exporter}} queued"
|
||||
},
|
||||
{
|
||||
"refId": "B",
|
||||
"expr": "otelcol_exporter_queue_capacity",
|
||||
"legendFormat": "{{exporter}} capacity"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 17,
|
||||
"type": "logs",
|
||||
"title": "Correlated API and worker logs",
|
||||
"description": "Expand a log line and follow its TraceID to Tempo.",
|
||||
"datasource": {
|
||||
"type": "loki",
|
||||
"uid": "lab-loki"
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 9,
|
||||
"w": 24,
|
||||
"x": 0,
|
||||
"y": 56
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "{service_name=~\"messenger-api|messenger-worker\"}"
|
||||
}
|
||||
],
|
||||
"options": {
|
||||
"showTime": true,
|
||||
"sortOrder": "Descending",
|
||||
"wrapLogMessage": false
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
apiVersion: 1
|
||||
providers:
|
||||
- name: Messenger
|
||||
orgId: 1
|
||||
folder: Lab 6
|
||||
type: file
|
||||
updateIntervalSeconds: 10
|
||||
options:
|
||||
path: /var/lib/grafana/dashboards
|
||||
@@ -0,0 +1,38 @@
|
||||
apiVersion: 1
|
||||
datasources:
|
||||
- name: Prometheus
|
||||
uid: lab-prometheus
|
||||
type: prometheus
|
||||
access: proxy
|
||||
url: http://prometheus:9090
|
||||
isDefault: true
|
||||
editable: false
|
||||
jsonData:
|
||||
timeInterval: 5s
|
||||
- name: Loki
|
||||
uid: lab-loki
|
||||
type: loki
|
||||
access: proxy
|
||||
url: http://loki:3100
|
||||
editable: false
|
||||
jsonData:
|
||||
derivedFields:
|
||||
- name: TraceID
|
||||
matcherRegex: '"trace_id":"([0-9a-f]{32})"'
|
||||
datasourceUid: lab-tempo
|
||||
url: '$${__value.raw}'
|
||||
- name: Tempo
|
||||
uid: lab-tempo
|
||||
type: tempo
|
||||
access: proxy
|
||||
url: http://tempo:3200
|
||||
editable: false
|
||||
jsonData:
|
||||
tracesToLogsV2:
|
||||
datasourceUid: lab-loki
|
||||
spanStartTimeShift: -30s
|
||||
spanEndTimeShift: 30s
|
||||
tags:
|
||||
- key: service.name
|
||||
value: service_name
|
||||
filterByTraceID: true
|
||||
@@ -0,0 +1,30 @@
|
||||
auth_enabled: false
|
||||
server:
|
||||
http_listen_port: 3100
|
||||
common:
|
||||
instance_addr: 127.0.0.1
|
||||
path_prefix: /loki
|
||||
storage:
|
||||
filesystem:
|
||||
chunks_directory: /loki/chunks
|
||||
rules_directory: /loki/rules
|
||||
replication_factor: 1
|
||||
ring:
|
||||
kvstore:
|
||||
store: inmemory
|
||||
schema_config:
|
||||
configs:
|
||||
- from: 2024-01-01
|
||||
store: tsdb
|
||||
object_store: filesystem
|
||||
schema: v13
|
||||
index:
|
||||
prefix: index_
|
||||
period: 24h
|
||||
compactor:
|
||||
working_directory: /loki/compactor
|
||||
retention_enabled: true
|
||||
delete_request_store: filesystem
|
||||
limits_config:
|
||||
allow_structured_metadata: true
|
||||
retention_period: 24h
|
||||
@@ -0,0 +1,13 @@
|
||||
global:
|
||||
scrape_interval: 5s
|
||||
evaluation_interval: 5s
|
||||
|
||||
rule_files: [/etc/prometheus/alerts.yaml]
|
||||
|
||||
scrape_configs:
|
||||
- job_name: messenger-otlp
|
||||
static_configs:
|
||||
- targets: [otel-collector:9464]
|
||||
- job_name: otel-collector
|
||||
static_configs:
|
||||
- targets: [otel-collector:8888]
|
||||
@@ -0,0 +1,69 @@
|
||||
# Диагностика ЛР 6
|
||||
|
||||
Команды рассчитаны на локальный стенд с синтетическими данными. Выполняйте
|
||||
сценарии по одному. После остальных проверок запустите автоматическую приёмку
|
||||
с тестовым Compose override по [инструкции](../../student-tests/lab6/README.md).
|
||||
|
||||
## Где смотреть
|
||||
|
||||
- [Dashboard Messenger](http://127.0.0.1:3006/d/lab6-messenger) показывает историю
|
||||
HTTP, worker, очереди, пула, CPU/RAM и доступные логи.
|
||||
- [Правила Prometheus](http://127.0.0.1:9096/alerts) показывают состояние каждого
|
||||
алерта. Переход `firing → inactive` означает, что условие сбросилось.
|
||||
- В Grafana Explore выберите Loki. По ответу API и его `X-Request-Id` найдите
|
||||
запись `{service_name="messenger-api"} | json | request_id="<id>"`.
|
||||
В записи откройте поле `TraceID` или вставьте значение `trace_id` в поиск Tempo.
|
||||
- В Tempo проверьте дочерние операции `postgres.query`, `postgres.acquire`
|
||||
и `s3.get`/`s3.put`. Для worker используйте
|
||||
`{service_name="messenger-worker"} | json | trace_id="<id>"` в Loki.
|
||||
|
||||
## PostgreSQL degradation
|
||||
|
||||
1. Создайте синтетического пользователя и получите сессию, пока БД отвечает.
|
||||
2. В отдельном терминале откройте сессию
|
||||
`docker compose exec -it postgres psql -U messenger -d messenger`. Выполните:
|
||||
|
||||
```sql
|
||||
BEGIN;
|
||||
LOCK TABLE public.users IN ACCESS EXCLUSIVE MODE;
|
||||
```
|
||||
|
||||
3. Не закрывая транзакцию, 45 секунд посылайте небольшим постоянным потоком
|
||||
авторизованные `GET /api/v1/users/me`. Проверьте `/health/ready` параллельно.
|
||||
Если LB доступен, запросы с ошибкой и здоровая readiness указывают на
|
||||
проблему внутри рабочего пути. `HighAPIErrorRate` должен стать `firing`.
|
||||
4. Найдите по `X-Request-Id` структурный лог ответа и откройте трассу. Проверьте
|
||||
длительность и ошибку дочернего чтения `postgres.query` с операцией
|
||||
`GetUserByID`. CPU и pool gauges помогают отличить блокировку от насыщения.
|
||||
5. В том же терминале PostgreSQL выполните `ROLLBACK;`. Продолжайте делать
|
||||
успешные запросы до опустошения 30-секундного окна и сброса алерта.
|
||||
|
||||
## Stopped worker
|
||||
|
||||
1. Выполните `docker compose stop worker`. Оставьте RabbitMQ, relay, DB и API
|
||||
включёнными.
|
||||
2. Через публичный API загрузите три исправных небольших PNG в свой чат.
|
||||
Ожидайте `202 Accepted`, рост `messenger_queue_depth` и возраста самой
|
||||
старой задачи. Подождите, пока `GrowingQueueLag` станет `firing`.
|
||||
3. В Tempo найдите producer. Пока worker остановлен, за producer для этих
|
||||
задач ещё нет consumer spans. Проверьте также состояние worker в Compose.
|
||||
4. Выполните `docker compose start worker`. Убедитесь, что все три вложения
|
||||
стали `ready`, глубина и возраст вернулись к нулю, а алерт стал `inactive`.
|
||||
|
||||
## Остановка Loki
|
||||
|
||||
1. Запишите текущее значение `otelcol_exporter_send_failed_log_records_total`
|
||||
и `otelcol_exporter_enqueue_failed_log_records_total` для
|
||||
`exporter="otlphttp/loki"` в Prometheus. Отсутствующий счётчик до первой
|
||||
ошибки считайте нулём.
|
||||
2. Выполните `docker compose stop loki`. Проверьте readiness, проведите серию
|
||||
синтетических записей и прочитайте их через API. Сохраните максимум
|
||||
`otelcol_exporter_queue_size`, вместимость
|
||||
`otelcol_exporter_queue_capacity` для Loki и
|
||||
`otelcol_process_memory_rss` за время опыта.
|
||||
3. Выполните `docker compose start loki`. Дождитесь `/ready` на порту 3106.
|
||||
Отправьте новый рабочий запрос, найдите его в Loki по `X-Request-Id`.
|
||||
Сравните счётчики после обработки очереди и проверьте, что Collector
|
||||
сохранил тот же контейнер без рестарта. `send_failed` и `enqueue_failed`
|
||||
считают записи в разных точках, первая величина относится к уже принятым
|
||||
экспортёром пачкам, вторая к записям, которые экспортёр не принял.
|
||||
@@ -0,0 +1,18 @@
|
||||
server:
|
||||
http_listen_port: 3200
|
||||
distributor:
|
||||
receivers:
|
||||
otlp:
|
||||
protocols:
|
||||
http:
|
||||
endpoint: 0.0.0.0:4318
|
||||
compactor:
|
||||
compaction:
|
||||
block_retention: 24h
|
||||
storage:
|
||||
trace:
|
||||
backend: local
|
||||
local:
|
||||
path: /var/tempo/blocks
|
||||
wal:
|
||||
path: /var/tempo/wal
|
||||
Executable
+65
@@ -0,0 +1,65 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
|
||||
for name in rustfs_root_access_key rustfs_root_secret_key s3_access_key_id s3_secret_access_key; do
|
||||
if [ ! -s "/run/secrets/$name" ]; then
|
||||
echo "missing S3 secret: $name" >&2
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
|
||||
root_key=$(cat /run/secrets/rustfs_root_access_key)
|
||||
root_secret=$(cat /run/secrets/rustfs_root_secret_key)
|
||||
app_key=$(cat /run/secrets/s3_access_key_id)
|
||||
app_secret=$(cat /run/secrets/s3_secret_access_key)
|
||||
|
||||
if [ "$root_key" = "$app_key" ]; then
|
||||
echo 'API and RustFS root access keys must differ' >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# These values become path and JSON fragments below.
|
||||
case "$S3_BUCKET" in
|
||||
messenger-attachments) ;;
|
||||
*) echo 'unsupported S3_BUCKET in bootstrap' >&2; exit 1 ;;
|
||||
esac
|
||||
|
||||
rc=/usr/bin/rc
|
||||
policy=messenger-attachments-api-v1
|
||||
|
||||
"$rc" alias set root http://s3:9000 "$root_key" "$root_secret" \
|
||||
--region "$S3_REGION" --bucket-lookup path >/dev/null
|
||||
"$rc" bucket create "root/$S3_BUCKET" --ignore-existing
|
||||
# A new bucket is private by default; this also removes a stale public policy.
|
||||
"$rc" bucket anonymous set private "root/$S3_BUCKET"
|
||||
|
||||
cat > /tmp/api-policy.json <<EOF
|
||||
{
|
||||
"Version": "2012-10-17",
|
||||
"Statement": [
|
||||
{
|
||||
"Effect": "Allow",
|
||||
"Action": ["s3:ListBucket", "s3:GetBucketLocation"],
|
||||
"Resource": "arn:aws:s3:::$S3_BUCKET"
|
||||
},
|
||||
{
|
||||
"Effect": "Allow",
|
||||
"Action": ["s3:PutObject", "s3:GetObject", "s3:DeleteObject"],
|
||||
"Resource": "arn:aws:s3:::$S3_BUCKET/*"
|
||||
}
|
||||
]
|
||||
}
|
||||
EOF
|
||||
|
||||
if ! "$rc" admin policy info root/ "$policy" >/dev/null 2>&1; then
|
||||
"$rc" admin policy create root/ "$policy" /tmp/api-policy.json
|
||||
fi
|
||||
if ! "$rc" admin user info root/ "$app_key" >/dev/null 2>&1; then
|
||||
"$rc" admin user add root/ "$app_key" "$app_secret" >/dev/null
|
||||
fi
|
||||
"$rc" admin policy attach root/ "$policy" --user "$app_key" >/dev/null
|
||||
|
||||
"$rc" alias set app http://s3:9000 "$app_key" "$app_secret" \
|
||||
--region "$S3_REGION" --bucket-lookup path >/dev/null
|
||||
"$rc" bucket list "app/$S3_BUCKET" >/dev/null
|
||||
echo "S3 bucket and API identity ready: $S3_BUCKET"
|
||||
@@ -0,0 +1,119 @@
|
||||
package broker
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"net"
|
||||
"time"
|
||||
|
||||
amqp "github.com/rabbitmq/amqp091-go"
|
||||
)
|
||||
|
||||
const (
|
||||
Exchange = "images"
|
||||
Queue = "image.jobs"
|
||||
Route = "process"
|
||||
dead = "image.invalid"
|
||||
)
|
||||
|
||||
type boundedConn struct{ net.Conn }
|
||||
|
||||
func (c boundedConn) Write(p []byte) (int, error) {
|
||||
if err := c.SetWriteDeadline(time.Now().Add(5 * time.Second)); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return c.Conn.Write(p)
|
||||
}
|
||||
|
||||
// Connect supplies a freshly declared channel after a broker connection loss.
|
||||
func Connect(url string) (*amqp.Connection, *amqp.Channel, error) {
|
||||
dial := amqp.DefaultDial(5 * time.Second)
|
||||
conn, err := amqp.DialConfig(url, amqp.Config{
|
||||
Heartbeat: 10 * time.Second, Locale: "en_US",
|
||||
Dial: func(network, addr string) (net.Conn, error) {
|
||||
c, err := dial(network, addr)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return boundedConn{c}, nil
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
// Connection errors may include credentials in a URL.
|
||||
return nil, nil, errors.New("RabbitMQ connection unavailable")
|
||||
}
|
||||
ch, err := conn.Channel()
|
||||
if err == nil {
|
||||
err = declare(ch)
|
||||
}
|
||||
if err != nil {
|
||||
Close(conn)
|
||||
return nil, nil, err
|
||||
}
|
||||
return conn, ch, nil
|
||||
}
|
||||
|
||||
func Close(conn *amqp.Connection) {
|
||||
_ = conn.CloseDeadline(time.Now().Add(time.Second))
|
||||
}
|
||||
|
||||
func declare(ch *amqp.Channel) error {
|
||||
if err := ch.ExchangeDeclare(Exchange, "direct", true, false, false, false, nil); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := ch.QueueDeclare(dead, true, false, false, false, nil); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := ch.QueueDeclare(Queue, true, false, false, false, amqp.Table{
|
||||
"x-queue-type": "classic", "x-dead-letter-exchange": "", "x-dead-letter-routing-key": dead,
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
return ch.QueueBind(Queue, Route, Exchange, false, nil)
|
||||
}
|
||||
|
||||
// Publisher has exactly one in-flight publish. A failed/ambiguous publish
|
||||
// invalidates its channel before the next event can be attempted.
|
||||
type Publisher struct {
|
||||
ch *amqp.Channel
|
||||
returns <-chan amqp.Return
|
||||
confirms <-chan amqp.Confirmation
|
||||
}
|
||||
|
||||
func NewPublisher(ch *amqp.Channel) (*Publisher, error) {
|
||||
if err := ch.Confirm(false); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &Publisher{
|
||||
ch: ch, returns: ch.NotifyReturn(make(chan amqp.Return, 1)),
|
||||
confirms: ch.NotifyPublish(make(chan amqp.Confirmation, 1)),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (p *Publisher) Publish(ctx context.Context, id string, payload []byte) error {
|
||||
seq := p.ch.GetNextPublishSeqNo()
|
||||
if err := p.ch.PublishWithContext(ctx, Exchange, Route, true, false, amqp.Publishing{
|
||||
ContentType: "application/json", DeliveryMode: amqp.Persistent, MessageId: id, Body: payload,
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
select {
|
||||
case <-p.returns:
|
||||
return errors.New("outbox message was not routed")
|
||||
case confirm, ok := <-p.confirms:
|
||||
if !ok || confirm.DeliveryTag != seq || !confirm.Ack {
|
||||
return errors.New("outbox publish was not confirmed")
|
||||
}
|
||||
// RabbitMQ sends basic.return before its confirm for an unrouted
|
||||
// mandatory publish. The client's frame reader enqueues both in order.
|
||||
// With a single in-flight message any queued return is for this ID.
|
||||
select {
|
||||
case <-p.returns:
|
||||
return errors.New("outbox message was not routed")
|
||||
default:
|
||||
return nil
|
||||
}
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,180 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
type Database struct {
|
||||
PostgresURL string
|
||||
RedisAddr string
|
||||
}
|
||||
|
||||
type S3 struct {
|
||||
Endpoint string
|
||||
Region string
|
||||
Bucket string
|
||||
AccessKeyID string
|
||||
SecretAccessKey string
|
||||
}
|
||||
|
||||
type Config struct {
|
||||
JobMaxAttempts int
|
||||
HTTP struct {
|
||||
Host string
|
||||
Port string
|
||||
}
|
||||
Database Database
|
||||
|
||||
Session struct {
|
||||
TTL time.Duration
|
||||
Store string
|
||||
}
|
||||
|
||||
S3 S3
|
||||
|
||||
AppEnv string
|
||||
CacheURL string
|
||||
}
|
||||
|
||||
func Load() (*Config, error) {
|
||||
config := &Config{}
|
||||
sessionTTL, err := getSessionTTL()
|
||||
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
config.Session.TTL = sessionTTL
|
||||
config.JobMaxAttempts, err = EnvInt("JOB_MAX_ATTEMPTS", 5, 1, 20)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
config.HTTP.Host = os.Getenv("APP_HOST")
|
||||
if config.HTTP.Host == "" {
|
||||
config.HTTP.Host = "0.0.0.0"
|
||||
}
|
||||
|
||||
config.HTTP.Port = os.Getenv("APP_PORT")
|
||||
if config.HTTP.Port == "" {
|
||||
config.HTTP.Port = "8080"
|
||||
}
|
||||
|
||||
config.AppEnv = os.Getenv("APP_ENV")
|
||||
if config.AppEnv == "" {
|
||||
config.AppEnv = "lab"
|
||||
}
|
||||
|
||||
store := os.Getenv("SESSION_STORE")
|
||||
|
||||
config.Session.Store = strings.TrimSpace(string(store))
|
||||
if config.Session.Store == "" {
|
||||
return nil, fmt.Errorf("empty session store data")
|
||||
}
|
||||
|
||||
config.Database, err = getDatabaseURL()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
config.S3, err = getS3Data()
|
||||
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return config, nil
|
||||
|
||||
}
|
||||
|
||||
func getSessionTTL() (time.Duration, error) {
|
||||
value, ok := os.LookupEnv("SESSION_TTL_SECONDS")
|
||||
if !ok {
|
||||
return time.Hour, nil
|
||||
}
|
||||
|
||||
seconds, err := strconv.ParseInt(value, 10, 64)
|
||||
if err != nil || seconds <= 0 || seconds > int64((1<<63-1)/int64(time.Second)) {
|
||||
return 0, fmt.Errorf("invalid SESSION_TTL_SECONDS: %q", value)
|
||||
}
|
||||
|
||||
return time.Duration(seconds) * time.Second, nil
|
||||
}
|
||||
|
||||
func LoadAttachmentCleanup() (postgresURL string, objectStorage S3, err error) {
|
||||
postgresURL, err = LoadPostgresURL()
|
||||
if err != nil {
|
||||
return "", S3{}, err
|
||||
}
|
||||
objectStorage, err = getS3Data()
|
||||
return postgresURL, objectStorage, err
|
||||
}
|
||||
|
||||
func LoadPostgresURL() (string, error) {
|
||||
postgresData, err := os.ReadFile(os.Getenv("DATABASE_URL_FILE"))
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("read database URL: %w", err)
|
||||
}
|
||||
postgresURL := strings.TrimSpace(string(postgresData))
|
||||
if postgresURL == "" {
|
||||
return "", fmt.Errorf("empty database URL")
|
||||
}
|
||||
return postgresURL, nil
|
||||
}
|
||||
|
||||
func getDatabaseURL() (Database, error) {
|
||||
postgresURL, err := LoadPostgresURL()
|
||||
if err != nil {
|
||||
return Database{}, err
|
||||
}
|
||||
db := Database{PostgresURL: postgresURL}
|
||||
|
||||
redisData := os.Getenv("REDIS_ADDR")
|
||||
|
||||
db.RedisAddr = strings.TrimSpace(string(redisData))
|
||||
|
||||
if db.RedisAddr == "" {
|
||||
return db, fmt.Errorf("redis addres is empty")
|
||||
}
|
||||
|
||||
return db, nil
|
||||
|
||||
}
|
||||
|
||||
func getS3Data() (S3, error) {
|
||||
s3 := S3{
|
||||
Endpoint: os.Getenv("S3_ENDPOINT"),
|
||||
Region: os.Getenv("S3_REGION"),
|
||||
Bucket: os.Getenv("S3_BUCKET"),
|
||||
}
|
||||
|
||||
if s3.Endpoint == "" {
|
||||
return s3, fmt.Errorf("empty s3 endpoint")
|
||||
}
|
||||
if s3.Region == "" {
|
||||
return s3, fmt.Errorf("empty s3 region")
|
||||
}
|
||||
if s3.Bucket == "" {
|
||||
return s3, fmt.Errorf("emtpy s3 bucket")
|
||||
}
|
||||
accessKeyIDPath := os.Getenv("S3_ACCESS_KEY_ID_FILE")
|
||||
|
||||
accessKeyID, err := os.ReadFile(accessKeyIDPath)
|
||||
if err != nil {
|
||||
return s3, fmt.Errorf("empty s3 access_key_id")
|
||||
}
|
||||
s3.AccessKeyID = strings.TrimSpace(string(accessKeyID))
|
||||
|
||||
secretAccessKeyPath := os.Getenv("S3_SECRET_ACCESS_KEY_FILE")
|
||||
secretAccessKey, err := os.ReadFile(secretAccessKeyPath)
|
||||
if err != nil {
|
||||
return s3, fmt.Errorf("empty s3 secret_access_key_file")
|
||||
}
|
||||
s3.SecretAccessKey = strings.TrimSpace(string(secretAccessKey))
|
||||
|
||||
return s3, nil
|
||||
|
||||
}
|
||||
@@ -0,0 +1,77 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
type Pipeline struct {
|
||||
BrokerURL string
|
||||
Concurrency int
|
||||
Prefetch int
|
||||
Timeout time.Duration
|
||||
Lease time.Duration
|
||||
Retention time.Duration
|
||||
CleanupGrace time.Duration
|
||||
CleanupPeriod time.Duration
|
||||
RetryBase time.Duration
|
||||
}
|
||||
|
||||
func EnvInt(name string, fallback, minimum, maximum int) (int, error) {
|
||||
value := os.Getenv(name)
|
||||
if value == "" {
|
||||
return fallback, nil
|
||||
}
|
||||
n, err := strconv.Atoi(value)
|
||||
if err != nil || n < minimum || n > maximum {
|
||||
return 0, fmt.Errorf("%s must be between %d and %d", name, minimum, maximum)
|
||||
}
|
||||
return n, nil
|
||||
}
|
||||
|
||||
func LoadPipeline() (Pipeline, error) {
|
||||
var cfg Pipeline
|
||||
path := os.Getenv("BROKER_URL_FILE")
|
||||
if path != "" {
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return cfg, fmt.Errorf("cannot read broker URL file: %w", err)
|
||||
}
|
||||
cfg.BrokerURL = strings.TrimSpace(string(data))
|
||||
}
|
||||
duration := func(dst *time.Duration, name string, fallback, minimum, maximum int) error {
|
||||
n, err := EnvInt(name, fallback, minimum, maximum)
|
||||
*dst = time.Duration(n) * time.Second
|
||||
return err
|
||||
}
|
||||
var err error
|
||||
if cfg.Concurrency, err = EnvInt("WORKER_CONCURRENCY", 1, 1, 4); err != nil {
|
||||
return cfg, err
|
||||
}
|
||||
if cfg.Prefetch, err = EnvInt("WORKER_PREFETCH", 1, 1, 16); err != nil {
|
||||
return cfg, err
|
||||
}
|
||||
for _, d := range []struct {
|
||||
dst *time.Duration
|
||||
name string
|
||||
fallback, minimum, maximum int
|
||||
}{
|
||||
{&cfg.Timeout, "JOB_TIMEOUT_SECONDS", 60, 1, 600},
|
||||
{&cfg.Lease, "JOB_LEASE_SECONDS", 90, 2, 900},
|
||||
{&cfg.Retention, "STAGING_RETENTION_SECONDS", 86400, 1, 604800},
|
||||
{&cfg.CleanupGrace, "CLEANUP_GRACE_SECONDS", 600, 1, 604800},
|
||||
{&cfg.CleanupPeriod, "CLEANUP_INTERVAL_SECONDS", 5, 1, 600},
|
||||
{&cfg.RetryBase, "JOB_RETRY_BASE_SECONDS", 2, 1, 15},
|
||||
} {
|
||||
if err := duration(d.dst, d.name, d.fallback, d.minimum, d.maximum); err != nil {
|
||||
return cfg, err
|
||||
}
|
||||
}
|
||||
if cfg.Lease < cfg.Timeout+time.Second {
|
||||
return cfg, fmt.Errorf("JOB_LEASE_SECONDS must exceed JOB_TIMEOUT_SECONDS")
|
||||
}
|
||||
return cfg, nil
|
||||
}
|
||||
@@ -0,0 +1,84 @@
|
||||
package database
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
|
||||
"highload/internal/observability"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
"github.com/jackc/pgx/v5/pgconn"
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
"go.opentelemetry.io/otel/attribute"
|
||||
"go.opentelemetry.io/otel/trace"
|
||||
)
|
||||
|
||||
func NewPostgres(ctx context.Context, databaseURL string) (*pgxpool.Pool, error) {
|
||||
config, err := pgxpool.ParseConfig(databaseURL)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
tracer := &postgresTrace{}
|
||||
config.ConnConfig.Tracer = tracer
|
||||
pool, err := pgxpool.NewWithConfig(ctx, config)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err = pool.Ping(ctx); err != nil {
|
||||
pool.Close()
|
||||
return nil, err
|
||||
}
|
||||
observability.WatchPool(pool, &tracer.waiting)
|
||||
return pool, nil
|
||||
}
|
||||
|
||||
type postgresTrace struct{ waiting atomic.Int64 }
|
||||
|
||||
func (p *postgresTrace) TraceAcquireStart(ctx context.Context, _ *pgxpool.Pool, _ pgxpool.TraceAcquireStartData) context.Context {
|
||||
p.waiting.Add(1)
|
||||
if !trace.SpanContextFromContext(ctx).IsValid() {
|
||||
return ctx
|
||||
}
|
||||
ctx, _ = observability.Tracer.Start(ctx, "postgres.acquire", trace.WithSpanKind(trace.SpanKindClient),
|
||||
trace.WithAttributes(attribute.String("db.system.name", "postgresql")))
|
||||
return ctx
|
||||
}
|
||||
|
||||
func (p *postgresTrace) TraceAcquireEnd(ctx context.Context, _ *pgxpool.Pool, data pgxpool.TraceAcquireEndData) {
|
||||
p.waiting.Add(-1)
|
||||
observability.End(trace.SpanFromContext(ctx), data.Err)
|
||||
}
|
||||
|
||||
func (*postgresTrace) TraceQueryStart(ctx context.Context, _ *pgx.Conn, data pgx.TraceQueryStartData) context.Context {
|
||||
if !trace.SpanContextFromContext(ctx).IsValid() {
|
||||
return ctx
|
||||
}
|
||||
operation := "statement"
|
||||
if name, ok := strings.CutPrefix(data.SQL, "-- name: "); ok {
|
||||
// sqlc embeds the static query name. SQL and arguments stay local.
|
||||
operation, _, _ = strings.Cut(name, " ")
|
||||
} else {
|
||||
switch data.SQL {
|
||||
case "begin", "commit", "rollback":
|
||||
operation = data.SQL
|
||||
}
|
||||
}
|
||||
ctx, _ = observability.Tracer.Start(ctx, "postgres.query", trace.WithSpanKind(trace.SpanKindClient),
|
||||
trace.WithAttributes(attribute.String("db.system.name", "postgresql"), attribute.String("db.operation.name", operation)))
|
||||
return ctx
|
||||
}
|
||||
|
||||
func (*postgresTrace) TraceQueryEnd(ctx context.Context, _ *pgx.Conn, data pgx.TraceQueryEndData) {
|
||||
span := trace.SpanFromContext(ctx)
|
||||
var pgErr *pgconn.PgError
|
||||
if errors.As(data.Err, &pgErr) {
|
||||
span.SetAttributes(attribute.String("db.response.status_code", pgErr.Code))
|
||||
}
|
||||
err := data.Err
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
err = nil
|
||||
}
|
||||
observability.End(span, err)
|
||||
}
|
||||
@@ -0,0 +1,31 @@
|
||||
package database
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
|
||||
"github.com/redis/go-redis/v9"
|
||||
)
|
||||
|
||||
func NewRedis(ctx context.Context, addr string) (*redis.Client, error) {
|
||||
|
||||
options := &redis.Options{
|
||||
Addr: addr,
|
||||
DialTimeout: 3 * time.Second,
|
||||
ReadTimeout: 3 * time.Second,
|
||||
WriteTimeout: 2 * time.Second,
|
||||
PoolSize: 10,
|
||||
PoolTimeout: 2 * time.Second,
|
||||
ContextTimeoutEnabled: true,
|
||||
}
|
||||
|
||||
client := redis.NewClient(options)
|
||||
|
||||
if err := client.Ping(ctx).Err(); err != nil {
|
||||
_ = client.Close()
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return client, nil
|
||||
|
||||
}
|
||||
@@ -0,0 +1,88 @@
|
||||
// Code generated by sqlc. DO NOT EDIT.
|
||||
// versions:
|
||||
// sqlc v1.31.1
|
||||
// source: attachment_variants.sql
|
||||
|
||||
package sqlc
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/jackc/pgx/v5/pgtype"
|
||||
)
|
||||
|
||||
const createAttachmentVariant = `-- name: CreateAttachmentVariant :exec
|
||||
INSERT INTO attachment_variants (
|
||||
attachment_id, pipeline_version, name, object_key, content_type, size_bytes, width, height, sha256
|
||||
) VALUES (
|
||||
$1, $2, $3, $4, $5, $6, $7, $8, $9
|
||||
)
|
||||
`
|
||||
|
||||
type CreateAttachmentVariantParams struct {
|
||||
AttachmentID pgtype.UUID
|
||||
PipelineVersion int32
|
||||
Name string
|
||||
ObjectKey string
|
||||
ContentType string
|
||||
SizeBytes int64
|
||||
Width int32
|
||||
Height int32
|
||||
Sha256 string
|
||||
}
|
||||
|
||||
func (q *Queries) CreateAttachmentVariant(ctx context.Context, arg CreateAttachmentVariantParams) error {
|
||||
_, err := q.db.Exec(ctx, createAttachmentVariant,
|
||||
arg.AttachmentID,
|
||||
arg.PipelineVersion,
|
||||
arg.Name,
|
||||
arg.ObjectKey,
|
||||
arg.ContentType,
|
||||
arg.SizeBytes,
|
||||
arg.Width,
|
||||
arg.Height,
|
||||
arg.Sha256,
|
||||
)
|
||||
return err
|
||||
}
|
||||
|
||||
const listAttachmentVariants = `-- name: ListAttachmentVariants :many
|
||||
SELECT attachment_id, pipeline_version, name, object_key, content_type, size_bytes, width, height, sha256 FROM attachment_variants
|
||||
WHERE attachment_id = $1 AND pipeline_version = $2
|
||||
ORDER BY name
|
||||
`
|
||||
|
||||
type ListAttachmentVariantsParams struct {
|
||||
AttachmentID pgtype.UUID
|
||||
PipelineVersion int32
|
||||
}
|
||||
|
||||
func (q *Queries) ListAttachmentVariants(ctx context.Context, arg ListAttachmentVariantsParams) ([]AttachmentVariant, error) {
|
||||
rows, err := q.db.Query(ctx, listAttachmentVariants, arg.AttachmentID, arg.PipelineVersion)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
var items []AttachmentVariant
|
||||
for rows.Next() {
|
||||
var i AttachmentVariant
|
||||
if err := rows.Scan(
|
||||
&i.AttachmentID,
|
||||
&i.PipelineVersion,
|
||||
&i.Name,
|
||||
&i.ObjectKey,
|
||||
&i.ContentType,
|
||||
&i.SizeBytes,
|
||||
&i.Width,
|
||||
&i.Height,
|
||||
&i.Sha256,
|
||||
); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
items = append(items, i)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return items, nil
|
||||
}
|
||||
@@ -0,0 +1,306 @@
|
||||
// Code generated by sqlc. DO NOT EDIT.
|
||||
// versions:
|
||||
// sqlc v1.31.1
|
||||
// source: attachments.sql
|
||||
|
||||
package sqlc
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/jackc/pgx/v5/pgtype"
|
||||
)
|
||||
|
||||
const attachmentExistsByObjectKey = `-- name: AttachmentExistsByObjectKey :one
|
||||
SELECT EXISTS (
|
||||
SELECT 1 FROM attachments
|
||||
WHERE object_key = $1::text OR staging_key = $1::text
|
||||
UNION ALL
|
||||
SELECT 1 FROM attachment_variants WHERE object_key = $1::text
|
||||
UNION ALL
|
||||
SELECT 1 FROM attachment_jobs WHERE published_original_key = $1::text
|
||||
)
|
||||
`
|
||||
|
||||
func (q *Queries) AttachmentExistsByObjectKey(ctx context.Context, objectKey string) (bool, error) {
|
||||
row := q.db.QueryRow(ctx, attachmentExistsByObjectKey, objectKey)
|
||||
var exists bool
|
||||
err := row.Scan(&exists)
|
||||
return exists, err
|
||||
}
|
||||
|
||||
const createAttachment = `-- name: CreateAttachment :one
|
||||
WITH legacy AS (
|
||||
INSERT INTO attachments
|
||||
(chat_id, owner_id, filename, object_key, staging_key, content_type, size_bytes, sha256)
|
||||
VALUES ($1, $2, $3,
|
||||
$4::text, $4::text,
|
||||
$5, $6, $7)
|
||||
RETURNING id, chat_id, owner_id, filename, object_key, content_type, size_bytes, sha256, created_at, updated_at, status, error_code, staging_key, pipeline_version
|
||||
), job AS (
|
||||
INSERT INTO attachment_jobs (attachment_id, pipeline_version)
|
||||
SELECT id, pipeline_version FROM legacy
|
||||
RETURNING id, attachment_id, pipeline_version, attempts_started, max_attempts, next_attempt_at, lease_generation, lease_until, created_at, updated_at, published_original_key
|
||||
), event AS (
|
||||
INSERT INTO outbox (job_id, payload, dedup_key)
|
||||
SELECT id, attachment_job_payload(id, attachment_id, pipeline_version, 1, created_at),
|
||||
'initial/' || id
|
||||
FROM job
|
||||
RETURNING job_id
|
||||
)
|
||||
SELECT legacy.id, legacy.chat_id, legacy.owner_id, legacy.filename, legacy.object_key, legacy.content_type, legacy.size_bytes, legacy.sha256, legacy.created_at, legacy.updated_at, legacy.status, legacy.error_code, legacy.staging_key, legacy.pipeline_version FROM legacy CROSS JOIN event
|
||||
`
|
||||
|
||||
type CreateAttachmentParams struct {
|
||||
ChatID pgtype.UUID
|
||||
OwnerID pgtype.UUID
|
||||
Filename string
|
||||
ObjectKey string
|
||||
ContentType string
|
||||
SizeBytes int64
|
||||
Sha256 string
|
||||
}
|
||||
|
||||
type CreateAttachmentRow struct {
|
||||
ID pgtype.UUID
|
||||
ChatID pgtype.UUID
|
||||
OwnerID pgtype.UUID
|
||||
Filename string
|
||||
ObjectKey pgtype.Text
|
||||
ContentType string
|
||||
SizeBytes int64
|
||||
Sha256 string
|
||||
CreatedAt pgtype.Timestamptz
|
||||
UpdatedAt pgtype.Timestamptz
|
||||
Status string
|
||||
ErrorCode pgtype.Text
|
||||
StagingKey pgtype.Text
|
||||
PipelineVersion int32
|
||||
}
|
||||
|
||||
// Compatibility for the synchronous API until CreateQueuedAttachment is wired in.
|
||||
func (q *Queries) CreateAttachment(ctx context.Context, arg CreateAttachmentParams) (CreateAttachmentRow, error) {
|
||||
row := q.db.QueryRow(ctx, createAttachment,
|
||||
arg.ChatID,
|
||||
arg.OwnerID,
|
||||
arg.Filename,
|
||||
arg.ObjectKey,
|
||||
arg.ContentType,
|
||||
arg.SizeBytes,
|
||||
arg.Sha256,
|
||||
)
|
||||
var i CreateAttachmentRow
|
||||
err := row.Scan(
|
||||
&i.ID,
|
||||
&i.ChatID,
|
||||
&i.OwnerID,
|
||||
&i.Filename,
|
||||
&i.ObjectKey,
|
||||
&i.ContentType,
|
||||
&i.SizeBytes,
|
||||
&i.Sha256,
|
||||
&i.CreatedAt,
|
||||
&i.UpdatedAt,
|
||||
&i.Status,
|
||||
&i.ErrorCode,
|
||||
&i.StagingKey,
|
||||
&i.PipelineVersion,
|
||||
)
|
||||
return i, err
|
||||
}
|
||||
|
||||
const createQueuedAttachment = `-- name: CreateQueuedAttachment :one
|
||||
INSERT INTO attachments (
|
||||
id, chat_id, owner_id, filename, content_type, size_bytes, sha256, staging_key, status
|
||||
) VALUES (
|
||||
$1, $2, $3, $4, $5, $6, $7, $8, 'queued'
|
||||
) RETURNING id, chat_id, owner_id, filename, object_key, content_type, size_bytes, sha256, created_at, updated_at, status, error_code, staging_key, pipeline_version
|
||||
`
|
||||
|
||||
type CreateQueuedAttachmentParams struct {
|
||||
ID pgtype.UUID
|
||||
ChatID pgtype.UUID
|
||||
OwnerID pgtype.UUID
|
||||
Filename string
|
||||
ContentType string
|
||||
SizeBytes int64
|
||||
Sha256 string
|
||||
StagingKey pgtype.Text
|
||||
}
|
||||
|
||||
func (q *Queries) CreateQueuedAttachment(ctx context.Context, arg CreateQueuedAttachmentParams) (Attachment, error) {
|
||||
row := q.db.QueryRow(ctx, createQueuedAttachment,
|
||||
arg.ID,
|
||||
arg.ChatID,
|
||||
arg.OwnerID,
|
||||
arg.Filename,
|
||||
arg.ContentType,
|
||||
arg.SizeBytes,
|
||||
arg.Sha256,
|
||||
arg.StagingKey,
|
||||
)
|
||||
var i Attachment
|
||||
err := row.Scan(
|
||||
&i.ID,
|
||||
&i.ChatID,
|
||||
&i.OwnerID,
|
||||
&i.Filename,
|
||||
&i.ObjectKey,
|
||||
&i.ContentType,
|
||||
&i.SizeBytes,
|
||||
&i.Sha256,
|
||||
&i.CreatedAt,
|
||||
&i.UpdatedAt,
|
||||
&i.Status,
|
||||
&i.ErrorCode,
|
||||
&i.StagingKey,
|
||||
&i.PipelineVersion,
|
||||
)
|
||||
return i, err
|
||||
}
|
||||
|
||||
const getAttachmentByID = `-- name: GetAttachmentByID :one
|
||||
SELECT id, chat_id, owner_id, filename, object_key, content_type, size_bytes, sha256, created_at, updated_at, status, error_code, staging_key, pipeline_version FROM attachments WHERE id = $1
|
||||
`
|
||||
|
||||
func (q *Queries) GetAttachmentByID(ctx context.Context, attachmentID pgtype.UUID) (Attachment, error) {
|
||||
row := q.db.QueryRow(ctx, getAttachmentByID, attachmentID)
|
||||
var i Attachment
|
||||
err := row.Scan(
|
||||
&i.ID,
|
||||
&i.ChatID,
|
||||
&i.OwnerID,
|
||||
&i.Filename,
|
||||
&i.ObjectKey,
|
||||
&i.ContentType,
|
||||
&i.SizeBytes,
|
||||
&i.Sha256,
|
||||
&i.CreatedAt,
|
||||
&i.UpdatedAt,
|
||||
&i.Status,
|
||||
&i.ErrorCode,
|
||||
&i.StagingKey,
|
||||
&i.PipelineVersion,
|
||||
)
|
||||
return i, err
|
||||
}
|
||||
|
||||
const getAttachmentForUser = `-- name: GetAttachmentForUser :one
|
||||
SELECT a.id, a.chat_id, a.owner_id, a.filename, a.object_key, a.content_type, a.size_bytes, a.sha256, a.created_at, a.updated_at, a.status, a.error_code, a.staging_key, a.pipeline_version
|
||||
FROM attachments AS a
|
||||
JOIN chat_members AS cm ON cm.chat_id = a.chat_id
|
||||
WHERE a.id = $1
|
||||
AND cm.user_id = $2
|
||||
`
|
||||
|
||||
type GetAttachmentForUserParams struct {
|
||||
AttachmentID pgtype.UUID
|
||||
UserID pgtype.UUID
|
||||
}
|
||||
|
||||
func (q *Queries) GetAttachmentForUser(ctx context.Context, arg GetAttachmentForUserParams) (Attachment, error) {
|
||||
row := q.db.QueryRow(ctx, getAttachmentForUser, arg.AttachmentID, arg.UserID)
|
||||
var i Attachment
|
||||
err := row.Scan(
|
||||
&i.ID,
|
||||
&i.ChatID,
|
||||
&i.OwnerID,
|
||||
&i.Filename,
|
||||
&i.ObjectKey,
|
||||
&i.ContentType,
|
||||
&i.SizeBytes,
|
||||
&i.Sha256,
|
||||
&i.CreatedAt,
|
||||
&i.UpdatedAt,
|
||||
&i.Status,
|
||||
&i.ErrorCode,
|
||||
&i.StagingKey,
|
||||
&i.PipelineVersion,
|
||||
)
|
||||
return i, err
|
||||
}
|
||||
|
||||
const getAttachmentsForMessages = `-- name: GetAttachmentsForMessages :many
|
||||
SELECT message_id, attachment_id, position
|
||||
FROM messages_attachments
|
||||
WHERE message_id = ANY($1::uuid[])
|
||||
ORDER BY message_id, position
|
||||
`
|
||||
|
||||
type GetAttachmentsForMessagesRow struct {
|
||||
MessageID pgtype.UUID
|
||||
AttachmentID pgtype.UUID
|
||||
Position int16
|
||||
}
|
||||
|
||||
func (q *Queries) GetAttachmentsForMessages(ctx context.Context, messageIds []pgtype.UUID) ([]GetAttachmentsForMessagesRow, error) {
|
||||
rows, err := q.db.Query(ctx, getAttachmentsForMessages, messageIds)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
var items []GetAttachmentsForMessagesRow
|
||||
for rows.Next() {
|
||||
var i GetAttachmentsForMessagesRow
|
||||
if err := rows.Scan(&i.MessageID, &i.AttachmentID, &i.Position); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
items = append(items, i)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return items, nil
|
||||
}
|
||||
|
||||
const listAttachmentKeys = `-- name: ListAttachmentKeys :many
|
||||
SELECT id, object_key, created_at
|
||||
FROM attachments
|
||||
WHERE object_key > $1
|
||||
ORDER BY object_key
|
||||
LIMIT $2
|
||||
`
|
||||
|
||||
type ListAttachmentKeysParams struct {
|
||||
AfterKey pgtype.Text
|
||||
PageSize int32
|
||||
}
|
||||
|
||||
type ListAttachmentKeysRow struct {
|
||||
ID pgtype.UUID
|
||||
ObjectKey pgtype.Text
|
||||
CreatedAt pgtype.Timestamptz
|
||||
}
|
||||
|
||||
func (q *Queries) ListAttachmentKeys(ctx context.Context, arg ListAttachmentKeysParams) ([]ListAttachmentKeysRow, error) {
|
||||
rows, err := q.db.Query(ctx, listAttachmentKeys, arg.AfterKey, arg.PageSize)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
var items []ListAttachmentKeysRow
|
||||
for rows.Next() {
|
||||
var i ListAttachmentKeysRow
|
||||
if err := rows.Scan(&i.ID, &i.ObjectKey, &i.CreatedAt); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
items = append(items, i)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return items, nil
|
||||
}
|
||||
|
||||
const tryLockAttachmentObjectKey = `-- name: TryLockAttachmentObjectKey :one
|
||||
SELECT pg_try_advisory_xact_lock(
|
||||
hashtextextended($1::text, 0)
|
||||
)::boolean AS locked
|
||||
`
|
||||
|
||||
func (q *Queries) TryLockAttachmentObjectKey(ctx context.Context, objectKey string) (bool, error) {
|
||||
row := q.db.QueryRow(ctx, tryLockAttachmentObjectKey, objectKey)
|
||||
var locked bool
|
||||
err := row.Scan(&locked)
|
||||
return locked, err
|
||||
}
|
||||
@@ -0,0 +1,556 @@
|
||||
// Code generated by sqlc. DO NOT EDIT.
|
||||
// versions:
|
||||
// sqlc v1.31.1
|
||||
// source: attachments_jobs.sql
|
||||
|
||||
package sqlc
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/jackc/pgx/v5/pgtype"
|
||||
)
|
||||
|
||||
const claimJob = `-- name: ClaimJob :one
|
||||
WITH claimed AS (
|
||||
UPDATE attachment_jobs AS j
|
||||
SET lease_generation = j.lease_generation + 1,
|
||||
lease_until = $1::timestamptz,
|
||||
attempts_started = j.attempts_started + 1,
|
||||
next_attempt_at = NULL,
|
||||
updated_at = clock_timestamp()
|
||||
FROM attachments AS a
|
||||
WHERE j.id = $2
|
||||
AND a.id = j.attachment_id AND a.pipeline_version = j.pipeline_version
|
||||
AND j.attempts_started < j.max_attempts
|
||||
AND $1::timestamptz > clock_timestamp()
|
||||
AND (
|
||||
(a.status = 'queued' AND j.lease_until IS NULL AND j.next_attempt_at <= clock_timestamp())
|
||||
OR (a.status = 'processing' AND j.lease_until <= clock_timestamp())
|
||||
)
|
||||
RETURNING j.id, j.attachment_id, j.pipeline_version, j.attempts_started, j.max_attempts, j.next_attempt_at, j.lease_generation, j.lease_until, j.created_at, j.updated_at, j.published_original_key
|
||||
), expired AS (
|
||||
UPDATE attachment_attempts AS attempt
|
||||
SET outcome = 'lease_expired', error_code = 'lease_expired', finished_at = claimed.updated_at
|
||||
FROM claimed
|
||||
WHERE attempt.job_id = claimed.id
|
||||
AND attempt.lease_generation = claimed.lease_generation - 1 AND attempt.finished_at IS NULL
|
||||
), recorded AS (
|
||||
INSERT INTO attachment_attempts (job_id, lease_generation, attempt, started_at)
|
||||
SELECT id, lease_generation, attempts_started, updated_at FROM claimed
|
||||
RETURNING job_id
|
||||
), processing AS (
|
||||
UPDATE attachments AS a
|
||||
SET status = 'processing', updated_at = claimed.updated_at
|
||||
FROM claimed
|
||||
WHERE a.id = claimed.attachment_id
|
||||
RETURNING a.id
|
||||
)
|
||||
SELECT claimed.id, claimed.attachment_id, claimed.pipeline_version, claimed.attempts_started, claimed.max_attempts, claimed.next_attempt_at, claimed.lease_generation, claimed.lease_until, claimed.created_at, claimed.updated_at, claimed.published_original_key FROM claimed JOIN recorded ON recorded.job_id = claimed.id
|
||||
JOIN processing ON processing.id = claimed.attachment_id
|
||||
`
|
||||
|
||||
type ClaimJobParams struct {
|
||||
NewLeaseUntil pgtype.Timestamptz
|
||||
JobID pgtype.UUID
|
||||
}
|
||||
|
||||
type ClaimJobRow struct {
|
||||
ID pgtype.UUID
|
||||
AttachmentID pgtype.UUID
|
||||
PipelineVersion int32
|
||||
AttemptsStarted int32
|
||||
MaxAttempts int32
|
||||
NextAttemptAt pgtype.Timestamptz
|
||||
LeaseGeneration int32
|
||||
LeaseUntil pgtype.Timestamptz
|
||||
CreatedAt pgtype.Timestamptz
|
||||
UpdatedAt pgtype.Timestamptz
|
||||
PublishedOriginalKey pgtype.Text
|
||||
}
|
||||
|
||||
func (q *Queries) ClaimJob(ctx context.Context, arg ClaimJobParams) (ClaimJobRow, error) {
|
||||
row := q.db.QueryRow(ctx, claimJob, arg.NewLeaseUntil, arg.JobID)
|
||||
var i ClaimJobRow
|
||||
err := row.Scan(
|
||||
&i.ID,
|
||||
&i.AttachmentID,
|
||||
&i.PipelineVersion,
|
||||
&i.AttemptsStarted,
|
||||
&i.MaxAttempts,
|
||||
&i.NextAttemptAt,
|
||||
&i.LeaseGeneration,
|
||||
&i.LeaseUntil,
|
||||
&i.CreatedAt,
|
||||
&i.UpdatedAt,
|
||||
&i.PublishedOriginalKey,
|
||||
)
|
||||
return i, err
|
||||
}
|
||||
|
||||
const clearStagingReference = `-- name: ClearStagingReference :execrows
|
||||
UPDATE attachments
|
||||
SET object_key = CASE WHEN status = 'failed' AND object_key = staging_key THEN NULL ELSE object_key END,
|
||||
staging_key = NULL
|
||||
WHERE id = $1 AND staging_key = $2
|
||||
`
|
||||
|
||||
type ClearStagingReferenceParams struct {
|
||||
ID pgtype.UUID
|
||||
StagingKey pgtype.Text
|
||||
}
|
||||
|
||||
func (q *Queries) ClearStagingReference(ctx context.Context, arg ClearStagingReferenceParams) (int64, error) {
|
||||
result, err := q.db.Exec(ctx, clearStagingReference, arg.ID, arg.StagingKey)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return result.RowsAffected(), nil
|
||||
}
|
||||
|
||||
const completeJob = `-- name: CompleteJob :execrows
|
||||
WITH completed AS (
|
||||
UPDATE attachment_jobs AS j
|
||||
SET lease_until = NULL, next_attempt_at = NULL, updated_at = clock_timestamp(),
|
||||
published_original_key = $1::text
|
||||
FROM attachments AS a
|
||||
WHERE j.id = $2
|
||||
AND j.lease_generation = $3 AND j.lease_until > clock_timestamp()
|
||||
AND a.id = j.attachment_id AND a.pipeline_version = j.pipeline_version AND a.status = 'processing'
|
||||
AND (SELECT count(*) FROM attachment_variants AS v
|
||||
WHERE v.attachment_id = a.id AND v.pipeline_version = j.pipeline_version) = 3
|
||||
RETURNING j.id, j.attachment_id, j.pipeline_version, j.attempts_started, j.max_attempts, j.next_attempt_at, j.lease_generation, j.lease_until, j.created_at, j.updated_at, j.published_original_key
|
||||
), recorded AS (
|
||||
UPDATE attachment_attempts AS attempt
|
||||
SET outcome = 'ready', finished_at = completed.updated_at
|
||||
FROM completed
|
||||
WHERE attempt.job_id = completed.id AND attempt.lease_generation = completed.lease_generation
|
||||
)
|
||||
UPDATE attachments AS a
|
||||
SET status = 'ready', object_key = $1::text, updated_at = completed.updated_at
|
||||
FROM completed
|
||||
WHERE a.id = completed.attachment_id
|
||||
`
|
||||
|
||||
type CompleteJobParams struct {
|
||||
ObjectKey string
|
||||
JobID pgtype.UUID
|
||||
LeaseGeneration int32
|
||||
}
|
||||
|
||||
func (q *Queries) CompleteJob(ctx context.Context, arg CompleteJobParams) (int64, error) {
|
||||
result, err := q.db.Exec(ctx, completeJob, arg.ObjectKey, arg.JobID, arg.LeaseGeneration)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return result.RowsAffected(), nil
|
||||
}
|
||||
|
||||
const createAttachmentJob = `-- name: CreateAttachmentJob :one
|
||||
INSERT INTO attachment_jobs (attachment_id, pipeline_version, next_attempt_at, max_attempts, created_at)
|
||||
VALUES ($1, $2,
|
||||
$3::timestamptz, $4, clock_timestamp())
|
||||
RETURNING id, attachment_id, pipeline_version, attempts_started, max_attempts, next_attempt_at, lease_generation, lease_until, created_at, updated_at, published_original_key
|
||||
`
|
||||
|
||||
type CreateAttachmentJobParams struct {
|
||||
AttachmentID pgtype.UUID
|
||||
PipelineVersion int32
|
||||
AvailableAt pgtype.Timestamptz
|
||||
MaxAttempts int32
|
||||
}
|
||||
|
||||
func (q *Queries) CreateAttachmentJob(ctx context.Context, arg CreateAttachmentJobParams) (AttachmentJob, error) {
|
||||
row := q.db.QueryRow(ctx, createAttachmentJob,
|
||||
arg.AttachmentID,
|
||||
arg.PipelineVersion,
|
||||
arg.AvailableAt,
|
||||
arg.MaxAttempts,
|
||||
)
|
||||
var i AttachmentJob
|
||||
err := row.Scan(
|
||||
&i.ID,
|
||||
&i.AttachmentID,
|
||||
&i.PipelineVersion,
|
||||
&i.AttemptsStarted,
|
||||
&i.MaxAttempts,
|
||||
&i.NextAttemptAt,
|
||||
&i.LeaseGeneration,
|
||||
&i.LeaseUntil,
|
||||
&i.CreatedAt,
|
||||
&i.UpdatedAt,
|
||||
&i.PublishedOriginalKey,
|
||||
)
|
||||
return i, err
|
||||
}
|
||||
|
||||
const failJob = `-- name: FailJob :execrows
|
||||
WITH failed AS (
|
||||
UPDATE attachment_jobs AS j
|
||||
SET lease_until = NULL, next_attempt_at = NULL, updated_at = clock_timestamp()
|
||||
FROM attachments AS a
|
||||
WHERE j.id = $2
|
||||
AND j.lease_generation = $3
|
||||
AND (
|
||||
j.lease_until > clock_timestamp()
|
||||
OR (j.lease_until <= clock_timestamp() AND j.attempts_started = j.max_attempts
|
||||
AND $1::text = 'retry_exhausted')
|
||||
)
|
||||
AND a.id = j.attachment_id AND a.pipeline_version = j.pipeline_version AND a.status = 'processing'
|
||||
RETURNING j.id, j.attachment_id, j.pipeline_version, j.attempts_started, j.max_attempts, j.next_attempt_at, j.lease_generation, j.lease_until, j.created_at, j.updated_at, j.published_original_key
|
||||
), recorded AS (
|
||||
UPDATE attachment_attempts AS attempt
|
||||
SET outcome = 'failed', error_code = $1::text, finished_at = failed.updated_at
|
||||
FROM failed
|
||||
WHERE attempt.job_id = failed.id AND attempt.lease_generation = failed.lease_generation
|
||||
)
|
||||
UPDATE attachments AS a
|
||||
SET status = 'failed', error_code = $1::text, updated_at = failed.updated_at
|
||||
FROM failed WHERE a.id = failed.attachment_id
|
||||
`
|
||||
|
||||
type FailJobParams struct {
|
||||
ErrorCode string
|
||||
JobID pgtype.UUID
|
||||
LeaseGeneration int32
|
||||
}
|
||||
|
||||
func (q *Queries) FailJob(ctx context.Context, arg FailJobParams) (int64, error) {
|
||||
result, err := q.db.Exec(ctx, failJob, arg.ErrorCode, arg.JobID, arg.LeaseGeneration)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return result.RowsAffected(), nil
|
||||
}
|
||||
|
||||
const getAttachmentByStagingKey = `-- name: GetAttachmentByStagingKey :one
|
||||
SELECT id, chat_id, owner_id, filename, object_key, content_type, size_bytes, sha256, created_at, updated_at, status, error_code, staging_key, pipeline_version FROM attachments WHERE staging_key = $1::text
|
||||
`
|
||||
|
||||
func (q *Queries) GetAttachmentByStagingKey(ctx context.Context, stagingKey string) (Attachment, error) {
|
||||
row := q.db.QueryRow(ctx, getAttachmentByStagingKey, stagingKey)
|
||||
var i Attachment
|
||||
err := row.Scan(
|
||||
&i.ID,
|
||||
&i.ChatID,
|
||||
&i.OwnerID,
|
||||
&i.Filename,
|
||||
&i.ObjectKey,
|
||||
&i.ContentType,
|
||||
&i.SizeBytes,
|
||||
&i.Sha256,
|
||||
&i.CreatedAt,
|
||||
&i.UpdatedAt,
|
||||
&i.Status,
|
||||
&i.ErrorCode,
|
||||
&i.StagingKey,
|
||||
&i.PipelineVersion,
|
||||
)
|
||||
return i, err
|
||||
}
|
||||
|
||||
const getAttachmentJobState = `-- name: GetAttachmentJobState :one
|
||||
SELECT
|
||||
j.id, j.attachment_id, j.pipeline_version,
|
||||
CASE WHEN a.pipeline_version = j.pipeline_version THEN a.status ELSE 'superseded' END::text AS status,
|
||||
j.lease_generation,
|
||||
j.lease_until,
|
||||
coalesce(j.lease_until <= clock_timestamp(), false)::boolean AS lease_expired,
|
||||
j.attempts_started,
|
||||
j.max_attempts,
|
||||
j.next_attempt_at
|
||||
FROM attachment_jobs AS j
|
||||
JOIN attachments AS a ON a.id = j.attachment_id
|
||||
WHERE j.id = $1::uuid
|
||||
`
|
||||
|
||||
type GetAttachmentJobStateRow struct {
|
||||
ID pgtype.UUID
|
||||
AttachmentID pgtype.UUID
|
||||
PipelineVersion int32
|
||||
Status string
|
||||
LeaseGeneration int32
|
||||
LeaseUntil pgtype.Timestamptz
|
||||
LeaseExpired bool
|
||||
AttemptsStarted int32
|
||||
MaxAttempts int32
|
||||
NextAttemptAt pgtype.Timestamptz
|
||||
}
|
||||
|
||||
func (q *Queries) GetAttachmentJobState(ctx context.Context, jobID pgtype.UUID) (GetAttachmentJobStateRow, error) {
|
||||
row := q.db.QueryRow(ctx, getAttachmentJobState, jobID)
|
||||
var i GetAttachmentJobStateRow
|
||||
err := row.Scan(
|
||||
&i.ID,
|
||||
&i.AttachmentID,
|
||||
&i.PipelineVersion,
|
||||
&i.Status,
|
||||
&i.LeaseGeneration,
|
||||
&i.LeaseUntil,
|
||||
&i.LeaseExpired,
|
||||
&i.AttemptsStarted,
|
||||
&i.MaxAttempts,
|
||||
&i.NextAttemptAt,
|
||||
)
|
||||
return i, err
|
||||
}
|
||||
|
||||
const getQueueStats = `-- name: GetQueueStats :one
|
||||
SELECT count(*)::bigint AS depth,
|
||||
COALESCE(EXTRACT(EPOCH FROM (statement_timestamp() - min(j.created_at))), 0)::double precision AS oldest_seconds
|
||||
FROM attachment_jobs AS j JOIN attachments AS a ON a.id = j.attachment_id
|
||||
WHERE a.status = 'queued' AND a.pipeline_version = j.pipeline_version
|
||||
`
|
||||
|
||||
type GetQueueStatsRow struct {
|
||||
Depth int64
|
||||
OldestSeconds float64
|
||||
}
|
||||
|
||||
func (q *Queries) GetQueueStats(ctx context.Context) (GetQueueStatsRow, error) {
|
||||
row := q.db.QueryRow(ctx, getQueueStats)
|
||||
var i GetQueueStatsRow
|
||||
err := row.Scan(&i.Depth, &i.OldestSeconds)
|
||||
return i, err
|
||||
}
|
||||
|
||||
const listFailedAttachmentJobs = `-- name: ListFailedAttachmentJobs :many
|
||||
SELECT j.id, j.attachment_id, j.attempts_started, a.error_code, a.updated_at
|
||||
FROM attachment_jobs j
|
||||
JOIN attachments a ON a.id = j.attachment_id AND a.pipeline_version = j.pipeline_version
|
||||
WHERE a.status = 'failed'
|
||||
ORDER BY a.updated_at DESC, j.id
|
||||
LIMIT $1
|
||||
`
|
||||
|
||||
type ListFailedAttachmentJobsRow struct {
|
||||
ID pgtype.UUID
|
||||
AttachmentID pgtype.UUID
|
||||
AttemptsStarted int32
|
||||
ErrorCode pgtype.Text
|
||||
UpdatedAt pgtype.Timestamptz
|
||||
}
|
||||
|
||||
func (q *Queries) ListFailedAttachmentJobs(ctx context.Context, batchSize int32) ([]ListFailedAttachmentJobsRow, error) {
|
||||
rows, err := q.db.Query(ctx, listFailedAttachmentJobs, batchSize)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
var items []ListFailedAttachmentJobsRow
|
||||
for rows.Next() {
|
||||
var i ListFailedAttachmentJobsRow
|
||||
if err := rows.Scan(
|
||||
&i.ID,
|
||||
&i.AttachmentID,
|
||||
&i.AttemptsStarted,
|
||||
&i.ErrorCode,
|
||||
&i.UpdatedAt,
|
||||
); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
items = append(items, i)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return items, nil
|
||||
}
|
||||
|
||||
const lockActiveAttachmentJob = `-- name: LockActiveAttachmentJob :one
|
||||
SELECT j.id, j.attachment_id, j.pipeline_version, j.attempts_started, j.max_attempts, j.next_attempt_at, j.lease_generation, j.lease_until, j.created_at, j.updated_at, j.published_original_key
|
||||
FROM attachment_jobs AS j JOIN attachments AS a ON a.id = j.attachment_id
|
||||
WHERE j.id = $1
|
||||
AND j.lease_generation = $2
|
||||
AND j.lease_until > clock_timestamp()
|
||||
AND a.status = 'processing' AND a.pipeline_version = j.pipeline_version
|
||||
FOR UPDATE OF j
|
||||
`
|
||||
|
||||
type LockActiveAttachmentJobParams struct {
|
||||
JobID pgtype.UUID
|
||||
LeaseGeneration int32
|
||||
}
|
||||
|
||||
// Hold this lock through variant insertion and CompleteJob. Roll back on a zero-row completion.
|
||||
func (q *Queries) LockActiveAttachmentJob(ctx context.Context, arg LockActiveAttachmentJobParams) (AttachmentJob, error) {
|
||||
row := q.db.QueryRow(ctx, lockActiveAttachmentJob, arg.JobID, arg.LeaseGeneration)
|
||||
var i AttachmentJob
|
||||
err := row.Scan(
|
||||
&i.ID,
|
||||
&i.AttachmentID,
|
||||
&i.PipelineVersion,
|
||||
&i.AttemptsStarted,
|
||||
&i.MaxAttempts,
|
||||
&i.NextAttemptAt,
|
||||
&i.LeaseGeneration,
|
||||
&i.LeaseUntil,
|
||||
&i.CreatedAt,
|
||||
&i.UpdatedAt,
|
||||
&i.PublishedOriginalKey,
|
||||
)
|
||||
return i, err
|
||||
}
|
||||
|
||||
const lockAttachmentJobForPipeline = `-- name: LockAttachmentJobForPipeline :one
|
||||
SELECT j.id, j.attachment_id, j.pipeline_version, j.attempts_started, j.max_attempts, j.next_attempt_at, j.lease_generation, j.lease_until, j.created_at, j.updated_at, j.published_original_key
|
||||
FROM attachment_jobs j
|
||||
WHERE j.attachment_id = $1 AND j.pipeline_version = $2
|
||||
FOR UPDATE
|
||||
`
|
||||
|
||||
type LockAttachmentJobForPipelineParams struct {
|
||||
AttachmentID pgtype.UUID
|
||||
PipelineVersion int32
|
||||
}
|
||||
|
||||
func (q *Queries) LockAttachmentJobForPipeline(ctx context.Context, arg LockAttachmentJobForPipelineParams) (AttachmentJob, error) {
|
||||
row := q.db.QueryRow(ctx, lockAttachmentJobForPipeline, arg.AttachmentID, arg.PipelineVersion)
|
||||
var i AttachmentJob
|
||||
err := row.Scan(
|
||||
&i.ID,
|
||||
&i.AttachmentID,
|
||||
&i.PipelineVersion,
|
||||
&i.AttemptsStarted,
|
||||
&i.MaxAttempts,
|
||||
&i.NextAttemptAt,
|
||||
&i.LeaseGeneration,
|
||||
&i.LeaseUntil,
|
||||
&i.CreatedAt,
|
||||
&i.UpdatedAt,
|
||||
&i.PublishedOriginalKey,
|
||||
)
|
||||
return i, err
|
||||
}
|
||||
|
||||
const lockAttachmentJobForReplay = `-- name: LockAttachmentJobForReplay :one
|
||||
SELECT j.id, j.attachment_id, j.pipeline_version, j.attempts_started, j.max_attempts, j.next_attempt_at, j.lease_generation, j.lease_until, j.created_at, j.updated_at, j.published_original_key
|
||||
FROM attachment_jobs j WHERE j.id = $1
|
||||
FOR UPDATE
|
||||
`
|
||||
|
||||
func (q *Queries) LockAttachmentJobForReplay(ctx context.Context, id pgtype.UUID) (AttachmentJob, error) {
|
||||
row := q.db.QueryRow(ctx, lockAttachmentJobForReplay, id)
|
||||
var i AttachmentJob
|
||||
err := row.Scan(
|
||||
&i.ID,
|
||||
&i.AttachmentID,
|
||||
&i.PipelineVersion,
|
||||
&i.AttemptsStarted,
|
||||
&i.MaxAttempts,
|
||||
&i.NextAttemptAt,
|
||||
&i.LeaseGeneration,
|
||||
&i.LeaseUntil,
|
||||
&i.CreatedAt,
|
||||
&i.UpdatedAt,
|
||||
&i.PublishedOriginalKey,
|
||||
)
|
||||
return i, err
|
||||
}
|
||||
|
||||
const replayFailed = `-- name: ReplayFailed :execrows
|
||||
WITH replayed AS (
|
||||
UPDATE attachment_jobs AS j
|
||||
SET max_attempts = j.attempts_started + $1::int,
|
||||
next_attempt_at = clock_timestamp(), updated_at = clock_timestamp()
|
||||
FROM attachments AS a
|
||||
WHERE j.id = $2
|
||||
AND a.id = j.attachment_id AND a.pipeline_version = j.pipeline_version
|
||||
AND a.status = 'failed' AND a.staging_key IS NOT NULL
|
||||
AND j.lease_until IS NULL AND j.next_attempt_at IS NULL
|
||||
AND NOT EXISTS (
|
||||
SELECT 1 FROM attachment_replays AS r WHERE r.request_id = $3::uuid
|
||||
)
|
||||
RETURNING j.id, j.attachment_id, j.pipeline_version, j.attempts_started, j.max_attempts, j.next_attempt_at, j.lease_generation, j.lease_until, j.created_at, j.updated_at, j.published_original_key
|
||||
), recorded AS (
|
||||
INSERT INTO attachment_replays (request_id, job_id, operator_id, reason, extra_attempts)
|
||||
SELECT $3::uuid, id, $4, $5, $1
|
||||
FROM replayed
|
||||
), event AS (
|
||||
INSERT INTO outbox (job_id, payload, available_at, dedup_key)
|
||||
SELECT id, attachment_job_payload(id, attachment_id, pipeline_version, attempts_started + 1, updated_at),
|
||||
next_attempt_at, 'replay/' || id || '/' || $3::uuid
|
||||
FROM replayed
|
||||
)
|
||||
UPDATE attachments AS a
|
||||
SET status = 'queued', error_code = NULL, updated_at = replayed.updated_at
|
||||
FROM replayed WHERE a.id = replayed.attachment_id
|
||||
`
|
||||
|
||||
type ReplayFailedParams struct {
|
||||
ExtraAttempts int32
|
||||
JobID pgtype.UUID
|
||||
RequestID pgtype.UUID
|
||||
OperatorID string
|
||||
Reason string
|
||||
}
|
||||
|
||||
func (q *Queries) ReplayFailed(ctx context.Context, arg ReplayFailedParams) (int64, error) {
|
||||
result, err := q.db.Exec(ctx, replayFailed,
|
||||
arg.ExtraAttempts,
|
||||
arg.JobID,
|
||||
arg.RequestID,
|
||||
arg.OperatorID,
|
||||
arg.Reason,
|
||||
)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return result.RowsAffected(), nil
|
||||
}
|
||||
|
||||
const scheduleRetry = `-- name: ScheduleRetry :execrows
|
||||
WITH scheduled AS (
|
||||
UPDATE attachment_jobs AS j
|
||||
SET lease_until = NULL, next_attempt_at = $1::timestamptz,
|
||||
updated_at = clock_timestamp()
|
||||
FROM attachments AS a
|
||||
WHERE j.id = $2
|
||||
AND j.lease_generation = $3 AND j.lease_until > clock_timestamp()
|
||||
AND a.id = j.attachment_id AND a.pipeline_version = j.pipeline_version AND a.status = 'processing'
|
||||
AND j.attempts_started < j.max_attempts
|
||||
AND $1::timestamptz > clock_timestamp()
|
||||
RETURNING j.id, j.attachment_id, j.pipeline_version, j.attempts_started, j.max_attempts, j.next_attempt_at, j.lease_generation, j.lease_until, j.created_at, j.updated_at, j.published_original_key
|
||||
), recorded AS (
|
||||
UPDATE attachment_attempts AS attempt
|
||||
SET outcome = 'retry', error_code = $4::text, finished_at = scheduled.updated_at
|
||||
FROM scheduled
|
||||
WHERE attempt.job_id = scheduled.id AND attempt.lease_generation = scheduled.lease_generation
|
||||
), event AS (
|
||||
INSERT INTO outbox (job_id, payload, available_at, dedup_key)
|
||||
SELECT id, attachment_job_payload(id, attachment_id, pipeline_version, attempts_started + 1, updated_at)
|
||||
|| jsonb_strip_nulls(jsonb_build_object(
|
||||
'traceparent', NULLIF($5::text, ''),
|
||||
'tracestate', NULLIF($6::text, '')
|
||||
)),
|
||||
next_attempt_at, 'retry/' || id || '/' || (attempts_started + 1)
|
||||
FROM scheduled
|
||||
)
|
||||
UPDATE attachments AS a
|
||||
SET status = 'queued', updated_at = scheduled.updated_at
|
||||
FROM scheduled WHERE a.id = scheduled.attachment_id
|
||||
`
|
||||
|
||||
type ScheduleRetryParams struct {
|
||||
AvailableAt pgtype.Timestamptz
|
||||
JobID pgtype.UUID
|
||||
LeaseGeneration int32
|
||||
ErrorCode string
|
||||
Traceparent string
|
||||
Tracestate string
|
||||
}
|
||||
|
||||
func (q *Queries) ScheduleRetry(ctx context.Context, arg ScheduleRetryParams) (int64, error) {
|
||||
result, err := q.db.Exec(ctx, scheduleRetry,
|
||||
arg.AvailableAt,
|
||||
arg.JobID,
|
||||
arg.LeaseGeneration,
|
||||
arg.ErrorCode,
|
||||
arg.Traceparent,
|
||||
arg.Tracestate,
|
||||
)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return result.RowsAffected(), nil
|
||||
}
|
||||
@@ -0,0 +1,27 @@
|
||||
// Code generated by sqlc. DO NOT EDIT.
|
||||
// versions:
|
||||
// sqlc v1.31.1
|
||||
// source: chat_members.sql
|
||||
|
||||
package sqlc
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/jackc/pgx/v5/pgtype"
|
||||
)
|
||||
|
||||
const createChatMember = `-- name: CreateChatMember :exec
|
||||
INSERT INTO chat_members(chat_id, user_id)
|
||||
VALUES ($1, $2)
|
||||
`
|
||||
|
||||
type CreateChatMemberParams struct {
|
||||
ChatID pgtype.UUID
|
||||
UserID pgtype.UUID
|
||||
}
|
||||
|
||||
func (q *Queries) CreateChatMember(ctx context.Context, arg CreateChatMemberParams) error {
|
||||
_, err := q.db.Exec(ctx, createChatMember, arg.ChatID, arg.UserID)
|
||||
return err
|
||||
}
|
||||
@@ -0,0 +1,118 @@
|
||||
// Code generated by sqlc. DO NOT EDIT.
|
||||
// versions:
|
||||
// sqlc v1.31.1
|
||||
// source: chats.sql
|
||||
|
||||
package sqlc
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/jackc/pgx/v5/pgtype"
|
||||
)
|
||||
|
||||
const createChat = `-- name: CreateChat :one
|
||||
INSERT INTO chats (title)
|
||||
VALUES ($1)
|
||||
RETURNING id, title, created_at
|
||||
`
|
||||
|
||||
func (q *Queries) CreateChat(ctx context.Context, title string) (Chat, error) {
|
||||
row := q.db.QueryRow(ctx, createChat, title)
|
||||
var i Chat
|
||||
err := row.Scan(&i.ID, &i.Title, &i.CreatedAt)
|
||||
return i, err
|
||||
}
|
||||
|
||||
const getChatByID = `-- name: GetChatByID :one
|
||||
|
||||
SELECT c.id, c.title, c.created_at, array_agg(cm.user_id)::uuid[] AS member_ids
|
||||
FROM chats c
|
||||
JOIN chat_members cm ON cm.chat_id = c.id
|
||||
WHERE c.id = $1
|
||||
GROUP BY c.id
|
||||
`
|
||||
|
||||
type GetChatByIDRow struct {
|
||||
ID pgtype.UUID
|
||||
Title string
|
||||
CreatedAt pgtype.Timestamptz
|
||||
MemberIds []pgtype.UUID
|
||||
}
|
||||
|
||||
func (q *Queries) GetChatByID(ctx context.Context, id pgtype.UUID) (GetChatByIDRow, error) {
|
||||
row := q.db.QueryRow(ctx, getChatByID, id)
|
||||
var i GetChatByIDRow
|
||||
err := row.Scan(
|
||||
&i.ID,
|
||||
&i.Title,
|
||||
&i.CreatedAt,
|
||||
&i.MemberIds,
|
||||
)
|
||||
return i, err
|
||||
}
|
||||
|
||||
const getChatsByUserID = `-- name: GetChatsByUserID :many
|
||||
SELECT c.id, c.title, c.created_at, array_agg(cm.user_id)::uuid[] AS member_ids
|
||||
FROM chats c
|
||||
JOIN chat_members cm ON cm.chat_id = c.id
|
||||
WHERE EXISTS (SELECT 1
|
||||
FROM chat_members membership
|
||||
WHERE membership.chat_id = c.id
|
||||
AND membership.user_id = $1)
|
||||
AND (
|
||||
$2::timestamptz IS NULL
|
||||
OR (c.created_at, c.id) >
|
||||
(
|
||||
$2::timestamptz,
|
||||
$3::uuid
|
||||
)
|
||||
)
|
||||
GROUP BY c.id
|
||||
ORDER BY c.created_at ASC, c.id ASC
|
||||
LIMIT $4
|
||||
`
|
||||
|
||||
type GetChatsByUserIDParams struct {
|
||||
UserID pgtype.UUID
|
||||
AfterCreatedAt pgtype.Timestamptz
|
||||
AfterID pgtype.UUID
|
||||
LimitCount int32
|
||||
}
|
||||
|
||||
type GetChatsByUserIDRow struct {
|
||||
ID pgtype.UUID
|
||||
Title string
|
||||
CreatedAt pgtype.Timestamptz
|
||||
MemberIds []pgtype.UUID
|
||||
}
|
||||
|
||||
func (q *Queries) GetChatsByUserID(ctx context.Context, arg GetChatsByUserIDParams) ([]GetChatsByUserIDRow, error) {
|
||||
rows, err := q.db.Query(ctx, getChatsByUserID,
|
||||
arg.UserID,
|
||||
arg.AfterCreatedAt,
|
||||
arg.AfterID,
|
||||
arg.LimitCount,
|
||||
)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
var items []GetChatsByUserIDRow
|
||||
for rows.Next() {
|
||||
var i GetChatsByUserIDRow
|
||||
if err := rows.Scan(
|
||||
&i.ID,
|
||||
&i.Title,
|
||||
&i.CreatedAt,
|
||||
&i.MemberIds,
|
||||
); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
items = append(items, i)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return items, nil
|
||||
}
|
||||
@@ -0,0 +1,32 @@
|
||||
// Code generated by sqlc. DO NOT EDIT.
|
||||
// versions:
|
||||
// sqlc v1.31.1
|
||||
|
||||
package sqlc
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
"github.com/jackc/pgx/v5/pgconn"
|
||||
)
|
||||
|
||||
type DBTX interface {
|
||||
Exec(context.Context, string, ...interface{}) (pgconn.CommandTag, error)
|
||||
Query(context.Context, string, ...interface{}) (pgx.Rows, error)
|
||||
QueryRow(context.Context, string, ...interface{}) pgx.Row
|
||||
}
|
||||
|
||||
func New(db DBTX) *Queries {
|
||||
return &Queries{db: db}
|
||||
}
|
||||
|
||||
type Queries struct {
|
||||
db DBTX
|
||||
}
|
||||
|
||||
func (q *Queries) WithTx(tx pgx.Tx) *Queries {
|
||||
return &Queries{
|
||||
db: tx,
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,99 @@
|
||||
// Code generated by sqlc. DO NOT EDIT.
|
||||
// versions:
|
||||
// sqlc v1.31.1
|
||||
// source: message_idempotence.sql
|
||||
|
||||
package sqlc
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/jackc/pgx/v5/pgtype"
|
||||
)
|
||||
|
||||
const deleteExpiredMessageIdempotency = `-- name: DeleteExpiredMessageIdempotency :exec
|
||||
DELETE FROM message_idempotency
|
||||
WHERE user_id = $1 AND chat_id = $2 AND idempotency_key = $3 AND expires_at <= now()
|
||||
`
|
||||
|
||||
type DeleteExpiredMessageIdempotencyParams struct {
|
||||
UserID pgtype.UUID
|
||||
ChatID pgtype.UUID
|
||||
IdempotencyKey string
|
||||
}
|
||||
|
||||
func (q *Queries) DeleteExpiredMessageIdempotency(ctx context.Context, arg DeleteExpiredMessageIdempotencyParams) error {
|
||||
_, err := q.db.Exec(ctx, deleteExpiredMessageIdempotency, arg.UserID, arg.ChatID, arg.IdempotencyKey)
|
||||
return err
|
||||
}
|
||||
|
||||
const getIdempotentMessage = `-- name: GetIdempotentMessage :one
|
||||
SELECT m.id, m.chat_id, m.sender_id, m.text, m.created_at
|
||||
FROM message_idempotency AS i
|
||||
JOIN messages AS m ON m.id = i.message_id
|
||||
WHERE i.user_id = $1
|
||||
AND i.chat_id = $2
|
||||
AND i.idempotency_key = $3
|
||||
`
|
||||
|
||||
type GetIdempotentMessageParams struct {
|
||||
UserID pgtype.UUID
|
||||
ChatID pgtype.UUID
|
||||
IdempotencyKey string
|
||||
}
|
||||
|
||||
func (q *Queries) GetIdempotentMessage(ctx context.Context, arg GetIdempotentMessageParams) (Message, error) {
|
||||
row := q.db.QueryRow(ctx, getIdempotentMessage, arg.UserID, arg.ChatID, arg.IdempotencyKey)
|
||||
var i Message
|
||||
err := row.Scan(
|
||||
&i.ID,
|
||||
&i.ChatID,
|
||||
&i.SenderID,
|
||||
&i.Text,
|
||||
&i.CreatedAt,
|
||||
)
|
||||
return i, err
|
||||
}
|
||||
|
||||
const insertMessageIdempotency = `-- name: InsertMessageIdempotency :execrows
|
||||
INSERT INTO message_idempotency(user_id, chat_id, idempotency_key, message_id)
|
||||
VALUES ($1, $2, $3, $4)
|
||||
ON CONFLICT (user_id, chat_id, idempotency_key) DO NOTHING
|
||||
`
|
||||
|
||||
type InsertMessageIdempotencyParams struct {
|
||||
UserID pgtype.UUID
|
||||
ChatID pgtype.UUID
|
||||
IdempotencyKey string
|
||||
MessageID pgtype.UUID
|
||||
}
|
||||
|
||||
func (q *Queries) InsertMessageIdempotency(ctx context.Context, arg InsertMessageIdempotencyParams) (int64, error) {
|
||||
result, err := q.db.Exec(ctx, insertMessageIdempotency,
|
||||
arg.UserID,
|
||||
arg.ChatID,
|
||||
arg.IdempotencyKey,
|
||||
arg.MessageID,
|
||||
)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return result.RowsAffected(), nil
|
||||
}
|
||||
|
||||
const lockMessageRequest = `-- name: LockMessageRequest :exec
|
||||
SELECT pg_advisory_xact_lock(hashtextextended(
|
||||
'message/' || ($1::uuid)::text || '/' ||
|
||||
($2::uuid)::text || '/' || $3::text, 0))
|
||||
`
|
||||
|
||||
type LockMessageRequestParams struct {
|
||||
UserID pgtype.UUID
|
||||
ChatID pgtype.UUID
|
||||
IdempotencyKey string
|
||||
}
|
||||
|
||||
func (q *Queries) LockMessageRequest(ctx context.Context, arg LockMessageRequestParams) error {
|
||||
_, err := q.db.Exec(ctx, lockMessageRequest, arg.UserID, arg.ChatID, arg.IdempotencyKey)
|
||||
return err
|
||||
}
|
||||
@@ -0,0 +1,91 @@
|
||||
// Code generated by sqlc. DO NOT EDIT.
|
||||
// versions:
|
||||
// sqlc v1.31.1
|
||||
// source: messages.sql
|
||||
|
||||
package sqlc
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/jackc/pgx/v5/pgtype"
|
||||
)
|
||||
|
||||
const createMessage = `-- name: CreateMessage :one
|
||||
INSERT INTO messages (chat_id, sender_id, text)
|
||||
VALUES ($1, $2, $3)
|
||||
RETURNING id, chat_id, sender_id, text, created_at
|
||||
`
|
||||
|
||||
type CreateMessageParams struct {
|
||||
ChatID pgtype.UUID
|
||||
SenderID pgtype.UUID
|
||||
Text string
|
||||
}
|
||||
|
||||
func (q *Queries) CreateMessage(ctx context.Context, arg CreateMessageParams) (Message, error) {
|
||||
row := q.db.QueryRow(ctx, createMessage, arg.ChatID, arg.SenderID, arg.Text)
|
||||
var i Message
|
||||
err := row.Scan(
|
||||
&i.ID,
|
||||
&i.ChatID,
|
||||
&i.SenderID,
|
||||
&i.Text,
|
||||
&i.CreatedAt,
|
||||
)
|
||||
return i, err
|
||||
}
|
||||
|
||||
const getMessagesByChatID = `-- name: GetMessagesByChatID :many
|
||||
|
||||
SELECT id, chat_id, sender_id, text, created_at
|
||||
FROM messages
|
||||
WHERE chat_id = $1
|
||||
AND (
|
||||
$2::timestamptz IS NULL
|
||||
OR (created_at, id) >
|
||||
($2::timestamptz,
|
||||
$3::uuid
|
||||
)
|
||||
)
|
||||
ORDER BY created_at ASC, id ASC
|
||||
LIMIT $4
|
||||
`
|
||||
|
||||
type GetMessagesByChatIDParams struct {
|
||||
ChatID pgtype.UUID
|
||||
AfterCreatedAt pgtype.Timestamptz
|
||||
AfterID pgtype.UUID
|
||||
LimitCount int32
|
||||
}
|
||||
|
||||
func (q *Queries) GetMessagesByChatID(ctx context.Context, arg GetMessagesByChatIDParams) ([]Message, error) {
|
||||
rows, err := q.db.Query(ctx, getMessagesByChatID,
|
||||
arg.ChatID,
|
||||
arg.AfterCreatedAt,
|
||||
arg.AfterID,
|
||||
arg.LimitCount,
|
||||
)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
var items []Message
|
||||
for rows.Next() {
|
||||
var i Message
|
||||
if err := rows.Scan(
|
||||
&i.ID,
|
||||
&i.ChatID,
|
||||
&i.SenderID,
|
||||
&i.Text,
|
||||
&i.CreatedAt,
|
||||
); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
items = append(items, i)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return items, nil
|
||||
}
|
||||
@@ -0,0 +1,107 @@
|
||||
// Code generated by sqlc. DO NOT EDIT.
|
||||
// versions:
|
||||
// sqlc v1.31.1
|
||||
// source: messages_attachments.sql
|
||||
|
||||
package sqlc
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/jackc/pgx/v5/pgtype"
|
||||
)
|
||||
|
||||
const createMessagesAttachments = `-- name: CreateMessagesAttachments :execrows
|
||||
INSERT INTO messages_attachments (message_id, attachment_id, chat_id, position)
|
||||
SELECT $1, $2, $3, $4
|
||||
FROM attachments
|
||||
WHERE id = $2 AND chat_id = $3
|
||||
`
|
||||
|
||||
type CreateMessagesAttachmentsParams struct {
|
||||
MessageID pgtype.UUID
|
||||
AttachmentID pgtype.UUID
|
||||
ChatID pgtype.UUID
|
||||
Position int16
|
||||
}
|
||||
|
||||
func (q *Queries) CreateMessagesAttachments(ctx context.Context, arg CreateMessagesAttachmentsParams) (int64, error) {
|
||||
result, err := q.db.Exec(ctx, createMessagesAttachments,
|
||||
arg.MessageID,
|
||||
arg.AttachmentID,
|
||||
arg.ChatID,
|
||||
arg.Position,
|
||||
)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return result.RowsAffected(), nil
|
||||
}
|
||||
|
||||
const getMessagesAttachments = `-- name: GetMessagesAttachments :many
|
||||
SELECT attachment_id, position
|
||||
FROM messages_attachments
|
||||
WHERE message_id = $1
|
||||
ORDER BY position ASC
|
||||
`
|
||||
|
||||
type GetMessagesAttachmentsRow struct {
|
||||
AttachmentID pgtype.UUID
|
||||
Position int16
|
||||
}
|
||||
|
||||
func (q *Queries) GetMessagesAttachments(ctx context.Context, messageID pgtype.UUID) ([]GetMessagesAttachmentsRow, error) {
|
||||
rows, err := q.db.Query(ctx, getMessagesAttachments, messageID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
var items []GetMessagesAttachmentsRow
|
||||
for rows.Next() {
|
||||
var i GetMessagesAttachmentsRow
|
||||
if err := rows.Scan(&i.AttachmentID, &i.Position); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
items = append(items, i)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return items, nil
|
||||
}
|
||||
|
||||
const lockAttachmentsForMessage = `-- name: LockAttachmentsForMessage :many
|
||||
SELECT id, status FROM attachments
|
||||
WHERE chat_id = $1 AND id = ANY($2::uuid[])
|
||||
ORDER BY id FOR SHARE
|
||||
`
|
||||
|
||||
type LockAttachmentsForMessageParams struct {
|
||||
ChatID pgtype.UUID
|
||||
AttachmentIds []pgtype.UUID
|
||||
}
|
||||
|
||||
type LockAttachmentsForMessageRow struct {
|
||||
ID pgtype.UUID
|
||||
Status string
|
||||
}
|
||||
|
||||
func (q *Queries) LockAttachmentsForMessage(ctx context.Context, arg LockAttachmentsForMessageParams) ([]LockAttachmentsForMessageRow, error) {
|
||||
rows, err := q.db.Query(ctx, lockAttachmentsForMessage, arg.ChatID, arg.AttachmentIds)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
var items []LockAttachmentsForMessageRow
|
||||
for rows.Next() {
|
||||
var i LockAttachmentsForMessageRow
|
||||
if err := rows.Scan(&i.ID, &i.Status); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
items = append(items, i)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return items, nil
|
||||
}
|
||||
@@ -0,0 +1,124 @@
|
||||
// Code generated by sqlc. DO NOT EDIT.
|
||||
// versions:
|
||||
// sqlc v1.31.1
|
||||
|
||||
package sqlc
|
||||
|
||||
import (
|
||||
"github.com/jackc/pgx/v5/pgtype"
|
||||
)
|
||||
|
||||
type Attachment struct {
|
||||
ID pgtype.UUID
|
||||
ChatID pgtype.UUID
|
||||
OwnerID pgtype.UUID
|
||||
Filename string
|
||||
ObjectKey pgtype.Text
|
||||
ContentType string
|
||||
SizeBytes int64
|
||||
Sha256 string
|
||||
CreatedAt pgtype.Timestamptz
|
||||
UpdatedAt pgtype.Timestamptz
|
||||
Status string
|
||||
ErrorCode pgtype.Text
|
||||
StagingKey pgtype.Text
|
||||
PipelineVersion int32
|
||||
}
|
||||
|
||||
type AttachmentAttempt struct {
|
||||
JobID pgtype.UUID
|
||||
LeaseGeneration int32
|
||||
Attempt int32
|
||||
StartedAt pgtype.Timestamptz
|
||||
FinishedAt pgtype.Timestamptz
|
||||
Outcome pgtype.Text
|
||||
ErrorCode pgtype.Text
|
||||
}
|
||||
|
||||
type AttachmentJob struct {
|
||||
ID pgtype.UUID
|
||||
AttachmentID pgtype.UUID
|
||||
PipelineVersion int32
|
||||
AttemptsStarted int32
|
||||
MaxAttempts int32
|
||||
NextAttemptAt pgtype.Timestamptz
|
||||
LeaseGeneration int32
|
||||
LeaseUntil pgtype.Timestamptz
|
||||
CreatedAt pgtype.Timestamptz
|
||||
UpdatedAt pgtype.Timestamptz
|
||||
PublishedOriginalKey pgtype.Text
|
||||
}
|
||||
|
||||
type AttachmentReplay struct {
|
||||
RequestID pgtype.UUID
|
||||
JobID pgtype.UUID
|
||||
OperatorID string
|
||||
Reason string
|
||||
ExtraAttempts int32
|
||||
CreatedAt pgtype.Timestamptz
|
||||
}
|
||||
|
||||
type AttachmentVariant struct {
|
||||
AttachmentID pgtype.UUID
|
||||
PipelineVersion int32
|
||||
Name string
|
||||
ObjectKey string
|
||||
ContentType string
|
||||
SizeBytes int64
|
||||
Width int32
|
||||
Height int32
|
||||
Sha256 string
|
||||
}
|
||||
|
||||
type Chat struct {
|
||||
ID pgtype.UUID
|
||||
Title string
|
||||
CreatedAt pgtype.Timestamptz
|
||||
}
|
||||
|
||||
type ChatMember struct {
|
||||
ChatID pgtype.UUID
|
||||
UserID pgtype.UUID
|
||||
}
|
||||
|
||||
type Message struct {
|
||||
ID pgtype.UUID
|
||||
ChatID pgtype.UUID
|
||||
SenderID pgtype.UUID
|
||||
Text string
|
||||
CreatedAt pgtype.Timestamptz
|
||||
}
|
||||
|
||||
type MessageIdempotency struct {
|
||||
UserID pgtype.UUID
|
||||
ChatID pgtype.UUID
|
||||
IdempotencyKey string
|
||||
MessageID pgtype.UUID
|
||||
CreatedAt pgtype.Timestamptz
|
||||
ExpiresAt pgtype.Timestamptz
|
||||
}
|
||||
|
||||
type MessagesAttachment struct {
|
||||
MessageID pgtype.UUID
|
||||
AttachmentID pgtype.UUID
|
||||
ChatID pgtype.UUID
|
||||
Position int16
|
||||
}
|
||||
|
||||
type Outbox struct {
|
||||
ID pgtype.UUID
|
||||
JobID pgtype.UUID
|
||||
Payload []byte
|
||||
AvailableAt pgtype.Timestamptz
|
||||
SentAt pgtype.Timestamptz
|
||||
CreatedAt pgtype.Timestamptz
|
||||
DedupKey string
|
||||
}
|
||||
|
||||
type User struct {
|
||||
ID pgtype.UUID
|
||||
Username string
|
||||
DisplayName string
|
||||
PasswordHash string
|
||||
CreatedAt pgtype.Timestamptz
|
||||
}
|
||||
@@ -0,0 +1,128 @@
|
||||
// Code generated by sqlc. DO NOT EDIT.
|
||||
// versions:
|
||||
// sqlc v1.31.1
|
||||
// source: outbox.sql
|
||||
|
||||
package sqlc
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/jackc/pgx/v5/pgtype"
|
||||
)
|
||||
|
||||
const createInitialOutboxEvent = `-- name: CreateInitialOutboxEvent :exec
|
||||
INSERT INTO outbox (job_id, payload, available_at, dedup_key)
|
||||
SELECT id, attachment_job_payload(id, attachment_id, pipeline_version, 1, created_at)
|
||||
|| jsonb_strip_nulls(jsonb_build_object(
|
||||
'traceparent', NULLIF($1::text, ''),
|
||||
'tracestate', NULLIF($2::text, '')
|
||||
)),
|
||||
next_attempt_at, 'initial/' || id
|
||||
FROM attachment_jobs WHERE id = $3::uuid
|
||||
`
|
||||
|
||||
type CreateInitialOutboxEventParams struct {
|
||||
Traceparent string
|
||||
Tracestate string
|
||||
JobID pgtype.UUID
|
||||
}
|
||||
|
||||
func (q *Queries) CreateInitialOutboxEvent(ctx context.Context, arg CreateInitialOutboxEventParams) error {
|
||||
_, err := q.db.Exec(ctx, createInitialOutboxEvent, arg.Traceparent, arg.Tracestate, arg.JobID)
|
||||
return err
|
||||
}
|
||||
|
||||
const enqueueExpiredLeases = `-- name: EnqueueExpiredLeases :many
|
||||
WITH expired AS (
|
||||
SELECT j.id, j.attachment_id, j.pipeline_version, j.attempts_started, j.max_attempts, j.next_attempt_at, j.lease_generation, j.lease_until, j.created_at, j.updated_at, j.published_original_key, 'recovery/' || j.id || '/' || j.lease_generation AS recovery_key
|
||||
FROM attachment_jobs AS j
|
||||
JOIN attachments AS a ON a.id = j.attachment_id AND a.pipeline_version = j.pipeline_version
|
||||
WHERE a.status = 'processing' AND j.lease_until <= clock_timestamp()
|
||||
AND NOT EXISTS (
|
||||
SELECT 1 FROM outbox AS o WHERE o.dedup_key = 'recovery/' || j.id || '/' || j.lease_generation
|
||||
)
|
||||
ORDER BY j.lease_until, j.id
|
||||
LIMIT $1
|
||||
FOR UPDATE OF j SKIP LOCKED
|
||||
)
|
||||
INSERT INTO outbox (job_id, payload, dedup_key)
|
||||
SELECT e.id, attachment_job_payload(e.id, e.attachment_id, e.pipeline_version,
|
||||
LEAST(e.attempts_started + 1, e.max_attempts), clock_timestamp())
|
||||
|| COALESCE((
|
||||
SELECT jsonb_strip_nulls(jsonb_build_object(
|
||||
'traceparent', o.payload->>'traceparent', 'tracestate', o.payload->>'tracestate'
|
||||
))
|
||||
FROM outbox AS o WHERE o.job_id = e.id
|
||||
ORDER BY o.created_at DESC, o.id DESC LIMIT 1
|
||||
), '{}'::jsonb),
|
||||
e.recovery_key
|
||||
FROM expired AS e
|
||||
ON CONFLICT (dedup_key) DO NOTHING
|
||||
RETURNING id, job_id, payload, available_at, sent_at, created_at, dedup_key
|
||||
`
|
||||
|
||||
func (q *Queries) EnqueueExpiredLeases(ctx context.Context, batchSize int32) ([]Outbox, error) {
|
||||
rows, err := q.db.Query(ctx, enqueueExpiredLeases, batchSize)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
var items []Outbox
|
||||
for rows.Next() {
|
||||
var i Outbox
|
||||
if err := rows.Scan(
|
||||
&i.ID,
|
||||
&i.JobID,
|
||||
&i.Payload,
|
||||
&i.AvailableAt,
|
||||
&i.SentAt,
|
||||
&i.CreatedAt,
|
||||
&i.DedupKey,
|
||||
); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
items = append(items, i)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return items, nil
|
||||
}
|
||||
|
||||
const markOutboxEventSent = `-- name: MarkOutboxEventSent :execrows
|
||||
UPDATE outbox SET sent_at = clock_timestamp()
|
||||
WHERE id = $1 AND sent_at IS NULL
|
||||
`
|
||||
|
||||
func (q *Queries) MarkOutboxEventSent(ctx context.Context, id pgtype.UUID) (int64, error) {
|
||||
result, err := q.db.Exec(ctx, markOutboxEventSent, id)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return result.RowsAffected(), nil
|
||||
}
|
||||
|
||||
const nextPendingOutboxEvent = `-- name: NextPendingOutboxEvent :one
|
||||
SELECT id, job_id, payload, available_at, sent_at, created_at, dedup_key
|
||||
FROM outbox
|
||||
WHERE sent_at IS NULL AND available_at <= statement_timestamp()
|
||||
ORDER BY available_at, id
|
||||
LIMIT 1
|
||||
FOR UPDATE SKIP LOCKED
|
||||
`
|
||||
|
||||
func (q *Queries) NextPendingOutboxEvent(ctx context.Context) (Outbox, error) {
|
||||
row := q.db.QueryRow(ctx, nextPendingOutboxEvent)
|
||||
var i Outbox
|
||||
err := row.Scan(
|
||||
&i.ID,
|
||||
&i.JobID,
|
||||
&i.Payload,
|
||||
&i.AvailableAt,
|
||||
&i.SentAt,
|
||||
&i.CreatedAt,
|
||||
&i.DedupKey,
|
||||
)
|
||||
return i, err
|
||||
}
|
||||
@@ -0,0 +1,77 @@
|
||||
// Code generated by sqlc. DO NOT EDIT.
|
||||
// versions:
|
||||
// sqlc v1.31.1
|
||||
// source: users.sql
|
||||
|
||||
package sqlc
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/jackc/pgx/v5/pgtype"
|
||||
)
|
||||
|
||||
const createUser = `-- name: CreateUser :one
|
||||
|
||||
INSERT INTO users (username, display_name, password_hash)
|
||||
VALUES ($1, $2, $3)
|
||||
RETURNING id, username, display_name, password_hash, created_at
|
||||
`
|
||||
|
||||
type CreateUserParams struct {
|
||||
Username string
|
||||
DisplayName string
|
||||
PasswordHash string
|
||||
}
|
||||
|
||||
func (q *Queries) CreateUser(ctx context.Context, arg CreateUserParams) (User, error) {
|
||||
row := q.db.QueryRow(ctx, createUser, arg.Username, arg.DisplayName, arg.PasswordHash)
|
||||
var i User
|
||||
err := row.Scan(
|
||||
&i.ID,
|
||||
&i.Username,
|
||||
&i.DisplayName,
|
||||
&i.PasswordHash,
|
||||
&i.CreatedAt,
|
||||
)
|
||||
return i, err
|
||||
}
|
||||
|
||||
const getUserByID = `-- name: GetUserByID :one
|
||||
|
||||
SELECT id, username, display_name, password_hash, created_at
|
||||
FROM users
|
||||
WHERE id = $1
|
||||
`
|
||||
|
||||
func (q *Queries) GetUserByID(ctx context.Context, id pgtype.UUID) (User, error) {
|
||||
row := q.db.QueryRow(ctx, getUserByID, id)
|
||||
var i User
|
||||
err := row.Scan(
|
||||
&i.ID,
|
||||
&i.Username,
|
||||
&i.DisplayName,
|
||||
&i.PasswordHash,
|
||||
&i.CreatedAt,
|
||||
)
|
||||
return i, err
|
||||
}
|
||||
|
||||
const getUserByUsername = `-- name: GetUserByUsername :one
|
||||
SELECT id, username, display_name, password_hash, created_at
|
||||
FROM users
|
||||
WHERE username = $1
|
||||
`
|
||||
|
||||
func (q *Queries) GetUserByUsername(ctx context.Context, username string) (User, error) {
|
||||
row := q.db.QueryRow(ctx, getUserByUsername, username)
|
||||
var i User
|
||||
err := row.Scan(
|
||||
&i.ID,
|
||||
&i.Username,
|
||||
&i.DisplayName,
|
||||
&i.PasswordHash,
|
||||
&i.CreatedAt,
|
||||
)
|
||||
return i, err
|
||||
}
|
||||
@@ -0,0 +1,111 @@
|
||||
// Code generated by sqlc. DO NOT EDIT.
|
||||
// versions:
|
||||
// sqlc v1.31.1
|
||||
// source: versioned_pipeline.sql
|
||||
|
||||
package sqlc
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/jackc/pgx/v5/pgtype"
|
||||
)
|
||||
|
||||
const getPipelineJobID = `-- name: GetPipelineJobID :one
|
||||
SELECT id FROM attachment_jobs
|
||||
WHERE attachment_id = $1::uuid AND pipeline_version = $2::int
|
||||
`
|
||||
|
||||
type GetPipelineJobIDParams struct {
|
||||
AttachmentID pgtype.UUID
|
||||
PipelineVersion int32
|
||||
}
|
||||
|
||||
func (q *Queries) GetPipelineJobID(ctx context.Context, arg GetPipelineJobIDParams) (pgtype.UUID, error) {
|
||||
row := q.db.QueryRow(ctx, getPipelineJobID, arg.AttachmentID, arg.PipelineVersion)
|
||||
var id pgtype.UUID
|
||||
err := row.Scan(&id)
|
||||
return id, err
|
||||
}
|
||||
|
||||
const getPublishedOriginalKey = `-- name: GetPublishedOriginalKey :one
|
||||
SELECT published_original_key FROM attachment_jobs
|
||||
WHERE attachment_id = $1::uuid AND pipeline_version = $2::int
|
||||
AND published_original_key IS NOT NULL
|
||||
`
|
||||
|
||||
type GetPublishedOriginalKeyParams struct {
|
||||
AttachmentID pgtype.UUID
|
||||
PipelineVersion int32
|
||||
}
|
||||
|
||||
func (q *Queries) GetPublishedOriginalKey(ctx context.Context, arg GetPublishedOriginalKeyParams) (pgtype.Text, error) {
|
||||
row := q.db.QueryRow(ctx, getPublishedOriginalKey, arg.AttachmentID, arg.PipelineVersion)
|
||||
var published_original_key pgtype.Text
|
||||
err := row.Scan(&published_original_key)
|
||||
return published_original_key, err
|
||||
}
|
||||
|
||||
const isPublishedOriginalKey = `-- name: IsPublishedOriginalKey :one
|
||||
SELECT EXISTS (SELECT 1 FROM attachment_jobs WHERE published_original_key = $1::text)
|
||||
`
|
||||
|
||||
func (q *Queries) IsPublishedOriginalKey(ctx context.Context, objectKey string) (bool, error) {
|
||||
row := q.db.QueryRow(ctx, isPublishedOriginalKey, objectKey)
|
||||
var exists bool
|
||||
err := row.Scan(&exists)
|
||||
return exists, err
|
||||
}
|
||||
|
||||
const queueAttachmentUpgrade = `-- name: QueueAttachmentUpgrade :one
|
||||
UPDATE attachments
|
||||
SET pipeline_version = 2, status = 'queued', staging_key = $1::text,
|
||||
object_key = NULL, error_code = NULL, updated_at = clock_timestamp()
|
||||
WHERE id = $2::uuid AND pipeline_version = 1
|
||||
RETURNING id, chat_id, owner_id, filename, object_key, content_type, size_bytes, sha256, created_at, updated_at, status, error_code, staging_key, pipeline_version
|
||||
`
|
||||
|
||||
type QueueAttachmentUpgradeParams struct {
|
||||
SourceKey string
|
||||
ID pgtype.UUID
|
||||
}
|
||||
|
||||
func (q *Queries) QueueAttachmentUpgrade(ctx context.Context, arg QueueAttachmentUpgradeParams) (Attachment, error) {
|
||||
row := q.db.QueryRow(ctx, queueAttachmentUpgrade, arg.SourceKey, arg.ID)
|
||||
var i Attachment
|
||||
err := row.Scan(
|
||||
&i.ID,
|
||||
&i.ChatID,
|
||||
&i.OwnerID,
|
||||
&i.Filename,
|
||||
&i.ObjectKey,
|
||||
&i.ContentType,
|
||||
&i.SizeBytes,
|
||||
&i.Sha256,
|
||||
&i.CreatedAt,
|
||||
&i.UpdatedAt,
|
||||
&i.Status,
|
||||
&i.ErrorCode,
|
||||
&i.StagingKey,
|
||||
&i.PipelineVersion,
|
||||
)
|
||||
return i, err
|
||||
}
|
||||
|
||||
const supersedeAttachmentJob = `-- name: SupersedeAttachmentJob :exec
|
||||
WITH superseded AS (
|
||||
UPDATE attachment_jobs
|
||||
SET lease_until = NULL, next_attempt_at = NULL, updated_at = clock_timestamp()
|
||||
WHERE id = $1::uuid
|
||||
RETURNING id, updated_at
|
||||
)
|
||||
UPDATE attachment_attempts AS attempt
|
||||
SET outcome = 'superseded', error_code = 'superseded', finished_at = superseded.updated_at
|
||||
FROM superseded
|
||||
WHERE attempt.job_id = superseded.id AND attempt.finished_at IS NULL
|
||||
`
|
||||
|
||||
func (q *Queries) SupersedeAttachmentJob(ctx context.Context, jobID pgtype.UUID) error {
|
||||
_, err := q.db.Exec(ctx, supersedeAttachmentJob, jobID)
|
||||
return err
|
||||
}
|
||||
@@ -0,0 +1,43 @@
|
||||
// Package fault provides one-shot pause points for an isolated lab Compose project.
|
||||
package fault
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"time"
|
||||
)
|
||||
|
||||
func Pause(ctx context.Context, point, id string) error {
|
||||
dir := os.Getenv("LAB5_FAULT_DIR")
|
||||
if dir == "" {
|
||||
return nil
|
||||
}
|
||||
base := filepath.Join(dir, point)
|
||||
if err := os.Rename(base+".arm", base+".used"); err != nil {
|
||||
if errors.Is(err, os.ErrNotExist) {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
if err := os.WriteFile(base+".hit", []byte(id+"\n"), 0644); err != nil {
|
||||
return err
|
||||
}
|
||||
wait, cancel := context.WithTimeout(ctx, 90*time.Second)
|
||||
defer cancel()
|
||||
poll := time.NewTicker(50 * time.Millisecond)
|
||||
defer poll.Stop()
|
||||
for {
|
||||
if _, err := os.Stat(base + ".release"); err == nil {
|
||||
return nil
|
||||
} else if !errors.Is(err, os.ErrNotExist) {
|
||||
return err
|
||||
}
|
||||
select {
|
||||
case <-wait.Done():
|
||||
return wait.Err()
|
||||
case <-poll.C:
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,164 @@
|
||||
package imageproc
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"image"
|
||||
_ "image/jpeg"
|
||||
_ "image/png"
|
||||
"io"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"highload/internal/model"
|
||||
)
|
||||
|
||||
const (
|
||||
MaxBytes = 10 * 1024 * 1024
|
||||
MaxPixels = 40_000_000
|
||||
)
|
||||
|
||||
var (
|
||||
ErrInvalidImage = errors.New("invalid image")
|
||||
ErrResourceLimit = errors.New("image processing resource limit")
|
||||
)
|
||||
|
||||
type File struct {
|
||||
Path string
|
||||
Variant model.AttachmentVariant
|
||||
}
|
||||
|
||||
type variantSpec struct {
|
||||
name, extension, mime string
|
||||
width, height int
|
||||
args []string
|
||||
}
|
||||
|
||||
func Generate(ctx context.Context, dir, input, mime string, version int32) ([]File, error) {
|
||||
var jpegQuality, webpQuality string
|
||||
switch version {
|
||||
case 1:
|
||||
jpegQuality, webpQuality = "85", "80"
|
||||
case 2:
|
||||
jpegQuality, webpQuality = "90", "72"
|
||||
default:
|
||||
return nil, fmt.Errorf("unsupported image pipeline version %d", version)
|
||||
}
|
||||
f, err := os.Open(input)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
cfg, format, decodeErr := image.DecodeConfig(io.LimitReader(f, MaxBytes))
|
||||
_ = f.Close()
|
||||
if decodeErr != nil || (format != "png" && format != "jpeg") || mime != "image/"+format ||
|
||||
cfg.Width <= 0 || cfg.Height <= 0 || cfg.Width > MaxPixels/cfg.Height {
|
||||
return nil, ErrInvalidImage
|
||||
}
|
||||
// Paths are generated on the server; input format is independently verified.
|
||||
source := format + ":" + input
|
||||
orient, err := magick(ctx, "identify", "-ping", "-format", "%[orientation]", source)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
w, h := cfg.Width, cfg.Height
|
||||
switch strings.TrimSpace(orient) {
|
||||
case "LeftTop", "RightTop", "RightBottom", "LeftBottom":
|
||||
w, h = h, w
|
||||
}
|
||||
tw, th := fit(w, h, 256)
|
||||
ow, oh := fit(w, h, 1280)
|
||||
side := min(w, h)
|
||||
specs := []variantSpec{
|
||||
{name: "thumbnail", extension: "jpg", mime: "image/jpeg", width: tw, height: th,
|
||||
args: []string{"-resize", geometry(tw, th), "-quality", jpegQuality}},
|
||||
{name: "crop", extension: "jpg", mime: "image/jpeg", width: 128, height: 128,
|
||||
args: []string{"-gravity", "center", "-crop", fmt.Sprintf("%dx%d+0+0", side, side), "+repage", "-resize", "128x128!", "-quality", jpegQuality}},
|
||||
{name: "optimized", extension: "webp", mime: "image/webp", width: ow, height: oh,
|
||||
args: []string{"-resize", geometry(ow, oh), "-quality", webpQuality, "-define", "webp:method=4"}},
|
||||
}
|
||||
files := make([]File, 0, len(specs))
|
||||
for _, spec := range specs {
|
||||
out := filepath.Join(dir, spec.name+"."+spec.extension)
|
||||
args := []string{"-regard-warnings", source, "-auto-orient"}
|
||||
if spec.mime == "image/jpeg" {
|
||||
args = append(args, "-background", "white", "-alpha", "remove", "-alpha", "off")
|
||||
}
|
||||
args = append(args, spec.args...)
|
||||
args = append(args, "-strip", out)
|
||||
if _, err := magick(ctx, args...); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
meta, err := inspectFile(ctx, out, spec)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
files = append(files, File{Path: out, Variant: meta})
|
||||
}
|
||||
return files, nil
|
||||
}
|
||||
|
||||
func fit(w, h, bound int) (int, int) {
|
||||
if max(w, h) <= bound {
|
||||
return w, h
|
||||
}
|
||||
if w >= h {
|
||||
return bound, max(1, h*bound/w)
|
||||
}
|
||||
return max(1, w*bound/h), bound
|
||||
}
|
||||
|
||||
func geometry(w, h int) string {
|
||||
return fmt.Sprintf("%dx%d!", w, h)
|
||||
}
|
||||
|
||||
func inspectFile(ctx context.Context, path string, spec variantSpec) (model.AttachmentVariant, error) {
|
||||
var meta model.AttachmentVariant
|
||||
desc, err := magick(ctx, "identify", "-ping", "-format", "%m %w %h", path)
|
||||
if err != nil {
|
||||
return meta, err
|
||||
}
|
||||
format := strings.ToUpper(strings.TrimPrefix(spec.mime, "image/"))
|
||||
if strings.TrimSpace(desc) != format+" "+strconv.Itoa(spec.width)+" "+strconv.Itoa(spec.height) {
|
||||
return meta, fmt.Errorf("generated variant has unexpected format or size")
|
||||
}
|
||||
f, err := os.Open(path)
|
||||
if err != nil {
|
||||
return meta, err
|
||||
}
|
||||
defer f.Close()
|
||||
sum := sha256.New()
|
||||
size, err := io.Copy(sum, f)
|
||||
if err != nil {
|
||||
return meta, err
|
||||
}
|
||||
return model.AttachmentVariant{Name: spec.name, ContentType: spec.mime, Width: spec.width, Height: spec.height,
|
||||
SizeBytes: size, SHA256: hex.EncodeToString(sum.Sum(nil))}, nil
|
||||
}
|
||||
|
||||
func magick(ctx context.Context, args ...string) (string, error) {
|
||||
cmd := exec.CommandContext(ctx, "magick", args...)
|
||||
out, err := cmd.CombinedOutput()
|
||||
if ctx.Err() != nil {
|
||||
return "", ctx.Err()
|
||||
}
|
||||
if err == nil {
|
||||
return string(out), nil
|
||||
}
|
||||
message := strings.ToLower(string(out))
|
||||
if strings.Contains(message, "cache resources exhausted") || strings.Contains(message, "time limit") ||
|
||||
strings.Contains(message, "memory allocation") || strings.Contains(message, "list length exceeds") {
|
||||
return "", ErrResourceLimit
|
||||
}
|
||||
if strings.Contains(message, "corrupt") || strings.Contains(message, "unexpected end") ||
|
||||
strings.Contains(message, "insufficient image data") || strings.Contains(message, "decode") {
|
||||
return "", ErrInvalidImage
|
||||
}
|
||||
return "", fmt.Errorf("ImageMagick command failed: %w", err)
|
||||
}
|
||||
@@ -0,0 +1,31 @@
|
||||
package model
|
||||
|
||||
import "time"
|
||||
|
||||
type Attachment struct {
|
||||
ID string
|
||||
ChatID string
|
||||
OwnerID string
|
||||
Filename string
|
||||
ObjectKey string
|
||||
StagingKey string
|
||||
ErrorCode *string
|
||||
Variants []AttachmentVariant
|
||||
ContentType string
|
||||
SizeBytes int64
|
||||
SHA256 string
|
||||
Status string
|
||||
PipelineVersion int32
|
||||
CreatedAt time.Time
|
||||
UpdatedAt time.Time
|
||||
}
|
||||
|
||||
type AttachmentVariant struct {
|
||||
Name string
|
||||
ObjectKey string
|
||||
ContentType string
|
||||
SizeBytes int64
|
||||
Width int
|
||||
Height int
|
||||
SHA256 string
|
||||
}
|
||||
@@ -0,0 +1,27 @@
|
||||
package model
|
||||
|
||||
import "time"
|
||||
|
||||
type AttachmentJob struct {
|
||||
ID string
|
||||
AttachmentID string
|
||||
PipelineVersion int32
|
||||
AttemptsStarted int32
|
||||
MaxAttempts int32
|
||||
LeaseGeneration int32
|
||||
LeaseUntil time.Time
|
||||
AcceptedAt time.Time
|
||||
}
|
||||
|
||||
type AttachmentJobState struct {
|
||||
AttachmentID string
|
||||
PipelineVersion int32
|
||||
ID string
|
||||
Status string
|
||||
LeaseGeneration int32
|
||||
AttemptsStarted int32
|
||||
MaxAttempts int32
|
||||
LeaseUntil *time.Time
|
||||
LeaseExpired bool
|
||||
NextAttemptAt *time.Time
|
||||
}
|
||||
@@ -0,0 +1,10 @@
|
||||
package model
|
||||
|
||||
import "time"
|
||||
|
||||
type Chat struct {
|
||||
ID string
|
||||
Title string
|
||||
MemberIDs []string
|
||||
CreatedAt time.Time
|
||||
}
|
||||
@@ -0,0 +1,12 @@
|
||||
package model
|
||||
|
||||
import "time"
|
||||
|
||||
type Message struct {
|
||||
ID string
|
||||
ChatID string
|
||||
SenderID string
|
||||
Text string
|
||||
AttachmentIDs []string
|
||||
CreatedAt time.Time
|
||||
}
|
||||
@@ -0,0 +1,11 @@
|
||||
package model
|
||||
|
||||
import "time"
|
||||
|
||||
type Session struct {
|
||||
TokenHash []byte
|
||||
UserID string
|
||||
CreatedAt time.Time
|
||||
ExpiresAt time.Time
|
||||
RevokedAt *time.Time
|
||||
}
|
||||
@@ -0,0 +1,11 @@
|
||||
package model
|
||||
|
||||
import "time"
|
||||
|
||||
type User struct {
|
||||
ID string
|
||||
Username string
|
||||
DisplayName string
|
||||
PasswordHash string
|
||||
CreatedAt time.Time
|
||||
}
|
||||
@@ -0,0 +1,65 @@
|
||||
package observability
|
||||
|
||||
import (
|
||||
"context"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
"go.opentelemetry.io/otel/attribute"
|
||||
"go.opentelemetry.io/otel/metric"
|
||||
)
|
||||
|
||||
func WatchPool(pool *pgxpool.Pool, waiting *atomic.Int64) {
|
||||
connections := instrument(meter.Int64ObservableGauge("messenger_db_pool_connections"))
|
||||
acquiring := instrument(meter.Int64ObservableGauge("messenger_db_pool_waiting"))
|
||||
waitTime := instrument(meter.Float64ObservableCounter("messenger_db_pool_wait", metric.WithUnit("s")))
|
||||
_, _ = meter.RegisterCallback(func(_ context.Context, o metric.Observer) error {
|
||||
stats := pool.Stat()
|
||||
o.ObserveInt64(connections, int64(stats.AcquiredConns()), metric.WithAttributes(attribute.String("state", "used")))
|
||||
o.ObserveInt64(connections, int64(stats.MaxConns()), metric.WithAttributes(attribute.String("state", "capacity")))
|
||||
o.ObserveInt64(acquiring, waiting.Load())
|
||||
o.ObserveFloat64(waitTime, stats.EmptyAcquireWaitTime().Seconds())
|
||||
return nil
|
||||
}, connections, acquiring, waitTime)
|
||||
}
|
||||
|
||||
type QueueSample struct {
|
||||
Depth int64
|
||||
OldestSeconds float64
|
||||
At time.Time
|
||||
}
|
||||
|
||||
func WatchQueue(queue string) func(QueueSample) {
|
||||
depth := instrument(meter.Int64ObservableGauge("messenger_queue_depth"))
|
||||
age := instrument(meter.Float64ObservableGauge("messenger_queue_oldest_age", metric.WithUnit("s")))
|
||||
freshness := instrument(meter.Float64ObservableGauge("messenger_queue_sample_age", metric.WithUnit("s")))
|
||||
var current atomic.Pointer[QueueSample]
|
||||
attrs := metric.WithAttributes(attribute.String("queue", queue))
|
||||
_, _ = meter.RegisterCallback(func(_ context.Context, o metric.Observer) error {
|
||||
if sample := current.Load(); sample != nil {
|
||||
o.ObserveInt64(depth, sample.Depth, attrs)
|
||||
o.ObserveFloat64(age, sample.OldestSeconds, attrs)
|
||||
o.ObserveFloat64(freshness, time.Since(sample.At).Seconds(), attrs)
|
||||
}
|
||||
return nil
|
||||
}, depth, age, freshness)
|
||||
return func(sample QueueSample) { current.Store(&sample) }
|
||||
}
|
||||
|
||||
var workerBusy atomic.Int64
|
||||
|
||||
func WatchWorker(lanes int) {
|
||||
busy := instrument(meter.Int64ObservableGauge("messenger_worker_busy"))
|
||||
capacity := instrument(meter.Int64ObservableGauge("messenger_worker_capacity"))
|
||||
_, _ = meter.RegisterCallback(func(_ context.Context, o metric.Observer) error {
|
||||
o.ObserveInt64(busy, workerBusy.Load())
|
||||
o.ObserveInt64(capacity, int64(lanes))
|
||||
return nil
|
||||
}, busy, capacity)
|
||||
}
|
||||
|
||||
func WorkerAttempt() func() {
|
||||
workerBusy.Add(1)
|
||||
return func() { workerBusy.Add(-1) }
|
||||
}
|
||||
@@ -0,0 +1,19 @@
|
||||
package observability
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"go.opentelemetry.io/otel/propagation"
|
||||
)
|
||||
|
||||
func Carrier(ctx context.Context) propagation.MapCarrier {
|
||||
carrier := propagation.MapCarrier{}
|
||||
(propagation.TraceContext{}).Inject(ctx, carrier)
|
||||
return carrier
|
||||
}
|
||||
|
||||
func JobContext(ctx context.Context, traceparent, tracestate string) context.Context {
|
||||
return (propagation.TraceContext{}).Extract(ctx, propagation.MapCarrier{
|
||||
"traceparent": traceparent, "tracestate": tracestate,
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,109 @@
|
||||
package observability
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"log/slog"
|
||||
"os"
|
||||
"time"
|
||||
|
||||
"go.opentelemetry.io/otel"
|
||||
"go.opentelemetry.io/otel/attribute"
|
||||
"go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp"
|
||||
"go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp"
|
||||
"go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp"
|
||||
"go.opentelemetry.io/otel/log/global"
|
||||
"go.opentelemetry.io/otel/propagation"
|
||||
sdklog "go.opentelemetry.io/otel/sdk/log"
|
||||
sdkmetric "go.opentelemetry.io/otel/sdk/metric"
|
||||
"go.opentelemetry.io/otel/sdk/resource"
|
||||
sdktrace "go.opentelemetry.io/otel/sdk/trace"
|
||||
)
|
||||
|
||||
const scope = "highload/messenger"
|
||||
const exportTimeout = 1500 * time.Millisecond
|
||||
|
||||
var W3C = propagation.TraceContext{}
|
||||
|
||||
// Start initializes all three signals. Export failures are asynchronous and
|
||||
// telemetry configuration is independent of business readiness.
|
||||
func Start(ctx context.Context, fallbackName string) func() {
|
||||
if os.Getenv("OTEL_EXPORTER_OTLP_ENDPOINT") == "" {
|
||||
return func() {}
|
||||
}
|
||||
// Exporter errors can contain endpoints. Suppress the raw error and limit
|
||||
// stderr noise to one message per second for a broken telemetry sink.
|
||||
otel.SetErrorHandler(otel.ErrorHandlerFunc(func(error) {
|
||||
reportExportError()
|
||||
}))
|
||||
shutdown, err := start(ctx, fallbackName)
|
||||
if err != nil {
|
||||
slog.Warn("OpenTelemetry initialization failed")
|
||||
return func() {}
|
||||
}
|
||||
return func() {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 4*time.Second)
|
||||
defer cancel()
|
||||
if err := shutdown(ctx); err != nil {
|
||||
slog.Warn("OpenTelemetry shutdown could not flush every signal")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func start(ctx context.Context, fallbackName string) (func(context.Context) error, error) {
|
||||
serviceName := os.Getenv("OTEL_SERVICE_NAME")
|
||||
if serviceName == "" {
|
||||
serviceName = fallbackName
|
||||
}
|
||||
version := os.Getenv("OTEL_SERVICE_VERSION")
|
||||
if version == "" {
|
||||
version = "lab6"
|
||||
}
|
||||
instance, err := os.Hostname()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
res := resource.NewSchemaless(
|
||||
attribute.String("service.name", serviceName),
|
||||
attribute.String("service.version", version),
|
||||
attribute.String("service.instance.id", instance),
|
||||
attribute.String("deployment.environment.name", "lab"),
|
||||
)
|
||||
traces, err := otlptracehttp.New(ctx, otlptracehttp.WithTimeout(exportTimeout),
|
||||
otlptracehttp.WithRetry(otlptracehttp.RetryConfig{Enabled: false}))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
metrics, err := otlpmetrichttp.New(ctx, otlpmetrichttp.WithTimeout(exportTimeout),
|
||||
otlpmetrichttp.WithRetry(otlpmetrichttp.RetryConfig{Enabled: false}))
|
||||
if err != nil {
|
||||
_ = traces.Shutdown(ctx)
|
||||
return nil, err
|
||||
}
|
||||
logs, err := otlploghttp.New(ctx, otlploghttp.WithTimeout(exportTimeout),
|
||||
otlploghttp.WithRetry(otlploghttp.RetryConfig{Enabled: false}))
|
||||
if err != nil {
|
||||
_ = metrics.Shutdown(ctx)
|
||||
_ = traces.Shutdown(ctx)
|
||||
return nil, err
|
||||
}
|
||||
tp := sdktrace.NewTracerProvider(sdktrace.WithResource(res),
|
||||
sdktrace.WithSampler(sdktrace.ParentBased(sdktrace.AlwaysSample())),
|
||||
sdktrace.WithBatcher(traces, sdktrace.WithMaxQueueSize(512), sdktrace.WithMaxExportBatchSize(128),
|
||||
sdktrace.WithBatchTimeout(500*time.Millisecond), sdktrace.WithExportTimeout(exportTimeout)))
|
||||
mp := sdkmetric.NewMeterProvider(sdkmetric.WithResource(res),
|
||||
sdkmetric.WithReader(sdkmetric.NewPeriodicReader(metrics,
|
||||
sdkmetric.WithInterval(2*time.Second), sdkmetric.WithTimeout(exportTimeout))))
|
||||
lp := sdklog.NewLoggerProvider(sdklog.WithResource(res),
|
||||
sdklog.WithProcessor(sdklog.NewBatchProcessor(logs, sdklog.WithMaxQueueSize(512),
|
||||
sdklog.WithExportMaxBatchSize(128), sdklog.WithExportInterval(500*time.Millisecond),
|
||||
sdklog.WithExportTimeout(exportTimeout))))
|
||||
otel.SetTracerProvider(tp)
|
||||
otel.SetMeterProvider(mp)
|
||||
global.SetLoggerProvider(lp)
|
||||
otel.SetTextMapPropagator(W3C)
|
||||
observeProcess(mp.Meter(scope))
|
||||
return func(ctx context.Context) error {
|
||||
return errors.Join(tp.Shutdown(ctx), mp.Shutdown(ctx), lp.Shutdown(ctx))
|
||||
}, nil
|
||||
}
|
||||
@@ -0,0 +1,143 @@
|
||||
package observability
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"log/slog"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"go.opentelemetry.io/otel"
|
||||
"go.opentelemetry.io/otel/attribute"
|
||||
"go.opentelemetry.io/otel/codes"
|
||||
"go.opentelemetry.io/otel/log"
|
||||
"go.opentelemetry.io/otel/log/global"
|
||||
"go.opentelemetry.io/otel/metric"
|
||||
"go.opentelemetry.io/otel/trace"
|
||||
)
|
||||
|
||||
var Tracer = otel.Tracer(scope)
|
||||
var meter = otel.Meter(scope)
|
||||
|
||||
var (
|
||||
httpRequests = instrument(meter.Int64Counter("messenger_http_requests"))
|
||||
httpLatency = instrument(meter.Float64Histogram("messenger_http_request_duration", metric.WithUnit("s"),
|
||||
metric.WithExplicitBucketBoundaries(.001, .003, .005, .01, .025, .05, .1, .25, .5, 1, 2, 5, 10)))
|
||||
jobs = instrument(meter.Int64Counter("messenger_jobs"))
|
||||
jobLatency = instrument(meter.Float64Histogram("messenger_job_duration", metric.WithUnit("s"),
|
||||
metric.WithExplicitBucketBoundaries(.01, .025, .05, .1, .25, .5, 1, 2, 5, 10, 20, 40, 60, 120)))
|
||||
readyLatency = instrument(meter.Float64Histogram("messenger_attachment_time_to_ready", metric.WithUnit("s"),
|
||||
metric.WithExplicitBucketBoundaries(.05, .1, .25, .5, 1, 2, 5, 10, 20, 40, 60, 120, 300)))
|
||||
lastExportError atomic.Int64
|
||||
)
|
||||
|
||||
func instrument[T any](value T, err error) T {
|
||||
if err != nil {
|
||||
panic(err) // All instrument names and descriptors are code constants.
|
||||
}
|
||||
return value
|
||||
}
|
||||
|
||||
func reportExportError() {
|
||||
now := time.Now().Unix()
|
||||
last := lastExportError.Load()
|
||||
if last != now && lastExportError.CompareAndSwap(last, now) {
|
||||
slog.Warn("OpenTelemetry export failed")
|
||||
}
|
||||
}
|
||||
|
||||
func ObserveHTTP(ctx context.Context, method, route string, code int, duration time.Duration) {
|
||||
attrs := metric.WithAttributes(attribute.String("method", method), attribute.String("route", route),
|
||||
attribute.String("status_class", strconv.Itoa(code/100)+"xx"))
|
||||
httpRequests.Add(ctx, 1, attrs)
|
||||
httpLatency.Record(ctx, duration.Seconds(), attrs)
|
||||
}
|
||||
|
||||
func ObserveJob(ctx context.Context, outcome string, duration time.Duration) {
|
||||
attrs := metric.WithAttributes(attribute.String("job_type", "attachment.process"), attribute.String("outcome", outcome))
|
||||
jobs.Add(ctx, 1, attrs)
|
||||
jobLatency.Record(ctx, duration.Seconds(), attrs)
|
||||
}
|
||||
|
||||
func ObserveReady(ctx context.Context, acceptedAt time.Time) {
|
||||
readyLatency.Record(ctx, max(time.Since(acceptedAt).Seconds(), 0),
|
||||
metric.WithAttributes(attribute.String("job_type", "attachment.process")))
|
||||
}
|
||||
|
||||
// Log exports only deliberate operational fields, never arbitrary slog values.
|
||||
// The JSON body allows LogQL queries and Grafana's trace link to use the same IDs.
|
||||
func Log(ctx context.Context, event string, level log.Severity, fields ...attribute.KeyValue) {
|
||||
values := make(map[string]any, len(fields)+3)
|
||||
values["event"] = event
|
||||
for _, field := range fields {
|
||||
values[string(field.Key)] = field.Value.AsInterface()
|
||||
}
|
||||
if span := trace.SpanContextFromContext(ctx); span.IsValid() {
|
||||
values["trace_id"], values["span_id"] = span.TraceID().String(), span.SpanID().String()
|
||||
}
|
||||
if requestID := RequestID(ctx); requestID != "" {
|
||||
values["request_id"] = requestID
|
||||
}
|
||||
body, err := json.Marshal(values)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
var record log.Record
|
||||
record.SetTimestamp(time.Now())
|
||||
record.SetSeverity(level)
|
||||
record.SetSeverityText(level.String())
|
||||
record.SetBody(log.StringValue(string(body)))
|
||||
global.GetLoggerProvider().Logger(scope).Emit(ctx, record)
|
||||
}
|
||||
|
||||
type requestIDKey struct{}
|
||||
|
||||
func WithRequestID(ctx context.Context, id string) context.Context {
|
||||
return context.WithValue(ctx, requestIDKey{}, id)
|
||||
}
|
||||
|
||||
func RequestID(ctx context.Context) string {
|
||||
id, _ := ctx.Value(requestIDKey{}).(string)
|
||||
return id
|
||||
}
|
||||
|
||||
func End(span trace.Span, err error) {
|
||||
if err != nil {
|
||||
kind := "operation_failed"
|
||||
switch {
|
||||
case errors.Is(err, context.DeadlineExceeded):
|
||||
kind = "deadline_exceeded"
|
||||
case errors.Is(err, context.Canceled):
|
||||
kind = "canceled"
|
||||
}
|
||||
span.SetAttributes(attribute.String("error.type", kind))
|
||||
span.SetStatus(codes.Error, kind)
|
||||
}
|
||||
span.End()
|
||||
}
|
||||
|
||||
func observeProcess(m metric.Meter) {
|
||||
cpu := instrument(m.Float64ObservableCounter("messenger_process_cpu_time", metric.WithUnit("s")))
|
||||
rss := instrument(m.Int64ObservableGauge("messenger_process_memory", metric.WithUnit("By")))
|
||||
_, _ = m.RegisterCallback(func(_ context.Context, observer metric.Observer) error {
|
||||
var usage syscall.Rusage
|
||||
if err := syscall.Getrusage(syscall.RUSAGE_SELF, &usage); err == nil {
|
||||
seconds := float64(usage.Utime.Sec+usage.Stime.Sec) + float64(usage.Utime.Usec+usage.Stime.Usec)/1e6
|
||||
observer.ObserveFloat64(cpu, seconds)
|
||||
}
|
||||
// The production binaries run on Linux. statm reports resident pages.
|
||||
if raw, err := os.ReadFile("/proc/self/statm"); err == nil {
|
||||
if fields := strings.Fields(string(raw)); len(fields) > 1 {
|
||||
if pages, err := strconv.ParseInt(fields[1], 10, 64); err == nil {
|
||||
observer.ObserveInt64(rss, pages*int64(os.Getpagesize()))
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}, cpu, rss)
|
||||
}
|
||||
@@ -0,0 +1,64 @@
|
||||
package pipeline
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log/slog"
|
||||
"time"
|
||||
|
||||
"highload/internal/repository/postgres"
|
||||
"highload/internal/storage"
|
||||
)
|
||||
|
||||
type Cleanup struct {
|
||||
Repo *postgres.AttachmentRepository
|
||||
Objects storage.ObjectCleanup
|
||||
Grace time.Duration
|
||||
Retention time.Duration
|
||||
DryRun bool
|
||||
}
|
||||
|
||||
func (c Cleanup) RunOnce(ctx context.Context) (int, error) {
|
||||
policy := postgres.CleanupPolicy{Now: time.Now(), Grace: c.Grace, Retention: c.Retention, DryRun: c.DryRun}
|
||||
removed := 0
|
||||
for _, prefix := range []string{"staging/", "results/", "attachments/"} {
|
||||
var token *string
|
||||
for {
|
||||
pageCtx, cancel := context.WithTimeout(ctx, 5*time.Second)
|
||||
objects, next, err := c.Objects.ListPage(pageCtx, token, prefix)
|
||||
cancel()
|
||||
if err != nil {
|
||||
return removed, err
|
||||
}
|
||||
for _, obj := range objects {
|
||||
eligible, err := c.Repo.CleanupObject(ctx, obj, policy, c.Objects.Delete)
|
||||
if err != nil {
|
||||
return removed, err
|
||||
}
|
||||
if eligible {
|
||||
removed++
|
||||
}
|
||||
}
|
||||
if next == nil {
|
||||
break
|
||||
}
|
||||
token = next
|
||||
}
|
||||
}
|
||||
return removed, nil
|
||||
}
|
||||
|
||||
func (c Cleanup) Serve(ctx context.Context, period time.Duration) {
|
||||
for ctx.Err() == nil {
|
||||
count, err := c.RunOnce(ctx)
|
||||
if err != nil && ctx.Err() == nil {
|
||||
slog.Warn("cleanup pass failed", "error", err)
|
||||
}
|
||||
if count > 0 {
|
||||
slog.Info("cleanup pass", "objects", count, "dry_run", c.DryRun)
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
case <-time.After(period):
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,410 @@
|
||||
package pipeline
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"log/slog"
|
||||
"math/rand/v2"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"time"
|
||||
|
||||
"highload/internal/broker"
|
||||
"highload/internal/config"
|
||||
"highload/internal/fault"
|
||||
"highload/internal/imageproc"
|
||||
"highload/internal/model"
|
||||
"highload/internal/observability"
|
||||
"highload/internal/repository"
|
||||
"highload/internal/storage"
|
||||
|
||||
"github.com/google/uuid"
|
||||
amqp "github.com/rabbitmq/amqp091-go"
|
||||
"go.opentelemetry.io/otel/attribute"
|
||||
"go.opentelemetry.io/otel/codes"
|
||||
"go.opentelemetry.io/otel/log"
|
||||
"go.opentelemetry.io/otel/trace"
|
||||
)
|
||||
|
||||
type WorkerRepository interface {
|
||||
ClaimJob(context.Context, string, time.Time) (*model.AttachmentJob, error)
|
||||
GetAttachmentJobState(context.Context, string) (*model.AttachmentJobState, error)
|
||||
GetAttachmentByID(context.Context, string) (*model.Attachment, error)
|
||||
CompleteJob(context.Context, string, int32, string, []model.AttachmentVariant) (bool, error)
|
||||
PublishedObjectKeys(context.Context, string, int32) ([]string, error)
|
||||
ScheduleRetry(context.Context, string, int32, string, time.Time) (bool, error)
|
||||
FailJob(context.Context, string, int32, string) (bool, error)
|
||||
}
|
||||
|
||||
type WorkerObjects interface {
|
||||
storage.ObjectWriter
|
||||
Head(context.Context, string) error
|
||||
}
|
||||
|
||||
type Worker struct {
|
||||
Repo WorkerRepository
|
||||
Objects WorkerObjects
|
||||
Config config.Pipeline
|
||||
}
|
||||
|
||||
type message struct {
|
||||
SchemaVersion int `json:"schema_version"`
|
||||
Type string `json:"type"`
|
||||
JobID string `json:"job_id"`
|
||||
AttachmentID string `json:"attachment_id"`
|
||||
PipelineVersion int32 `json:"pipeline_version"`
|
||||
Attempt int `json:"attempt"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
Traceparent string `json:"traceparent,omitempty"`
|
||||
Tracestate string `json:"tracestate,omitempty"`
|
||||
}
|
||||
|
||||
func decodeMessage(payload []byte) (message, error) {
|
||||
var msg message
|
||||
if len(payload) > 4096 {
|
||||
return msg, errors.New("oversize job")
|
||||
}
|
||||
d := json.NewDecoder(bytes.NewReader(payload))
|
||||
d.DisallowUnknownFields()
|
||||
if err := d.Decode(&msg); err != nil {
|
||||
return msg, err
|
||||
}
|
||||
if err := d.Decode(new(any)); err != io.EOF {
|
||||
return msg, errors.New("trailing job data")
|
||||
}
|
||||
jobID, err1 := uuid.Parse(msg.JobID)
|
||||
attachmentID, err2 := uuid.Parse(msg.AttachmentID)
|
||||
if err1 != nil || err2 != nil || msg.SchemaVersion != 1 || msg.Type != "attachment.process.v1" ||
|
||||
msg.PipelineVersion <= 0 || msg.Attempt <= 0 || msg.CreatedAt.IsZero() {
|
||||
return msg, errors.New("invalid job envelope")
|
||||
}
|
||||
msg.JobID, msg.AttachmentID = jobID.String(), attachmentID.String()
|
||||
return msg, nil
|
||||
}
|
||||
|
||||
func (w Worker) Serve(ctx context.Context) {
|
||||
for ctx.Err() == nil {
|
||||
if err := w.consume(ctx); err != nil && ctx.Err() == nil {
|
||||
slog.Warn("worker consumer reconnecting", "error", err)
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
case <-time.After(500 * time.Millisecond):
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (w Worker) consume(ctx context.Context) error {
|
||||
conn, ch, err := broker.Connect(w.Config.BrokerURL)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer broker.Close(conn)
|
||||
if err := ch.Qos(w.Config.Prefetch, 0, false); err != nil {
|
||||
return err
|
||||
}
|
||||
deliveries, err := ch.ConsumeWithContext(ctx, broker.Queue, "", false, false, false, false, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case delivery, ok := <-deliveries:
|
||||
if !ok {
|
||||
return errors.New("broker delivery channel closed")
|
||||
}
|
||||
msg, err := decodeMessage(delivery.Body)
|
||||
if err != nil {
|
||||
slog.Warn("rejecting invalid job envelope")
|
||||
if err := delivery.Reject(false); err != nil {
|
||||
return err
|
||||
}
|
||||
continue
|
||||
}
|
||||
if err := w.consumeDelivery(ctx, delivery, msg); err != nil {
|
||||
// Closing the channel requeues an unacknowledged delivery.
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (w Worker) consumeDelivery(ctx context.Context, delivery amqp.Delivery, msg message) (err error) {
|
||||
ctx = observability.JobContext(ctx, msg.Traceparent, msg.Tracestate)
|
||||
ctx, span := observability.Tracer.Start(ctx, "attachment.process",
|
||||
trace.WithSpanKind(trace.SpanKindConsumer), trace.WithAttributes(
|
||||
attribute.String("messaging.system", "rabbitmq"),
|
||||
attribute.String("messaging.destination.name", broker.Queue),
|
||||
attribute.String("messaging.operation.name", "process"),
|
||||
attribute.String("messaging.message.id", msg.JobID),
|
||||
))
|
||||
defer func() { observability.End(span, err) }()
|
||||
observability.Log(ctx, "job.received", log.SeverityInfo,
|
||||
attribute.String("job_id", msg.JobID), attribute.Int("envelope_attempt", msg.Attempt))
|
||||
reject, err := w.handle(ctx, msg)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if reject {
|
||||
span.SetAttributes(attribute.String("messaging.delivery.disposition", "rejected"))
|
||||
return delivery.Reject(false)
|
||||
}
|
||||
span.SetAttributes(attribute.String("messaging.delivery.disposition", "acked"))
|
||||
return delivery.Ack(false)
|
||||
}
|
||||
|
||||
func (w Worker) handle(ctx context.Context, msg message) (bool, error) {
|
||||
state, err := w.Repo.GetAttachmentJobState(ctx, msg.JobID)
|
||||
if errors.Is(err, repository.ErrNotFound) {
|
||||
return true, nil
|
||||
}
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
if state.AttachmentID != msg.AttachmentID || state.PipelineVersion != msg.PipelineVersion {
|
||||
return true, nil
|
||||
}
|
||||
if state.Status == "superseded" {
|
||||
return false, nil
|
||||
}
|
||||
job, err := w.Repo.ClaimJob(ctx, msg.JobID, time.Now().Add(w.Config.Lease))
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
if job == nil {
|
||||
return false, w.handleUnclaimed(ctx, msg)
|
||||
}
|
||||
slog.Info("job claimed", "job_id", job.ID, "attempt", job.AttemptsStarted, "lease_generation", job.LeaseGeneration)
|
||||
|
||||
outcome, err := w.runAttempt(ctx, job, msg)
|
||||
if err != nil || outcome != "ready" {
|
||||
return false, err
|
||||
}
|
||||
|
||||
pauseCtx, pauseCancel := context.WithTimeout(ctx, 30*time.Second)
|
||||
defer pauseCancel()
|
||||
return false, fault.Pause(pauseCtx, "before_ack", job.ID)
|
||||
}
|
||||
|
||||
func (w Worker) runAttempt(ctx context.Context, job *model.AttachmentJob, msg message) (outcome string, err error) {
|
||||
started := time.Now()
|
||||
span := trace.SpanFromContext(ctx)
|
||||
span.SetAttributes(attribute.Int("job.attempt", int(job.AttemptsStarted)))
|
||||
done := observability.WorkerAttempt()
|
||||
defer func() {
|
||||
done()
|
||||
observability.ObserveJob(ctx, outcome, time.Since(started))
|
||||
span.SetAttributes(attribute.String("job.outcome", outcome))
|
||||
if outcome != "ready" {
|
||||
span.SetStatus(codes.Error, outcome)
|
||||
}
|
||||
observability.Log(ctx, "job.finished", log.SeverityInfo,
|
||||
attribute.String("job_id", job.ID), attribute.Int("attempt", int(job.AttemptsStarted)),
|
||||
attribute.String("outcome", outcome))
|
||||
}()
|
||||
attemptCtx, cancel := context.WithTimeout(ctx, w.Config.Timeout)
|
||||
failure := w.process(attemptCtx, job)
|
||||
cancel()
|
||||
if ctx.Err() != nil {
|
||||
return "interrupted", ctx.Err()
|
||||
}
|
||||
if failure == nil {
|
||||
return "ready", nil
|
||||
}
|
||||
if errors.Is(failure, errLostLease) {
|
||||
return "lease_lost", w.handleUnclaimed(ctx, msg)
|
||||
}
|
||||
return w.saveFailure(ctx, job, failure)
|
||||
}
|
||||
|
||||
func (w Worker) handleUnclaimed(ctx context.Context, msg message) error {
|
||||
state, err := w.Repo.GetAttachmentJobState(ctx, msg.JobID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
switch state.Status {
|
||||
case "ready":
|
||||
checkCtx, cancel := context.WithTimeout(ctx, 5*time.Second)
|
||||
defer cancel()
|
||||
keys, err := w.Repo.PublishedObjectKeys(checkCtx, state.AttachmentID, state.PipelineVersion)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, key := range keys {
|
||||
if err := w.Objects.Head(checkCtx, key); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
case "failed", "queued", "superseded":
|
||||
return nil // A queued retry is already in the transactionally durable outbox.
|
||||
case "processing":
|
||||
if state.AttemptsStarted >= state.MaxAttempts && state.LeaseExpired {
|
||||
_, err = w.Repo.FailJob(ctx, msg.JobID, state.LeaseGeneration, "retry_exhausted")
|
||||
return err
|
||||
}
|
||||
return nil // The recovery loop publishes one event for this expired lease.
|
||||
default:
|
||||
return errors.New("unrecognized job state")
|
||||
}
|
||||
}
|
||||
|
||||
var (
|
||||
errLostLease = errors.New("job lease lost")
|
||||
errOriginalMismatch = errors.New("staging hash or size differs")
|
||||
)
|
||||
|
||||
func (w Worker) process(ctx context.Context, job *model.AttachmentJob) error {
|
||||
att, err := w.Repo.GetAttachmentByID(ctx, job.AttachmentID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if att.PipelineVersion != job.PipelineVersion {
|
||||
return errLostLease
|
||||
}
|
||||
if att.StagingKey == "" {
|
||||
return errOriginalMismatch
|
||||
}
|
||||
dir, err := os.MkdirTemp("", "attachment-job-")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer os.RemoveAll(dir)
|
||||
source := filepath.Join(dir, "input")
|
||||
if err := w.download(ctx, att, source); err != nil {
|
||||
return err
|
||||
}
|
||||
files, err := imageproc.Generate(ctx, dir, source, att.ContentType, job.PipelineVersion)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := fault.Pause(ctx, "before_objects", job.ID); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
prefix := fmt.Sprintf("results/%s/v%d/%d/", job.AttachmentID, job.PipelineVersion, job.LeaseGeneration)
|
||||
originalKey := prefix + "original"
|
||||
if err := w.putFile(ctx, originalKey, att.ContentType, source, att.SizeBytes); err != nil {
|
||||
return err
|
||||
}
|
||||
variants := make([]model.AttachmentVariant, 0, len(files))
|
||||
for _, f := range files {
|
||||
v := f.Variant
|
||||
v.ObjectKey = prefix + filepath.Base(f.Path)
|
||||
if err := w.putFile(ctx, v.ObjectKey, v.ContentType, f.Path, v.SizeBytes); err != nil {
|
||||
return err
|
||||
}
|
||||
variants = append(variants, v)
|
||||
}
|
||||
if err := fault.Pause(ctx, "after_objects", job.ID); err != nil {
|
||||
return err
|
||||
}
|
||||
done, err := w.Repo.CompleteJob(ctx, job.ID, job.LeaseGeneration, originalKey, variants)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !done {
|
||||
slog.Info("job finalization rejected", "job_id", job.ID, "lease_generation", job.LeaseGeneration)
|
||||
return errLostLease
|
||||
}
|
||||
observability.ObserveReady(ctx, job.AcceptedAt)
|
||||
observability.Log(ctx, "job.ready", log.SeverityInfo, attribute.String("job_id", job.ID))
|
||||
slog.Info("job ready", "job_id", job.ID, "lease_generation", job.LeaseGeneration)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (w Worker) download(ctx context.Context, att *model.Attachment, path string) error {
|
||||
if att.SizeBytes <= 0 || att.SizeBytes > imageproc.MaxBytes {
|
||||
return errOriginalMismatch
|
||||
}
|
||||
|
||||
reader, err := w.Objects.Get(ctx, att.StagingKey)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer reader.Close()
|
||||
f, err := os.Create(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
sum := sha256.New()
|
||||
n, copyErr := io.Copy(io.MultiWriter(f, sum), io.LimitReader(reader, att.SizeBytes+1))
|
||||
closeErr := f.Close()
|
||||
if copyErr != nil {
|
||||
return copyErr
|
||||
}
|
||||
if closeErr != nil {
|
||||
return closeErr
|
||||
}
|
||||
if n != att.SizeBytes || hex.EncodeToString(sum.Sum(nil)) != att.SHA256 {
|
||||
return errOriginalMismatch
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (w Worker) putFile(ctx context.Context, key, mime, path string, size int64) error {
|
||||
f, err := os.Open(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer f.Close()
|
||||
return w.Objects.Put(ctx, key, mime, f, size)
|
||||
}
|
||||
|
||||
func (w Worker) saveFailure(ctx context.Context, job *model.AttachmentJob, failure error) (string, error) {
|
||||
code := "processing_failed"
|
||||
permanent := false
|
||||
switch {
|
||||
case errors.Is(failure, errOriginalMismatch), errors.Is(failure, imageproc.ErrInvalidImage):
|
||||
code, permanent = "invalid_image", true
|
||||
case errors.Is(failure, imageproc.ErrResourceLimit):
|
||||
code, permanent = "resource_limit", true
|
||||
case errors.Is(failure, storage.ErrObjectNotFound):
|
||||
code, permanent = "original_missing", true
|
||||
case errors.Is(failure, storage.ErrServiceUnavailable):
|
||||
code = "s3_unavailable"
|
||||
case errors.Is(failure, context.DeadlineExceeded):
|
||||
code = "processing_timeout"
|
||||
}
|
||||
dbCtx, cancel := context.WithTimeout(ctx, 5*time.Second)
|
||||
defer cancel()
|
||||
if permanent || job.AttemptsStarted >= job.MaxAttempts {
|
||||
if !permanent {
|
||||
code = "retry_exhausted"
|
||||
}
|
||||
changed, err := w.Repo.FailJob(dbCtx, job.ID, job.LeaseGeneration, code)
|
||||
if err != nil {
|
||||
return "error", err
|
||||
}
|
||||
if !changed {
|
||||
return "lease_lost", nil
|
||||
}
|
||||
observability.Log(ctx, "job.failed", log.SeverityWarn, attribute.String("job_id", job.ID),
|
||||
attribute.String("error_code", code))
|
||||
slog.Warn("job failed", "job_id", job.ID, "error_code", code)
|
||||
return "failed", nil
|
||||
}
|
||||
// The floor avoids immediate requeue loops. The upper bound grows with
|
||||
// each attempt and is capped to keep the lab recoverable.
|
||||
upper := min(30*time.Second, w.Config.RetryBase<<min(job.AttemptsStarted-1, 4))
|
||||
delay := time.Second + time.Duration(rand.Int64N(int64(upper-time.Second+1)))
|
||||
changed, err := w.Repo.ScheduleRetry(dbCtx, job.ID, job.LeaseGeneration, code, time.Now().Add(delay))
|
||||
if err != nil {
|
||||
return "error", err
|
||||
}
|
||||
if !changed {
|
||||
return "lease_lost", nil
|
||||
}
|
||||
observability.Log(ctx, "job.retry_scheduled", log.SeverityInfo, attribute.String("job_id", job.ID),
|
||||
attribute.String("error_code", code), attribute.Int64("delay_ms", delay.Milliseconds()))
|
||||
slog.Info("job retry scheduled", "job_id", job.ID, "error_code", code, "delay_ms", delay.Milliseconds())
|
||||
return "retry", nil
|
||||
}
|
||||
@@ -0,0 +1,36 @@
|
||||
package repository
|
||||
|
||||
import (
|
||||
"context"
|
||||
"highload/internal/model"
|
||||
"time"
|
||||
)
|
||||
|
||||
type AttachmentRepository interface {
|
||||
CreateAttachment(ctx context.Context, params CreateAttachmentParams) (*model.Attachment, error)
|
||||
GetAttachmentForUser(ctx context.Context, userID, attachmentID string) (*model.Attachment, error)
|
||||
ExistsByObjectKey(ctx context.Context, key string) (bool, error)
|
||||
WithObjectKeyLock(ctx context.Context, key string, fn func(context.Context, AttachmentTx) error) (bool, error)
|
||||
}
|
||||
|
||||
type AttachmentKey struct {
|
||||
ID string
|
||||
ObjectKey string
|
||||
CreatedAt time.Time
|
||||
}
|
||||
|
||||
type AttachmentReconcileRepository interface {
|
||||
ListAttachmentKeys(ctx context.Context, afterKey string, pageSize int32) ([]AttachmentKey, error)
|
||||
GetAttachmentByID(ctx context.Context, id string) (*model.Attachment, error)
|
||||
WithObjectKeyLock(ctx context.Context, key string, fn func(context.Context, AttachmentTx) error) (bool, error)
|
||||
}
|
||||
|
||||
type CreateAttachmentParams struct {
|
||||
ChatID string
|
||||
OwnerID string
|
||||
Filename string
|
||||
ObjectKey string
|
||||
ContentType string
|
||||
Sha256 string
|
||||
SizeBytes int64
|
||||
}
|
||||
@@ -0,0 +1,13 @@
|
||||
package repository
|
||||
|
||||
import (
|
||||
"context"
|
||||
"highload/internal/model"
|
||||
)
|
||||
|
||||
type AttachmentTx interface {
|
||||
CreateAttachment(ctx context.Context, params CreateAttachmentParams) (*model.Attachment, error)
|
||||
CreateQueued(ctx context.Context, id, chatID, ownerID, mime, sha256, key, filename string, size int64) (*model.Attachment, error)
|
||||
ExistsByObjectKey(ctx context.Context, key string) (bool, error)
|
||||
GetAttachmentByID(ctx context.Context, id string) (*model.Attachment, error)
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user