184 changed files with 20450 additions and 54 deletions
+17 -13
View File
@@ -1,30 +1,34 @@
# Copy to .env for your LOCAL synthetic test environment; never commit .env. # Copy to .env for your LOCAL synthetic test environment; never commit .env.
# These are logical interface suggestions; map actual framework names in REPORT.md.
APP_HOST=0.0.0.0 APP_HOST=0.0.0.0
APP_PORT=8080 APP_PORT=8080
APP_ENV=lab APP_ENV=lab
SESSION_TTL_SECONDS=3600 SESSION_TTL_SECONDS=3600
# Choose credentials locally. File paths contain no secret values. # Choose credentials locally. File paths contain no secret values.
DATABASE_URL_FILE=/run/secrets/database_url DATABASE_URL_FILE=/run/secrets/database_url
# Optional on 3, required use on 4 in lab2; Redis OR Valkey: SESSION_STORE=redis
SESSION_STORE=postgres REDIS_ADDR=redis:6379
CACHE_URL_FILE=/run/secrets/cache_url
API_REPLICAS=2
IDEMPOTENCY_TTL_SECONDS=86400
S3_ENDPOINT=http://s3:9000 S3_ENDPOINT=http://s3:9000
S3_REGION=us-east-1 S3_REGION=us-east-1
S3_BUCKET=messenger-attachments S3_BUCKET=messenger-attachments
S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id
S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key
MAX_UPLOAD_BYTES=10485760
MAX_IMAGE_PIXELS=40000000
BROKER_URL_FILE=/run/secrets/broker_url BROKER_URL_FILE=/run/secrets/broker_url
WORKER_CONCURRENCY=2 WORKER_CONCURRENCY=1
WORKER_PREFETCH=1
JOB_MAX_ATTEMPTS=5 JOB_MAX_ATTEMPTS=5
JOB_TIMEOUT_SECONDS=60 JOB_TIMEOUT_SECONDS=60
JOB_LEASE_SECONDS=90
JOB_RETRY_BASE_SECONDS=2
STAGING_RETENTION_SECONDS=86400 STAGING_RETENTION_SECONDS=86400
OTEL_SERVICE_NAME=messenger-api CLEANUP_GRACE_SECONDS=600
# Worker must set a distinct service.name. CLEANUP_INTERVAL_SECONDS=5
# Lab 6 observability. Compose sets per-process service.name.
LAB6_HTTP_PORT=8086
LAB6_POSTGRES_PORT=15486
LAB6_GRAFANA_PORT=3006
LAB6_PROMETHEUS_PORT=9096
LAB6_LOKI_PORT=3106
LAB6_TEMPO_PORT=3206
OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4318 OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4318
OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf OTEL_SERVICE_VERSION=lab6
OTEL_RESOURCE_ATTRIBUTES=deployment.environment.name=lab
+23 -14
View File
@@ -1,18 +1,27 @@
## Лабораторная 6 ## Лабораторная 6
Автор / группа: TODO Олег Ремез, БПИ2304. Целевая оценка — 4.
Целевая оценка (3/4/5): TODO
Трек 5, если заявлен: TODO
Что реализовано и что перенесено: TODO Перенесён мессенджер ЛР 5 с двумя API-репликами, outbox и обработкой
Команды запуска из чистого clone: TODO вложений. Добавлены OpenTelemetry, Collector, Prometheus, Loki, Tempo и
Результаты проверок и ссылка на REPORT.md: TODO Grafana. W3C-контекст связывает загрузку с обработкой и повторной попыткой.
Ограничения / известные ошибки: TODO
Источники и использование ИИ: TODO
- [ ] Весь обязательный минимум текущей и предыдущих работ сохранён с объявленными переходами. ### Запуск
- [ ] Контракт и публичные тесты не ослаблены.
- [ ] make check и make test выполнены; приложен реальный вывод. ```sh
- [ ] Сценарии отказов/безопасности своего уровня воспроизведены. sh scripts/init-local.sh
- [ ] Все секреты и приватные ключи находятся вне Git; отчёт не содержит tokens. docker compose --env-file .env up -d --build --wait --wait-timeout 180
- [ ] README задания сохранён, REPORT.md содержит инструкцию именно моего решения. make check test BASE_URL=http://127.0.0.1:8086
```
### Проверки
Все девять проверок приёмки прошли. При остановке worker и блокировке БД
оба локальных алерта сработали и вернулись в inactive. За 34 с недоступности
Loki сохранились восемь подтверждённых сообщений, потери логов составили
96 записей. Копия для сдачи содержит конфигурацию и обезличенные результаты.
Для запуска локальные секреты создаёт `init-local.sh`.
Отчёт содержит SLO, ограничения экспериментов, ссылки на первичные источники
и описание участия Codex в реализации и тестировании.
[Отчёт](REPORT.md) · [Сводка приёмки](evidence/lab6-grade4/summary.json).
+21
View File
@@ -0,0 +1,21 @@
name: Lab 6 application
on: [push, pull_request]
permissions:
contents: read
jobs:
compose-smoke:
runs-on: ubuntu-latest
timeout-minutes: 20
env:
COMPOSE_PROJECT_NAME: lab6-ci
steps:
- uses: actions/checkout@v4
- name: Initialize disposable configuration
run: sh scripts/init-local.sh
- name: Build and start
run: docker compose --env-file .env up -d --build --wait --wait-timeout 180
- name: Run contract smoke
run: make check test BASE_URL=http://127.0.0.1:8086
- name: Remove the CI project
if: always()
run: docker compose --env-file .env down --volumes
+45
View File
@@ -0,0 +1,45 @@
FROM golang:1.27.1-alpine AS builder
WORKDIR /app
COPY go.mod go.sum ./
RUN go mod download
COPY cmd/api/ ./cmd/api/
COPY cmd/jobs/ ./cmd/jobs/
COPY cmd/relay/ ./cmd/relay/
COPY internal/ ./internal/
RUN --mount=type=cache,id=highload-go-build,target=/root/.cache/go-build,sharing=locked \
CGO_ENABLED=0 GOOS=linux go build -p 2 -trimpath -o /app/server ./cmd/api && \
CGO_ENABLED=0 GOOS=linux go build -p 2 -trimpath -o /app/jobs ./cmd/jobs && \
CGO_ENABLED=0 GOOS=linux go build -p 2 -trimpath -o /app/relay ./cmd/relay
FROM alpine:3.22 AS runtime
RUN addgroup -S app && adduser -S app -G app
WORKDIR /app
USER app
FROM runtime AS cleanup
COPY --from=builder /app/jobs ./jobs
ENTRYPOINT ["/app/jobs"]
CMD ["cleanup"]
FROM runtime AS relay
COPY --from=builder /app/relay ./relay
CMD ["./relay"]
FROM runtime AS api
COPY --from=builder /app/server ./server
EXPOSE 8080
HEALTHCHECK --interval=10s --timeout=3s --start-period=3s --retries=3 \
CMD wget -q -O /dev/null http://127.0.0.1:8080/health/ready || exit 1
CMD ["./server"]
+20
View File
@@ -0,0 +1,20 @@
FROM golang:1.27.1-alpine AS builder
WORKDIR /app
COPY go.mod go.sum ./
RUN go mod download
COPY cmd/migrate/ ./cmd/migrate/
COPY db/migrations/ ./db/migrations/
RUN --mount=type=cache,id=highload-go-build,target=/root/.cache/go-build,sharing=locked \
CGO_ENABLED=0 GOOS=linux go build -p 2 -trimpath -o /app/migrate ./cmd/migrate
FROM alpine:3.22
RUN addgroup -S app && adduser -S app -G app
WORKDIR /app
COPY --from=builder /app/migrate ./migrate
USER app
ENTRYPOINT ["/app/migrate"]
+23
View File
@@ -0,0 +1,23 @@
FROM golang:1.27.1-alpine AS builder
WORKDIR /app
COPY go.mod go.sum ./
RUN go mod download
COPY cmd/worker/ ./cmd/worker/
COPY internal/ ./internal/
RUN --mount=type=cache,id=highload-go-build,target=/root/.cache/go-build,sharing=locked \
CGO_ENABLED=0 GOOS=linux go build -p 2 -trimpath -o /app/worker ./cmd/worker
FROM alpine:3.22@sha256:49340b346798875b5592e5d420241f2f9cfba5f47e62ebb1446a1b34734e898f
RUN apk add --no-cache imagemagick imagemagick-jpeg imagemagick-webp \
&& addgroup -S app && adduser -S -G app app
COPY infra/imagemagick/policy.xml /etc/ImageMagick-7/policy.xml
COPY --from=builder /app/worker /usr/local/bin/worker
WORKDIR /tmp
USER app
CMD ["/usr/local/bin/worker"]
+171 -23
View File
@@ -1,37 +1,185 @@
# Отчёт — лабораторная 6 # Лабораторная 6. Наблюдаемость
- Автор, группа: TODO - Автор и группа: Олег Ремез, БПИ2304.
- Целевая оценка; выбранный трек 5, если нужен: TODO - Целевая оценка: 4. Ветка реализации — `solution/BPI2304-remez`.
- Commit/PR: TODO после публикации - Реализация и отчёт подготовлены с помощью Codex. Независимые сценарии приёмки выполнил отдельный тестовый чат.
- Использованные источники, библиотеки и помощь ИИ: TODO
## Запуск из чистого clone ## Запуск из чистого клона
TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных. Нужны Docker с Compose v2, OpenSSL, make и Python 3.14. Приложение
собирается на Go 1.27.1 в Docker. Compose закрепляет версии образов. Для первого
запуска нужно скачивание образов и модулей Go.
## Архитектура и решения ```sh
sh scripts/init-local.sh
docker compose --env-file .env config --quiet
docker compose --env-file .env up -d --build --wait --wait-timeout 180
curl --fail http://127.0.0.1:8086/health/ready
make check test BASE_URL=http://127.0.0.1:8086
```
TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab5, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации. Миграции выполняет сервис `migrate`; API и worker ждут его успешного завершения.
`init-local.sh` создаёт локальные `.env` и секреты в `secrets/`; при повторном
запуске сохраняет существующие значения. Наружные порты можно изменить
переменными `LAB6_*_PORT`, внутренний порт API и количество реплик согласованы
с настройками HAProxy. API использует Redis по `redis:6379`. Настройки worker,
повторов и очистки перечислены в `.env.example`. Лимиты загрузки и обработки
изображений заданы в `internal/imageproc`: 10 МиБ и 40 млн пикселей.
- API через HAProxy доступен на [8086](http://127.0.0.1:8086/health/ready).
- Dashboard [Messenger observability](http://127.0.0.1:3006/d/lab6-messenger)
и [Prometheus alerts](http://127.0.0.1:9096/alerts) доступны по loopback.
- Loki и Tempo отвечают на `127.0.0.1:3106` и `127.0.0.1:3206`. Grafana
автоматически получает источники и dashboard из `infra/otel/grafana`.
- `docker compose down` останавливает стенд и сохраняет историю в именованных
томах. `docker compose down --volumes` удаляет данные этого Compose-проекта.
## Архитектура
API, relay и worker отправляют OTLP/HTTP на Collector 0.131.0. Он выставляет
метрики на scrape endpoint для Prometheus 3.5.0, отправляет логи через Loki
OTLP endpoint в Loki 3.5.0, трассы через OTLP в Tempo 2.8.1. Prometheus хранит
временные ряды, Loki индексирует логи по служебным labels, Tempo хранит дерево
spans. Grafana 12.3.1 читает эти данные из трёх provisioned datasources.
Окно хранения всех трёх сигналов составляет до 24 часов. У Prometheus также задан предел TSDB 512 МБ.
Сохранён pipeline ЛР 5. HAProxy распределяет запросы между двумя API,
сессии хранятся в Redis, PostgreSQL содержит сообщения, метаданные вложений,
задания и outbox, RustFS хранит объекты, RabbitMQ доставляет задания. Загрузка
пишет staging в S3 и в одной DB-транзакции сохраняет вложение, задание и
outbox. Relay публикует сообщение c broker confirmation. Worker захватывает
задание по lease, пишет оригинал и варианты в S3, завершает состояние БД и
только потом подтверждает доставку. Повторная выдача сохраняет прежнюю
семантику lease, retry и готовности вложения.
`traceparent` и необязательный `tracestate` попадают в первоначальный outbox
вместе с заданием. Relay извлекает W3C context, создаёт `rabbitmq.publish`
типа Producer и инъецирует его W3C context в публикуемый JSON. Worker
извлекает этот контекст для `attachment.process` типа Consumer. Повторный
outbox сохраняет контекст текущего consumer. Следующий producer и consumer
продолжают то же дерево. Дочерние операции S3 и PostgreSQL видны до
`CompleteJob` и `commit`. Лог запроса содержит сгенерированный сервером
`X-Request-Id`, trace ID и span ID, лог worker содержит технический job ID
и IDs активного span. Код экспортирует только явно заданные поля, включая
имя операции sqlc без SQL и значений параметров. Секреты, тела запросов,
файлы и бизнес-идентификаторы в labels исключены.
Использованы Go OpenTelemetry SDK 1.38.0 и SDK логов 0.14.0. В resource
указаны `service.name` (`messenger-api`, `messenger-relay`,
`messenger-worker`), `service.version`, `service.instance.id` и
`deployment.environment.name=lab`. HTTP и DB атрибуты следуют именованию
OTel semantic conventions (`http.request.method`, `http.route`,
`http.response.status_code`, `db.system.name`, `db.operation.name`);
инструменты `messenger_*` заданы приложением. Ссылки на первичные материалы
доступны в [задании](README.md#первичные-материалы). Использованы также
[Go SDK](https://opentelemetry.io/docs/languages/go/instrumentation/) и
[Loki OTLP](https://grafana.com/docs/loki/latest/send-data/otel/).
## Сигналы и условие SLO
| Сигнал | Prometheus и расчёт | Единица |
| --- | --- | --- |
| API RPS и ошибки | `messenger_http_requests_total`; `sum(rate(...[1m]))` по репликам, ошибки фильтруются `status_class="5xx"` | запросы/с |
| HTTP latency | `histogram_quantile(0.50\|0.95\|0.99, sum by (le) (rate(messenger_http_request_duration_seconds_bucket[1m])))` с исключением health | с |
| Доля ошибок | 5xx / все бизнес-ответы API за то же окно; health исключается из обеих частей | доля |
| Исход и длительность работы | `messenger_jobs_total` и `messenger_job_duration_seconds_bucket` с `job_type`, `outcome` | попытки/с, с |
| Глубина и возраст очереди | `messenger_queue_depth` и `messenger_queue_oldest_age_seconds` по `queue="image.jobs"` | задания, с |
| Пул БД | `messenger_db_pool_connections{state="used"\|"capacity"}`, `messenger_db_pool_waiting` | соединения, активные ожидания acquire |
| Занятость worker | `sum(messenger_worker_busy) / sum(messenger_worker_capacity)` | доля |
| CPU и память процесса | `rate(messenger_process_cpu_time_seconds_total[1m])`, `messenger_process_memory_bytes` (RSS) | ядра, байты |
| Время до ready | `messenger_attachment_time_to_ready_seconds_bucket` от записи задания перед durable commit до commit результата | с |
Очередь измеряется SQL-снимком текущих jobs со статусом `queued` раз в две
секунды. В глубину входят задания, ещё ожидающие публикации через outbox, и
отложенные retry. Возраст считается с первого принятия соответствующей
версии задания. Задания, которые worker уже захватил, учитываются в
`messenger_worker_busy`. `messenger_queue_sample_age_seconds` защищает
алерт от устаревшего снимка при недоступной БД.
**SLI доступности API** — доля ответов без 5xx среди всех завершённых
бизнес-запросов к API. Считаются ответы всех реплик, кроме `/health/live`
и `/health/ready`. Возвращённые приложением 4xx входят в знаменатель.
**SLO** — не менее 99% в скользящем окне 5 минут при наличии запросов.
В окне без трафика SLI не вычисляется. PromQL для проверки:
```promql
1 - (
sum(increase(messenger_http_requests_total{route!~"/health/(live|ready)",status_class="5xx"}[5m]))
or 0 * sum(increase(messenger_http_requests_total{route!~"/health/(live|ready)"}[5m]))
) / (sum(increase(messenger_http_requests_total{route!~"/health/(live|ready)"}[5m])) > 0)
```
Prometheus проверяет правила каждые 5 секунд. `HighAPIErrorRate` срабатывает
при 5xx >10% в окне 30 секунд и достаточном трафике (оценка ≥5 запросов),
если условие держится 15 секунд. `GrowingQueueLag` требует непустой
очереди и свежего снимка с возрастом старейшей задачи >15 секунд в течение
15 секунд. Оба алерта используют короткие локальные пороги для учебных
инцидентов. Команды и последовательность диагностики описаны в
[runbook](infra/otel/runbook.md).
## Сбои контура телеметрии
У логов и трасс отдельные async очереди в Go SDK на 512 записей/спанов.
Экспорт имеет таймаут 1,5 с и отключённые SDK retry, период метрик 2 с.
Collector использует memory limiter 96 МиБ (spike 24 МиБ, проверка раз в
секунду), сборку пачек до 128 элементов и отдельные очереди Loki/Tempo по
16 пачек с одним consumer на экспортёр. Таймаут попытки передачи 2 с, лимит
retry 5 с. Контейнер Collector ограничен 256 МиБ. При длительном отказе
экспортёр отбрасывает данные. Метрики send/enqueue failures показывают,
сколько записей потеряно на стороне Collector. Пайплайн работы с
сообщениями и вложениями не ждёт экспорта. Телеметрия не участвует в
readiness API.
## Доказательства критериев ## Доказательства критериев
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | Проверка 16 ноября 2026 года на Apple M2 с 8 ГБ RAM. Docker Engine
| --- | --- | --- | --- | 28.4.0, Compose 2.39.4, arm64, Python 3.14.6. Ограничения контейнеров API
| Публичный smoke | make test | TODO | TODO | составляют 2 CPU и 512 МиБ на реплику, worker 2 CPU и 1536 МиБ с одним
| Остальные пункты уровня 3 | TODO | TODO | TODO | обработчиком. Тестовик использовал отдельный Compose-проект
| Все пункты уровня 4, если заявлен | TODO | TODO | TODO | `highload-lab6-student` и порт API 28106. Запуск и полный список сценариев
| Выбранный трек 5, если заявлен | TODO | TODO | TODO | описаны в [приёмке](student-tests/lab6/README.md). Все девять проверок
прошли. [Сводка](evidence/lab6-grade4/summary.json) указывает, в каком
[JSONL-прогоне](evidence/lab6-grade4/README.md) наблюдался каждый результат.
TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены. ### Инциденты
## Эксперименты и отказы | Воздействие | Что показали сохранённые сигналы | Восстановление |
| --- | --- | --- |
| Остановлен worker | Три загрузки получили 202. Prometheus наблюдал глубину 3 и возраст 18,59 с на первом замере выше порога. `GrowingQueueLag` стал `firing` через 39,82 с после начала загрузок. | Все 3 вложения стали `ready`, глубина 0, повторное чтение подтвердило SHA-256. Через 9,04 с после старта worker правило стало `inactive`. |
| В PostgreSQL удержан `ACCESS EXCLUSIVE` на `public.users` | `/health/ready` сохранил 200. 23 запроса `/api/v1/users/me` вернули 503 за 1,007–1,065 с. В Tempo дочерний `postgres.query` завершился ошибкой за 1,001 с, Loki показал 503 с тем же ID запроса. `HighAPIErrorRate` стал `firing` через 32,73 с. | После снятия блокировки запросы вернули 200. Через 23,81 с правило стало `inactive`. |
| Остановлен Loki | За 34 с API успешно ответил на 278 чтений и подтвердил 8 сообщений. Очередь достигла предела 16 пачек, RSS Collector за время отказа достиг 88,6 МиБ из 256 МиБ. Счётчик потерь вырос на 96 записей, первый рост обнаружен через 8,69 с. | Все восемь сообщений прочитаны без изменений. Collector сохранил тот же экземпляр без рестарта и OOM. Через 16,94 с после возобновления Loki тест нашёл новую запись по trace ID и ID запроса. |
TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения. Интервалы отсчитываются от действия теста до наблюдения через API
Prometheus. Они включают дискретность scrape и вычисления правил. В
первых двух сценариях поток создавали три загрузки либо последовательные
запросы чтения. В третьем проверялись успешные записи, непрерывные
чтения и ограничение очереди Collector. Это измерения конкретного стенда
на синтетических данных, без оценки предельного RPS или устойчивости
при длительной production-нагрузке. Во время холодного старта второго
Compose-проекта в одном пробном запуске Loki-сценария появился 503 с
длительностью около секунды. Итоговый эксперимент выполнен на прогретом
тестовом проекте после остановки второго стенда.
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | ### Матрица проверки
| --- | --- | --- | --- |
| TODO | TODO | TODO | TODO |
## Ограничения и следующие шаги | Критерий | Проверка | Наблюдаемый результат |
| --- | --- | --- |
| API ЛР 5 с включённой телеметрией | `make check`, `make test` | Публичный smoke прошёл с двумя API за HAProxy, мигратором, RabbitMQ, worker и S3. |
| Запуск и provisioning | `backend-readiness`, `provisioning` | Три backend готовы, dashboard и три datasource загружены автоматически. `promtool` принял оба правила; Collector принял конфигурацию. |
| API → broker → worker → S3/DB, включая retry | `trace-retry` | После временного разрыва доступа worker к S3 первая попытка сохранила `s3_unavailable`, вторая дошла до `ready`. В Tempo найдено одно дерево с 1 Server, 2 Producer и 2 Consumer spans. Ошибка первого `s3.get` и успешные S3/DB операции второй попытки дочерние к соответствующим Consumer. В обоих outbox payload тот же trace ID и `tracestate`. |
| Структурные логи и отсутствие чувствительных значений | `trace-retry`, проверки Loki и Tempo | Логи API и worker сопоставлены со spans, запись API найдена по `X-Request-Id`. Проверены синтетические пароль, Basic credential, Bearer token, текст сообщения, имя файла и фрагмент изображения. |
| Метрики и низкая кардинальность | `metrics` | Восемь разных чатов дали общий route template. В Prometheus обнаружены HTTP/job/ready buckets, pool, worker, очередь и ресурсы процесса. Проверка сравнила `+Inf` с `_count` и вычислила все три quantile из bucket. |
| Рост очереди и восстановление | `worker-outage` | Три ready из трёх, глубина до 3 и обратно до 0, зафиксированы `firing` и `inactive`. |
| Ошибки и восстановление PostgreSQL | `db-outage` | Сопоставили 503, лог и дочерний DB span. Зафиксированы `firing` и `inactive`. |
| Недоступность telemetry storage | `loki-outage` | Восемь из восьми сообщений сохранены, измерены потери в потоке логов, очередь и RSS в пределах лимитов. |
TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза. Для сравнения квантили HTTP в одном тестовом снимке составили около
2,2 мс, 209 мс и 1,0 с (p50/p95/p99). Значения вычисляются интерполяцией
по бакетам. Этот снимок включает тестовые запросы и время прогрева стенда.
После пересоздания контейнеров Prometheus, Loki, Tempo и Grafana на прежних томах повторно прочитаны исторический отсчёт метрики, запись по request ID и тот же server span. Grafana загрузила dashboard. [Результат проверки сохранности](evidence/telemetry-persistence.json).
18 ноября стенд повторно собран и запущен из ветки решения. Публичный
`make check test BASE_URL=http://127.0.0.1:8086`, `go test ./...`,
`go vet ./...` и `sqlc compile` завершились успешно. Grafana вернула
HTTP 200 для provisioned dashboard `lab6-messenger`.
+157
View File
@@ -0,0 +1,157 @@
package main
import (
"context"
"errors"
"highload/internal/config"
"highload/internal/database"
"highload/internal/database/sqlc"
"highload/internal/observability"
"highload/internal/repository/postgres"
"highload/internal/repository/redis"
"highload/internal/service"
"highload/internal/storage/s3"
httpdeliver "highload/internal/transport/http"
"highload/internal/transport/http/handler"
"highload/internal/transport/http/middleware"
"log/slog"
"net"
"net/http"
"os"
"os/signal"
"syscall"
"time"
)
func main() {
cfg, err := config.Load()
if err != nil {
slog.Error("invalid configuration", "error", err)
os.Exit(1)
}
sessionStore := cfg.Session.Store
if sessionStore != "redis" {
slog.Error("session store is "+sessionStore+"must redis", "error", nil)
os.Exit(1)
}
defer observability.Start(context.Background(), "messenger-api")()
databaseURL := cfg.Database.PostgresURL
pool, err := database.NewPostgres(context.Background(), databaseURL)
if err != nil {
slog.Error("postgres error", "error", err)
os.Exit(1)
}
defer pool.Close()
if os.Getenv("PG_POOL_DIAGNOSTICS") == "1" {
diagnosticsCtx, stopDiagnostics := context.WithCancel(context.Background())
defer stopDiagnostics()
go logPoolStats(diagnosticsCtx, pool)
}
queries := sqlc.New(pool)
redisAddr := cfg.Database.RedisAddr
redisClient, err := database.NewRedis(context.Background(), redisAddr)
if err != nil {
slog.Error("redis error", "error", err)
os.Exit(1)
}
defer redisClient.Close()
instanceID, err := os.Hostname()
if err != nil || instanceID == "" {
slog.Error("cannot get instance ID", "error", err)
os.Exit(1)
}
s3Storage, err := s3.NewStorage(context.Background(), &cfg.S3)
if err != nil {
slog.Error("cannot create new storage", "error", err)
os.Exit(1)
}
userRepo := postgres.NewUserRepository(queries)
chatRepo := postgres.NewChatRepository(queries, pool)
messageRepo := postgres.NewMessageRepository(queries, pool)
sessionRepo := redis.NewSessionRepository(redisClient)
attachmentRepo := postgres.NewAttachmentRepository(queries, pool, int32(cfg.JobMaxAttempts))
authService := service.NewAuthService(userRepo, sessionRepo, cfg.Session.TTL)
userService := service.NewUserService(userRepo)
chatService := service.NewChatService(chatRepo, userRepo, userService)
messageService := service.NewMessageService(messageRepo, chatService, userService)
attachmentService := service.NewAttachmentService(attachmentRepo, chatService, s3Storage)
healthHandler := handler.NewHealthHandler(pool, redisClient, s3Storage)
authHandler := handler.NewAuthHandler(authService)
userHandler := handler.NewUserHandler(userService)
chatHandler := handler.NewChatHandler(chatService)
messageHandler := handler.NewMessageHandler(messageService)
attachmentHandler := handler.NewAttachmentHandler(attachmentService)
mid := middleware.NewMiddleware(authService, instanceID)
router := httpdeliver.NewRouter(mid, authHandler, healthHandler, chatHandler, userHandler, messageHandler, attachmentHandler)
server := &http.Server{
Addr: net.JoinHostPort(cfg.HTTP.Host, cfg.HTTP.Port),
Handler: router.Handler(),
ReadHeaderTimeout: 5 * time.Second,
ReadTimeout: 10 * time.Second,
WriteTimeout: 10 * time.Second,
IdleTimeout: 30 * time.Second,
}
listener, err := net.Listen("tcp", server.Addr)
if err != nil {
slog.Error("server failed to start", "error", err)
return
}
slog.Info("server started", "addr", server.Addr)
serverErr := make(chan error, 1)
go func() {
err := server.Serve(listener)
if err != nil && !errors.Is(err, http.ErrServerClosed) {
serverErr <- err
slog.Error("server failed", "error", err)
}
}()
ctx, stop := signal.NotifyContext(
context.Background(),
os.Interrupt,
syscall.SIGTERM,
)
defer stop()
select {
case <-ctx.Done():
slog.Info("shutdown signal received")
case err := <-serverErr:
slog.Error("server failed", "error", err)
return
}
shutdownCtx, cancel := context.WithTimeout(
context.Background(),
15*time.Second,
)
defer cancel()
if err := server.Shutdown(shutdownCtx); err != nil {
slog.Error("shutdown failed", "error", err)
}
}
+29
View File
@@ -0,0 +1,29 @@
package main
import (
"context"
"log/slog"
"time"
"github.com/jackc/pgx/v5/pgxpool"
)
// logPoolStats is opt-in diagnostics for the isolated scaling experiment.
// It contains only aggregate counters, never connection strings or SQL values.
func logPoolStats(ctx context.Context, pool *pgxpool.Pool) {
ticker := time.NewTicker(time.Second)
defer ticker.Stop()
for {
select {
case <-ctx.Done():
return
case <-ticker.C:
s := pool.Stat()
slog.Info("pg_pool_stats", "max_conns", s.MaxConns(),
"acquired", s.AcquiredConns(), "idle", s.IdleConns(),
"acquire_count", s.AcquireCount(), "empty_count", s.EmptyAcquireCount(),
"empty_wait_ns", s.EmptyAcquireWaitTime().Nanoseconds(),
"canceled_count", s.CanceledAcquireCount())
}
}
}
+164
View File
@@ -0,0 +1,164 @@
package main
import (
"context"
"encoding/json"
"errors"
"flag"
"fmt"
"highload/internal/config"
"highload/internal/database"
"highload/internal/database/sqlc"
"highload/internal/repository/postgres"
"highload/internal/service"
"highload/internal/storage/s3"
"log/slog"
"os"
"os/signal"
"syscall"
"time"
"github.com/jackc/pgx/v5/pgconn"
)
type cleanupConfig struct {
once bool
dryRun bool
gracePeriod time.Duration
interval time.Duration
passTimeout time.Duration
reconcile bool
restoreID string
backupFile string
}
func main() {
var cfg cleanupConfig
flag.BoolVar(&cfg.once, "once", false, "Run once and exit")
flag.BoolVar(&cfg.dryRun, "dry-run", true, "Do not actually delete files")
flag.DurationVar(&cfg.gracePeriod, "grace-period", 10*time.Minute, "Grace period for old files")
flag.DurationVar(&cfg.interval, "interval", 5*time.Minute, "Interval between runs")
flag.DurationVar(&cfg.passTimeout, "pass-timeout", 2*time.Minute, "Timeout for a single pass")
flag.BoolVar(&cfg.reconcile, "reconcile", false, "Scan both stores once and print JSON observations")
flag.StringVar(&cfg.restoreID, "restore-id", "", "Restore the missing original for this attachment ID")
flag.StringVar(&cfg.backupFile, "backup-file", "", "Path to the original backup used for restore")
flag.Parse()
if cfg.gracePeriod <= 0 || cfg.interval <= 0 || cfg.passTimeout <= 0 {
slog.Error("all duration flags must be positive")
os.Exit(1)
}
if (cfg.restoreID == "") != (cfg.backupFile == "") || (cfg.reconcile && cfg.restoreID != "") {
slog.Error("use -reconcile alone, or supply both -restore-id and -backup-file")
os.Exit(1)
}
if err := run(cfg); err != nil {
slog.Error("attachment cleanup failed", "error", err)
os.Exit(1)
}
}
func run(cfg cleanupConfig) error {
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
defer stop()
databaseURL, s3Config, err := config.LoadAttachmentCleanup()
if err != nil {
return fmt.Errorf("load cleanup configuration: %w", err)
}
startupCtx, cancelStartup := context.WithTimeout(ctx, 10*time.Second)
defer cancelStartup()
pool, err := database.NewPostgres(startupCtx, databaseURL)
if err != nil {
var configErr *pgconn.ParseConfigError
if errors.As(err, &configErr) {
return errors.New("invalid PostgreSQL connection string")
}
return fmt.Errorf("connect to PostgreSQL: %w", err)
}
defer pool.Close()
s3Storage, err := s3.NewStorage(startupCtx, &s3Config)
if err != nil {
return fmt.Errorf("create S3 client: %w", err)
}
cancelStartup()
attachmentRepo := postgres.NewAttachmentRepository(sqlc.New(pool), pool)
options := &service.AttachmentCleanupOptions{
GracePeriod: cfg.gracePeriod,
DryRun: cfg.dryRun,
ListTimeout: 5 * time.Second,
}
attachmentCleanupService := service.NewAttachmentCleanupService(attachmentRepo, s3Storage, options)
if cfg.reconcile || cfg.restoreID != "" {
passCtx, cancel := context.WithTimeout(ctx, cfg.passTimeout)
defer cancel()
reconciler := service.NewAttachmentReconciler(attachmentRepo, s3Storage, *options)
start := time.Now()
report := struct {
DryRun bool `json:"dry_run"`
DurationMS int64 `json:"duration_ms"`
Complete bool `json:"complete"`
Orphans *service.OrphanReport `json:"orphan_scan,omitempty"`
Missing *service.MissingReport `json:"missing_scan,omitempty"`
Restore string `json:"restore,omitempty"`
}{DryRun: cfg.dryRun}
var actionErr error
if cfg.reconcile {
missing, err := reconciler.ScanMissing(passCtx)
report.Missing = &missing
actionErr = err
if err == nil {
orphans, err := attachmentCleanupService.ScanOrphans(passCtx)
report.Orphans = &orphans
actionErr = err
}
} else {
file, err := os.Open(cfg.backupFile)
if err != nil {
return fmt.Errorf("open backup original: %w", err)
}
defer file.Close()
report.Restore, actionErr = reconciler.Restore(passCtx, cfg.restoreID, file)
}
report.DurationMS = time.Since(start).Milliseconds()
report.Complete = actionErr == nil
return errors.Join(actionErr, json.NewEncoder(os.Stdout).Encode(report))
}
runPass := func() error {
passCtx, cancelPass := context.WithTimeout(ctx, cfg.passTimeout)
defer cancelPass()
return attachmentCleanupService.RunOnce(passCtx)
}
if cfg.once {
return runPass()
}
if err := runPass(); err != nil && ctx.Err() == nil {
slog.Error("attachment cleanup pass failed", "error", err)
}
ticker := time.NewTicker(cfg.interval)
defer ticker.Stop()
for {
select {
case <-ctx.Done():
return nil
case <-ticker.C:
if ctx.Err() != nil {
return nil
}
if err := runPass(); err != nil && ctx.Err() == nil {
slog.Error("attachment cleanup pass failed", "error", err)
}
}
}
}
+133
View File
@@ -0,0 +1,133 @@
package main
import (
"context"
"encoding/json"
"flag"
"fmt"
"log/slog"
"os"
"os/signal"
"strings"
"syscall"
"time"
"highload/internal/config"
"highload/internal/database"
"highload/internal/database/sqlc"
"highload/internal/pipeline"
"highload/internal/repository/postgres"
"highload/internal/storage/s3"
)
func main() {
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
defer stop()
if err := run(ctx, os.Args[1:]); err != nil {
slog.Error("attachment jobs command failed", "error", err)
os.Exit(1)
}
}
func run(ctx context.Context, args []string) error {
const usage = "usage: jobs failed | replay | upgrade | cleanup"
if len(args) == 0 {
return fmt.Errorf("%s", usage)
}
flags := flag.NewFlagSet(args[0], flag.ContinueOnError)
job := flags.String("job-id", "", "failed job ID")
request := flags.String("request-id", "", "unique operator request UUID")
operator := flags.String("operator", "", "operator ID")
reason := flags.String("reason", "", "replay reason")
extra := flags.Int("extra-attempts", 1, "budget for replay (1–5)")
attachment := flags.String("attachment-id", "", "attachment ID to upgrade")
version := flags.Int("to-version", 0, "target pipeline version (2)")
dry := flags.Bool("dry-run", false, "report eligible objects without deleting")
once := flags.Bool("once", false, "execute one cleanup pass")
if err := flags.Parse(args[1:]); err != nil {
return err
}
command := args[0]
if command != "failed" && command != "cleanup" && command != "replay" && command != "upgrade" {
return fmt.Errorf("%s", usage)
}
if command == "replay" && (*job == "" || *request == "" || strings.TrimSpace(*operator) == "" ||
strings.TrimSpace(*reason) == "" || *extra < 1 || *extra > 5) {
return fmt.Errorf("replay needs job-id, request-id, operator, reason and 1–5 extra-attempts")
}
if command == "upgrade" && (*attachment == "" || *version != 2) {
return fmt.Errorf("upgrade needs attachment-id and to-version=2")
}
attemptLimit := 5
if command == "upgrade" {
var err error
attemptLimit, err = config.EnvInt("JOB_MAX_ATTEMPTS", 5, 1, 20)
if err != nil {
return err
}
}
url, storageConfig, err := config.LoadAttachmentCleanup()
if err != nil {
return err
}
pool, err := database.NewPostgres(ctx, url)
if err != nil {
return fmt.Errorf("PostgreSQL unavailable")
}
defer pool.Close()
repo := postgres.NewAttachmentRepository(sqlc.New(pool), pool, int32(attemptLimit))
if command == "failed" {
dbCtx, cancel := context.WithTimeout(ctx, 5*time.Second)
defer cancel()
rows, err := repo.ListFailedJobs(dbCtx)
if err != nil {
return err
}
type failed struct {
JobID string `json:"job_id"`
AttachmentID string `json:"attachment_id"`
ErrorCode string `json:"error_code"`
Attempts int32 `json:"attempts"`
UpdatedAt time.Time `json:"updated_at"`
}
result := make([]failed, 0, len(rows))
for _, row := range rows {
result = append(result, failed{row.ID.String(), row.AttachmentID.String(), row.ErrorCode.String, row.AttemptsStarted, row.UpdatedAt.Time})
}
return json.NewEncoder(os.Stdout).Encode(result)
}
objects, err := s3.NewStorage(ctx, &storageConfig)
if err != nil {
return err
}
if command == "replay" {
changed, err := repo.ReplayFailed(ctx, *job, *request, *operator, *reason, int32(*extra), objects.Head)
if err != nil {
return err
}
return json.NewEncoder(os.Stdout).Encode(map[string]any{"job_id": *job, "replayed": changed})
}
if command == "upgrade" {
jobID, queued, err := repo.UpgradePipeline(ctx, *attachment, objects.Head)
if err != nil {
return err
}
return json.NewEncoder(os.Stdout).Encode(map[string]any{
"attachment_id": *attachment, "pipeline_version": *version, "job_id": jobID, "queued": queued,
})
}
cfg, err := config.LoadPipeline()
if err != nil {
return err
}
cleanup := pipeline.Cleanup{Repo: repo, Objects: objects, Grace: cfg.CleanupGrace, Retention: cfg.Retention, DryRun: *dry}
if *once {
count, err := cleanup.RunOnce(ctx)
if err != nil {
return err
}
return json.NewEncoder(os.Stdout).Encode(map[string]any{"objects": count, "dry_run": *dry})
}
cleanup.Serve(ctx, cfg.CleanupPeriod)
return nil
}
+254
View File
@@ -0,0 +1,254 @@
package main
import (
"encoding/json"
"errors"
"io"
"log/slog"
"os"
"strings"
"testing"
"time"
"highload/db/migrations"
"highload/internal/database/sqlc"
"github.com/jackc/pgx/v5"
"github.com/jackc/pgx/v5/pgtype"
"github.com/jackc/pgx/v5/pgxpool"
"github.com/pressly/goose/v3"
)
func TestAsyncAttachmentsMigrationAndJobTransitions(t *testing.T) {
db, ctx := migrationPostgresFixture(t)
provider, err := goose.NewProvider(goose.DialectPostgres, db, migrations.Files,
goose.WithDisableGlobalRegistry(true), goose.WithSlog(slog.New(slog.NewTextHandler(io.Discard, nil))))
if err != nil {
t.Fatal(err)
}
if _, err := provider.UpTo(ctx, 6); err != nil {
t.Fatal(err)
}
const original = "attachments/step-two-original"
_, err = db.ExecContext(ctx, `
INSERT INTO users (id, username, display_name, password_hash) VALUES
('00000000-0000-4000-8000-000000000001', 'step_two', 'Step two', 'synthetic');
INSERT INTO chats (id, title) VALUES ('00000000-0000-4000-8000-000000000002', 'Step two');
INSERT INTO chat_members VALUES
('00000000-0000-4000-8000-000000000002', '00000000-0000-4000-8000-000000000001');
INSERT INTO messages (id, chat_id, sender_id, text) VALUES
('00000000-0000-4000-8000-000000000003', '00000000-0000-4000-8000-000000000002',
'00000000-0000-4000-8000-000000000001', 'Old message');
INSERT INTO attachments (id, chat_id, owner_id, filename, object_key, content_type, size_bytes, sha256) VALUES
('00000000-0000-4000-8000-000000000004', '00000000-0000-4000-8000-000000000002',
'00000000-0000-4000-8000-000000000001', 'old.png', 'attachments/step-two-original',
'image/png', 123, repeat('a', 64));
INSERT INTO messages_attachments (message_id, attachment_id, chat_id, position) VALUES
('00000000-0000-4000-8000-000000000003', '00000000-0000-4000-8000-000000000004',
'00000000-0000-4000-8000-000000000002', 0);
`)
if err != nil {
t.Fatal(err)
}
if _, err := provider.UpTo(ctx, 7); err != nil {
t.Fatal(err)
}
if _, err := provider.Down(ctx); err != nil {
t.Fatal(err)
}
var preservedKey string
if err := db.QueryRowContext(ctx, "SELECT object_key FROM attachments JOIN messages_attachments ON attachment_id = id").Scan(&preservedKey); err != nil || preservedKey != original {
t.Fatalf("rollback lost the legacy attachment reference: %v", err)
}
if _, err := provider.UpTo(ctx, 7); err != nil {
t.Fatal(err)
}
if _, err := provider.UpTo(ctx, lab5MigrationCount); err != nil {
t.Fatal(err)
}
dsn, err := os.ReadFile(os.Getenv("DATABASE_URL_FILE"))
if err != nil {
t.Fatal("cannot read test connection configuration")
}
pool, err := pgxpool.New(ctx, string(dsn))
if err != nil {
t.Fatal("cannot configure test pool")
}
defer pool.Close()
q := sqlc.New(pool)
var jobID, attachmentID pgtype.UUID
var payload []byte
err = pool.QueryRow(ctx, `
SELECT j.id, a.id, o.payload
FROM messages_attachments AS m
JOIN attachments AS a ON a.id = m.attachment_id
JOIN attachment_jobs AS j ON j.attachment_id = a.id
JOIN outbox AS o ON o.job_id = j.id
WHERE a.status = 'queued' AND a.object_key = $1 AND a.staging_key = $1
AND o.dedup_key = 'initial/' || j.id
`, original).Scan(&jobID, &attachmentID, &payload)
if err != nil {
t.Fatalf("legacy attachment was not queued with its initial event: %v", err)
}
var event struct {
JobID string `json:"job_id"`
Attempt int `json:"attempt"`
CreatedAt time.Time `json:"created_at"`
}
if err := json.Unmarshal(payload, &event); err != nil || event.JobID != jobID.String() ||
event.Attempt != 1 || event.CreatedAt.IsZero() {
t.Fatalf("invalid backfill payload: %v", err)
}
start := make(chan struct{})
results := make(chan error, 2)
claim := sqlc.ClaimJobParams{
JobID: jobID, NewLeaseUntil: pgtype.Timestamptz{Time: time.Now().Add(time.Minute), Valid: true},
}
for range 2 {
go func() {
<-start
_, err := q.ClaimJob(ctx, claim)
results <- err
}()
}
close(start)
winners := 0
for range 2 {
switch err := <-results; {
case err == nil:
winners++
case errors.Is(err, pgx.ErrNoRows):
default:
t.Fatalf("concurrent claim failed: %v", err)
}
}
if winners != 1 {
t.Fatalf("gave the lease to %d workers", winners)
}
exec := func(query string, args ...any) {
t.Helper()
if _, err := pool.Exec(ctx, query, args...); err != nil {
t.Fatal(err)
}
}
exec("UPDATE attachment_jobs SET lease_until = clock_timestamp() - interval '1 second' WHERE id = $1", jobID)
for _, want := range []int{1, 0} {
recovery, err := q.EnqueueExpiredLeases(ctx, 10)
if err != nil || len(recovery) != want {
t.Fatalf("recovery events = %d, want %d: %v", len(recovery), want, err)
}
if want == 1 {
exec("UPDATE outbox SET sent_at = now() WHERE id = $1", recovery[0].ID)
}
}
row, err := q.ClaimJob(ctx, claim)
if err != nil || row.LeaseGeneration != 2 || row.AttemptsStarted != 2 {
t.Fatalf("expired lease was not replaced correctly: %v", err)
}
fail := sqlc.FailJobParams{JobID: jobID, LeaseGeneration: 1, ErrorCode: "invalid_image"}
if n, err := q.FailJob(ctx, fail); err != nil || n != 0 {
t.Fatalf("stale worker could fail attachment: rows=%d, err=%v", n, err)
}
retry := sqlc.ScheduleRetryParams{
JobID: jobID, LeaseGeneration: 2, ErrorCode: "s3_unavailable",
AvailableAt: pgtype.Timestamptz{Time: time.Now().Add(time.Minute), Valid: true},
}
if n, err := q.ScheduleRetry(ctx, retry); err != nil || n != 1 {
t.Fatalf("retry could not be scheduled: rows=%d, err=%v", n, err)
}
if _, err := q.ClaimJob(ctx, claim); !errors.Is(err, pgx.ErrNoRows) {
t.Fatalf("worker bypassed the retry delay: %v", err)
}
exec("UPDATE attachment_jobs SET next_attempt_at = now(), max_attempts = 3 WHERE id = $1", jobID)
row, err = q.ClaimJob(ctx, claim)
if err != nil || row.LeaseGeneration != 3 {
t.Fatalf("due retry could not claim job: %v", err)
}
exec("UPDATE attachment_jobs SET lease_until = clock_timestamp() - interval '1 second' WHERE id = $1", jobID)
if _, err := q.ClaimJob(ctx, claim); !errors.Is(err, pgx.ErrNoRows) {
t.Fatalf("worker bypassed the attempt limit: %v", err)
}
fail.LeaseGeneration = 3
fail.ErrorCode = "retry_exhausted"
if n, err := q.FailJob(ctx, fail); err != nil || n != 1 {
t.Fatalf("exhausted job could not be failed: rows=%d, err=%v", n, err)
}
var requestID pgtype.UUID
if err := requestID.Scan("00000000-0000-4000-8000-000000000005"); err != nil {
t.Fatal(err)
}
replay := sqlc.ReplayFailedParams{
JobID: jobID, RequestID: requestID, OperatorID: "operator", Reason: "test replay", ExtraAttempts: 1,
}
for _, want := range []int64{1, 0} {
if n, err := q.ReplayFailed(ctx, replay); err != nil || n != want {
t.Fatalf("replay changed %d rows, want %d: %v", n, want, err)
}
}
row, err = q.ClaimJob(ctx, claim)
if err != nil || row.AttemptsStarted != 4 || row.LeaseGeneration != 4 {
t.Fatalf("replay did not preserve attempt and lease counters: %v", err)
}
tx, err := pool.Begin(ctx)
if err != nil {
t.Fatal(err)
}
defer tx.Rollback(ctx)
locked := q.WithTx(tx)
if _, err := locked.LockActiveAttachmentJob(ctx, sqlc.LockActiveAttachmentJobParams{JobID: jobID, LeaseGeneration: 4}); err != nil {
t.Fatal(err)
}
complete := sqlc.CompleteJobParams{JobID: jobID, LeaseGeneration: 4, ObjectKey: "results/original"}
if n, err := locked.CompleteJob(ctx, complete); err != nil || n != 0 {
t.Fatalf("published result without variants: rows=%d, err=%v", n, err)
}
variant := sqlc.CreateAttachmentVariantParams{
AttachmentID: attachmentID, PipelineVersion: 1, Name: "thumbnail", ObjectKey: "results/thumb",
ContentType: "image/jpeg", SizeBytes: 123, Width: 16, Height: 10, Sha256: strings.Repeat("a", 64),
}
if err := locked.CreateAttachmentVariant(ctx, variant); err != nil {
t.Fatal(err)
}
if n, err := locked.CompleteJob(ctx, complete); err != nil || n != 0 {
t.Fatalf("published result without crop and optimized: rows=%d, err=%v", n, err)
}
variant.Name, variant.ObjectKey = "crop", "results/crop"
if err := locked.CreateAttachmentVariant(ctx, variant); err != nil {
t.Fatal(err)
}
variant.Name, variant.ObjectKey, variant.ContentType = "optimized", "results/webp", "image/webp"
if err := locked.CreateAttachmentVariant(ctx, variant); err != nil {
t.Fatal(err)
}
if n, err := locked.CompleteJob(ctx, complete); err != nil || n != 1 {
t.Fatalf("current owner could not publish result: rows=%d, err=%v", n, err)
}
if err := tx.Commit(ctx); err != nil {
t.Fatal(err)
}
if _, err := q.ClaimJob(ctx, claim); !errors.Is(err, pgx.ErrNoRows) {
t.Fatalf("worker claimed a ready job: %v", err)
}
published, err := q.GetPublishedOriginalKey(ctx, sqlc.GetPublishedOriginalKeyParams{
AttachmentID: attachmentID, PipelineVersion: 1,
})
if err != nil || published.String != complete.ObjectKey {
t.Fatalf("published original not recorded for its version: %v", err)
}
var attempts, events, replays int
err = pool.QueryRow(ctx, `
SELECT (SELECT count(*) FROM attachment_attempts WHERE job_id = $1 AND finished_at IS NOT NULL),
(SELECT count(*) FROM outbox WHERE job_id = $1),
(SELECT count(*) FROM attachment_replays WHERE job_id = $1)
`, jobID).Scan(&attempts, &events, &replays)
if err != nil || attempts != 4 || events != 4 || replays != 1 {
t.Fatalf("unexpected history: attempts=%d, events=%d, replays=%d: %v", attempts, events, replays, err)
}
}
+152
View File
@@ -0,0 +1,152 @@
package main
import (
"context"
"database/sql"
"errors"
"flag"
"log/slog"
"os"
"os/signal"
"strconv"
"strings"
"syscall"
"time"
"highload/db/migrations"
"github.com/jackc/pgx/v5"
"github.com/jackc/pgx/v5/pgconn"
"github.com/jackc/pgx/v5/stdlib"
"github.com/pressly/goose/v3"
"github.com/pressly/goose/v3/lock"
)
// Migration errors retain their cause for classification, but deliberately do
// not print connection strings, PostgreSQL DETAIL fields or raw SQL.
type migrationError struct {
stage string
cause error
}
func (e *migrationError) Error() string { return e.stage }
func (e *migrationError) Unwrap() error { return e.cause }
type boundedSessionLocker struct {
lock.SessionLocker
}
func (l boundedSessionLocker) SessionUnlock(ctx context.Context, conn *sql.Conn) error {
// Goose unlocks with a detached context; bound this cleanup separately.
ctx, cancel := context.WithTimeout(ctx, 5*time.Second)
defer cancel()
return l.SessionLocker.SessionUnlock(ctx, conn)
}
func main() {
command := flag.String("command", "up", "migration action: up or status")
timeout := flag.Duration("timeout", 2*time.Minute, "total migration/connection/lock budget")
flag.Parse()
logger := slog.New(slog.NewJSONHandler(os.Stdout, nil))
if flag.NArg() != 0 || (*command != "up" && *command != "status") || *timeout < time.Second {
logger.Error("invalid migration arguments", "allowed_commands", "up,status",
"minimum_timeout", "1s")
os.Exit(1)
}
signalCtx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
defer stop()
ctx, cancel := context.WithTimeout(signalCtx, *timeout)
defer cancel()
if err := run(ctx, *command, *timeout, logger); err != nil {
fields := []any{"stage", err.Error(), "timeout", errors.Is(err, context.DeadlineExceeded),
"cancelled", errors.Is(err, context.Canceled)}
var pgErr *pgconn.PgError
if errors.As(err, &pgErr) {
fields = append(fields, "sqlstate", pgErr.Code)
}
var partial *goose.PartialError
if errors.As(err, &partial) && partial.Failed != nil && partial.Failed.Source != nil {
fields = append(fields, "migration", partial.Failed.Source.Path,
"version", partial.Failed.Source.Version)
}
logger.Error("migration failed", fields...)
os.Exit(1)
}
}
func run(ctx context.Context, command string, timeout time.Duration, logger *slog.Logger) error {
fail := func(stage string, err error) error {
return &migrationError{stage: stage, cause: err}
}
path := os.Getenv("DATABASE_URL_FILE")
if path == "" {
path = "/run/secrets/database_url"
}
data, err := os.ReadFile(path)
if err != nil {
return fail("read database secret file", err)
}
url := strings.TrimSpace(string(data))
if url == "" {
return fail("database secret file is empty", errors.New("empty database URL"))
}
cfg, err := pgx.ParseConfig(url)
if err != nil {
return fail("parse database secret", err)
}
cfg.ConnectTimeout = 5 * time.Second
cfg.RuntimeParams["application_name"] = "highload-migrate"
cfg.RuntimeParams["lock_timeout"] = "10000"
cfg.RuntimeParams["statement_timeout"] = strconv.FormatInt(min(timeout.Milliseconds(), 2147483647), 10)
db := stdlib.OpenDB(*cfg)
db.SetMaxOpenConns(2)
db.SetMaxIdleConns(2)
db.SetConnMaxLifetime(5 * time.Minute)
defer db.Close()
if err := db.PingContext(ctx); err != nil {
return fail("connect to PostgreSQL", err)
}
locker, err := lock.NewPostgresSessionLocker(
lock.WithLockTimeout(1, 120),
lock.WithUnlockTimeout(1, 5),
)
if err != nil {
return fail("configure migration lock", err)
}
provider, err := goose.NewProvider(goose.DialectPostgres, db, migrations.Files,
goose.WithSessionLocker(boundedSessionLocker{locker}),
goose.WithDisableGlobalRegistry(true),
goose.WithSlog(logger),
// Goose verbose logging includes SQL statements. Log only selected
// migration metadata ourselves, even when a migration fails.
goose.WithVerbose(false),
)
if err != nil {
return fail("load embedded migrations", err)
}
if command == "status" {
statuses, err := provider.Status(ctx)
if err != nil {
return fail("read migration status", err)
}
for _, status := range statuses {
logger.Info("migration status", "file", status.Source.Path,
"version", status.Source.Version, "state", status.State)
}
return nil
}
started := time.Now()
results, err := provider.Up(ctx)
if err != nil {
return fail("apply pending migrations", err)
}
for _, result := range results {
logger.Info("migration applied", "file", result.Source.Path,
"version", result.Source.Version, "duration_seconds", result.Duration.Seconds())
}
logger.Info("migrations completed", "applied", len(results),
"duration_seconds", time.Since(started).Seconds())
return nil
}
+111
View File
@@ -0,0 +1,111 @@
package main
import (
"context"
"database/sql"
"errors"
"io"
"log/slog"
"os"
"path/filepath"
"strings"
"testing"
"time"
"highload/db/migrations"
"github.com/jackc/pgx/v5"
"github.com/jackc/pgx/v5/stdlib"
"github.com/pressly/goose/v3"
)
func TestConnectionSecretErrorsAreSafe(t *testing.T) {
for _, test := range []struct {
name string
content string
missing bool
}{
{name: "missing", missing: true},
{name: "empty", content: " \n\t"},
{name: "malformed", content: "postgres://synthetic:fixture-password@localhost/%zz"},
} {
t.Run(test.name, func(t *testing.T) {
path := filepath.Join(t.TempDir(), "database_url")
if !test.missing {
if err := os.WriteFile(path, []byte(test.content), 0600); err != nil {
t.Fatal("cannot create test secret")
}
}
t.Setenv("DATABASE_URL_FILE", path)
logger := slog.New(slog.NewJSONHandler(io.Discard, nil))
err := run(context.Background(), "up", time.Second, logger)
if err == nil {
t.Fatal("invalid connection secret was accepted")
}
if strings.Contains(err.Error(), "fixture-password") ||
strings.Contains(err.Error(), "postgres://") {
t.Fatal("migration error exposed a connection secret")
}
var classified *migrationError
if !errors.As(err, &classified) || classified.Unwrap() == nil {
t.Fatal("migration error lost its classification/cause")
}
})
}
}
func TestEmbeddedMigrationsAreVersionedAndCollectable(t *testing.T) {
cfg, err := pgx.ParseConfig("postgres://synthetic@localhost/synthetic?sslmode=disable")
if err != nil {
t.Fatal("cannot configure test driver")
}
db := stdlib.OpenDB(*cfg)
defer db.Close()
// NewProvider collects sources without connecting to PostgreSQL.
provider, err := goose.NewProvider(goose.DialectPostgres, db, migrations.Files)
if err != nil {
t.Fatal("embedded migration names/versions cannot be collected by Goose")
}
sources := provider.ListSources()
if len(sources) == 0 {
t.Fatal("migrator build contains no SQL")
}
var previous int64
for _, source := range sources {
if source.Version <= previous {
t.Fatal("embedded versions are not unique and increasing")
}
previous = source.Version
data, err := migrations.Files.ReadFile(source.Path)
if err != nil || !strings.Contains(string(data), "-- +goose Up") {
t.Fatal("embedded migration is unavailable or has no Up section")
}
}
}
type deadlineProbe struct {
deadline time.Time
cancel error
}
func (*deadlineProbe) SessionLock(context.Context, *sql.Conn) error { return nil }
func (p *deadlineProbe) SessionUnlock(ctx context.Context, _ *sql.Conn) error {
p.deadline, _ = ctx.Deadline()
p.cancel = ctx.Err()
return nil
}
func TestUnlockHasBudgetEvenWithDetachedContext(t *testing.T) {
original, cancel := context.WithCancel(context.Background())
cancel()
probe := &deadlineProbe{}
locker := boundedSessionLocker{probe}
started := time.Now()
if err := locker.SessionUnlock(context.WithoutCancel(original), nil); err != nil {
t.Fatal("unlock delegation failed")
}
if probe.cancel != nil || probe.deadline.IsZero() ||
probe.deadline.Before(started) || probe.deadline.After(started.Add(6*time.Second)) {
t.Fatal("detached cleanup did not receive a bounded, live context")
}
}
+212
View File
@@ -0,0 +1,212 @@
package main
import (
"bytes"
"context"
"database/sql"
"encoding/json"
"errors"
"log/slog"
"net/url"
"os"
"path/filepath"
"strings"
"sync"
"testing"
"time"
"github.com/google/uuid"
"github.com/jackc/pgx/v5"
"github.com/jackc/pgx/v5/pgconn"
"github.com/jackc/pgx/v5/stdlib"
)
// These tests need an explicitly supplied disposable PostgreSQL database.
// Each test creates and drops only its own randomly named schema.
func migrationPostgresFixture(t *testing.T) (*sql.DB, context.Context) {
t.Helper()
connectionURL := os.Getenv("MIGRATE_TEST_DATABASE_URL")
if connectionURL == "" {
t.Skip("set MIGRATE_TEST_DATABASE_URL to run PostgreSQL integration tests")
}
cfg, err := pgx.ParseConfig(connectionURL)
if err != nil {
t.Fatal("invalid test database configuration")
}
cfg.ConnectTimeout = 5 * time.Second
admin := stdlib.OpenDB(*cfg)
t.Cleanup(func() { admin.Close() })
ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
t.Cleanup(cancel)
schema := "migration_test_" + strings.ReplaceAll(uuid.NewString(), "-", "")
quoted := pgx.Identifier{schema}.Sanitize()
if _, err := admin.ExecContext(ctx, "CREATE SCHEMA "+quoted); err != nil {
t.Fatal("cannot create isolated migration test schema")
}
t.Cleanup(func() {
cleanupCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
if _, err := admin.ExecContext(cleanupCtx, "DROP SCHEMA "+quoted+" CASCADE"); err != nil {
t.Error("cannot remove isolated migration test schema")
}
})
cfg.RuntimeParams["search_path"] = schema
// ConnString retains the original parsed URL; changing RuntimeParams alone
// would not pass search_path to the separate migrator connection.
scopedURL, err := url.Parse(connectionURL)
if err != nil || (scopedURL.Scheme != "postgres" && scopedURL.Scheme != "postgresql") {
t.Fatal("MIGRATE_TEST_DATABASE_URL must be a PostgreSQL URL")
}
query := scopedURL.Query()
query.Set("search_path", schema)
scopedURL.RawQuery = query.Encode()
path := filepath.Join(t.TempDir(), "database_url")
if err := os.WriteFile(path, []byte(scopedURL.String()), 0600); err != nil {
t.Fatal("cannot write test connection secret")
}
t.Setenv("DATABASE_URL_FILE", path)
db := stdlib.OpenDB(*cfg)
t.Cleanup(func() { db.Close() })
return db, ctx
}
func migrationLogRecords(t *testing.T, output *bytes.Buffer) []map[string]any {
t.Helper()
var records []map[string]any
for _, line := range bytes.Split(bytes.TrimSpace(output.Bytes()), []byte("\n")) {
if len(line) == 0 {
continue
}
var record map[string]any
if err := json.Unmarshal(line, &record); err != nil {
t.Fatal("migrator emitted a non-JSON log record")
}
for key := range record {
switch key {
case "time", "level", "msg", "file", "version", "state", "applied", "duration_seconds":
default:
t.Fatalf("unexpected log field %q; SQL or connection details must not be logged", key)
}
}
if strings.Contains(string(line), "CREATE TABLE") || strings.Contains(string(line), "postgres://") {
t.Fatal("migration log contains SQL or a connection URL")
}
records = append(records, record)
}
return records
}
func completedMigrationCount(t *testing.T, output *bytes.Buffer) int {
t.Helper()
for _, record := range migrationLogRecords(t, output) {
if record["msg"] == "migrations completed" {
count, ok := record["applied"].(float64)
if !ok {
t.Fatal("migration completion log has no numeric applied count")
}
return int(count)
}
}
t.Fatal("successful migration run has no completion log")
return 0
}
const lab5MigrationCount = 8
func TestPostgresMigrationsRepeatWithoutChangingDataOrLoggingSQL(t *testing.T) {
db, ctx := migrationPostgresFixture(t)
var output bytes.Buffer
logger := slog.New(slog.NewJSONHandler(&output, nil))
if err := run(ctx, "up", time.Minute, logger); err != nil {
t.Fatal("migration of empty schema failed")
}
if count := completedMigrationCount(t, &output); count != lab5MigrationCount {
t.Fatalf("empty schema: applied %d migrations, want %d", count, lab5MigrationCount)
}
if _, err := db.ExecContext(ctx, "INSERT INTO users (username, display_name, password_hash) VALUES ($1, $2, $3)",
"migration_fixture", "Migration fixture", "synthetic-test-only-hash"); err != nil {
t.Fatal("cannot insert persistent migration fixture")
}
output.Reset()
if err := run(ctx, "up", time.Minute, logger); err != nil {
t.Fatal("repeat migration failed")
}
if count := completedMigrationCount(t, &output); count != 0 {
t.Fatalf("repeat run: applied %d migrations, want 0", count)
}
var users, versions int
if err := db.QueryRowContext(ctx, "SELECT count(*) FROM users WHERE username = $1", "migration_fixture").Scan(&users); err != nil || users != 1 {
t.Fatal("repeat migration lost or duplicated existing data")
}
if err := db.QueryRowContext(ctx, "SELECT count(*) FROM goose_db_version WHERE version_id > 0 AND is_applied").Scan(&versions); err != nil || versions != lab5MigrationCount {
t.Fatal("repeat migration changed version history")
}
output.Reset()
if err := run(ctx, "status", time.Minute, logger); err != nil {
t.Fatal("migration status failed")
}
records := migrationLogRecords(t, &output)
if len(records) != lab5MigrationCount {
t.Fatal("status did not report all seven migrations")
}
for _, record := range records {
if record["state"] != "applied" {
t.Fatal("status did not report applied migrations")
}
}
}
func TestPostgresConcurrentMigratorsApplyEachVersionOnce(t *testing.T) {
db, ctx := migrationPostgresFixture(t)
var outputs [2]bytes.Buffer
var runErrors [2]error
var workers sync.WaitGroup
start := make(chan struct{})
for i := range outputs {
workers.Add(1)
go func(i int) {
defer workers.Done()
<-start
runErrors[i] = run(ctx, "up", time.Minute, slog.New(slog.NewJSONHandler(&outputs[i], nil)))
}(i)
}
close(start)
workers.Wait()
var applied int
for i, err := range runErrors {
if err != nil {
t.Fatal("concurrent migration run failed")
}
applied += completedMigrationCount(t, &outputs[i])
}
if applied != lab5MigrationCount {
t.Fatalf("concurrent runners applied %d migrations in total, want %d", applied, lab5MigrationCount)
}
var versions, distinctVersions int
if err := db.QueryRowContext(ctx, "SELECT count(*), count(DISTINCT version_id) FROM goose_db_version WHERE version_id > 0 AND is_applied").Scan(&versions, &distinctVersions); err != nil || versions != lab5MigrationCount || distinctVersions != lab5MigrationCount {
t.Fatal("concurrent runners duplicated or lost version history")
}
}
func TestPostgresMigrationFailureRollsBackAndPreservesCause(t *testing.T) {
db, ctx := migrationPostgresFixture(t)
// The first migration creates users, then conflicts with this chats table.
if _, err := db.ExecContext(ctx, "CREATE TABLE chats (marker text)"); err != nil {
t.Fatal("cannot create conflicting test table")
}
var output bytes.Buffer
err := run(ctx, "up", time.Minute, slog.New(slog.NewJSONHandler(&output, nil)))
var pgErr *pgconn.PgError
if err == nil || err.Error() != "apply pending migrations" || !errors.As(err, &pgErr) || pgErr.Code != "42P07" {
t.Fatal("migration failure did not retain a safely reportable duplicate-table error")
}
migrationLogRecords(t, &output)
var usersAbsent bool
if err := db.QueryRowContext(ctx, "SELECT to_regclass('users') IS NULL").Scan(&usersAbsent); err != nil || !usersAbsent {
t.Fatal("failed migration left a partially applied users table")
}
var versions int
if err := db.QueryRowContext(ctx, "SELECT count(*) FROM goose_db_version WHERE version_id > 0 AND is_applied").Scan(&versions); err != nil || versions != 0 {
t.Fatal("failed migration was recorded as applied")
}
}
+167
View File
@@ -0,0 +1,167 @@
package main
import (
"context"
"database/sql"
"io"
"log/slog"
"testing"
"github.com/pressly/goose/v3"
"highload/db/migrations"
)
func sessionSchema(t *testing.T, db *sql.DB, ctx context.Context) []string {
t.Helper()
rows, err := db.QueryContext(ctx, `
SELECT 'column:'||attname||':'||format_type(atttypid,atttypmod)||':'||attnotnull||':'||
COALESCE(pg_get_expr(d.adbin,d.adrelid),'')
FROM pg_attribute a LEFT JOIN pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum
WHERE a.attrelid='sessions'::regclass AND a.attnum>0 AND NOT a.attisdropped
UNION ALL SELECT 'constraint:'||conname||':'||pg_get_constraintdef(oid)
FROM pg_constraint WHERE conrelid='sessions'::regclass
UNION ALL SELECT 'index:'||indexname||':'||indexdef
FROM pg_indexes WHERE schemaname=current_schema() AND tablename='sessions'
ORDER BY 1`)
if err != nil {
t.Fatal("cannot inspect restored session columns/defaults/constraints/indexes")
}
defer rows.Close()
var result []string
for rows.Next() {
var s string
if rows.Scan(&s) != nil {
t.Fatal("schema scan failed")
}
result = append(result, s)
}
if rows.Err() != nil || len(result) < 10 {
t.Fatal("incomplete session structure snapshot")
}
return result
}
func businessData(t *testing.T, db *sql.DB, ctx context.Context) []string {
t.Helper()
result := []string{}
for _, table := range []string{"users", "chats", "chat_members", "messages"} {
rows, err := db.QueryContext(ctx, "SELECT row_to_json(t)::text FROM "+table+" t ORDER BY row_to_json(t)::text")
if err != nil {
t.Fatal("business table disappeared during migration")
}
count := 0
for rows.Next() {
var value string
if rows.Scan(&value) != nil {
t.Fatal("data scan failed")
}
result = append(result, table+value)
count++
}
err = rows.Err()
rows.Close()
if err != nil || count == 0 {
t.Fatal("business fixture missing during migration")
}
}
return result
}
func equalStrings(a, b []string) bool {
if len(a) != len(b) {
return false
}
for i := range a {
if a[i] != b[i] {
return false
}
}
return true
}
func TestPostgresMigrationThreeDownTwoDownOneAndReapply(t *testing.T) {
db, ctx := migrationPostgresFixture(t)
provider, err := goose.NewProvider(goose.DialectPostgres, db, migrations.Files,
goose.WithDisableGlobalRegistry(true), goose.WithSlog(slog.New(slog.NewTextHandler(io.Discard, nil))))
if err != nil {
t.Fatal("cannot load actual embedded migrations")
}
if _, err = provider.UpTo(ctx, 1); err != nil {
t.Fatal("cannot apply version 1")
}
v1 := sessionSchema(t, db, ctx)
if _, err = db.ExecContext(ctx, `
INSERT INTO users(id,username,display_name,password_hash,created_at) VALUES
('00000000-0000-4000-8000-000000000001','roundtrip_user','Roundtrip','synthetic-hash','2026-01-01T00:00:00Z');
INSERT INTO chats(id,title,created_at) VALUES
('00000000-0000-4000-8000-000000000002','Roundtrip','2026-01-01T00:00:00Z');
INSERT INTO chat_members VALUES('00000000-0000-4000-8000-000000000002','00000000-0000-4000-8000-000000000001');
INSERT INTO messages(id,chat_id,sender_id,text,created_at) VALUES
('00000000-0000-4000-8000-000000000003','00000000-0000-4000-8000-000000000002','00000000-0000-4000-8000-000000000001','Keep this message','2026-01-01T00:00:00Z');
INSERT INTO sessions VALUES(decode(repeat('ab',32),'hex'),'00000000-0000-4000-8000-000000000001','2026-01-01T00:00:00Z','2026-01-02T00:00:00Z',NULL);
`); err != nil {
t.Fatal("cannot create migration preservation fixtures")
}
original := businessData(t, db, ctx)
if _, err = provider.UpTo(ctx, 2); err != nil {
t.Fatal("cannot apply version 2")
}
v2 := sessionSchema(t, db, ctx)
assertVersion := func(want int64) {
t.Helper()
got, e := provider.GetDBVersion(ctx)
if e != nil || got != want {
t.Fatalf("want migration version %d, got %d", want, got)
}
if !equalStrings(original, businessData(t, db, ctx)) {
t.Fatal("migration changed unrelated users/chats/members/messages")
}
}
assertAbsent := func() {
t.Helper()
var absent bool
if db.QueryRowContext(ctx, "SELECT to_regclass('sessions') IS NULL").Scan(&absent) != nil || !absent {
t.Fatal("version 3 must remove SQL sessions")
}
}
if _, err = provider.Up(ctx); err != nil {
t.Fatal("cannot migrate to latest version")
}
assertVersion(lab5MigrationCount)
assertAbsent()
if _, err = provider.DownTo(ctx, 2); err != nil {
t.Fatal("latest -> 2 failed")
}
assertVersion(2)
if !equalStrings(v2, sessionSchema(t, db, ctx)) {
t.Fatal("3 -> 2 did not restore exact version-2 session columns, defaults, keys, FK, CHECK and indexes")
}
var count int
if db.QueryRowContext(ctx, "SELECT count(*) FROM sessions").Scan(&count) != nil || count != 0 {
t.Fatal("rollback must restore an empty sessions table; deleted credentials cannot be resurrected")
}
if _, err = db.ExecContext(ctx, `INSERT INTO sessions(token_hash,user_id,expires_at) VALUES(decode(repeat('cd',32),'hex'),'00000000-0000-4000-8000-000000000001',now()+interval '1 hour')`); err != nil {
t.Fatal("restored table/default/FK cannot accept a valid session")
}
for _, statement := range []string{
`INSERT INTO sessions(token_hash,user_id,expires_at) VALUES(decode(repeat('ef',32),'hex'),'00000000-0000-4000-8000-000000000099',now()+interval '1 hour')`,
`INSERT INTO sessions(token_hash,user_id,expires_at) VALUES(decode(repeat('ef',32),'hex'),'00000000-0000-4000-8000-000000000001',now()-interval '1 hour')`,
`INSERT INTO sessions(token_hash,user_id,expires_at) VALUES(decode(repeat('cd',32),'hex'),'00000000-0000-4000-8000-000000000001',now()+interval '1 hour')`,
} {
if _, err = db.ExecContext(ctx, statement); err == nil {
t.Fatal("restored table lost FK, expiry CHECK or primary key")
}
}
if _, err = provider.DownTo(ctx, 1); err != nil {
t.Fatal("2 -> 1 failed")
}
assertVersion(1)
if !equalStrings(v1, sessionSchema(t, db, ctx)) {
t.Fatal("2 -> 1 did not restore original sessions structure without created_at default")
}
if _, err = provider.Up(ctx); err != nil {
t.Fatal("1 -> latest reapplication failed")
}
assertVersion(lab5MigrationCount)
assertAbsent()
}
+179
View File
@@ -0,0 +1,179 @@
package main
import (
"context"
"encoding/json"
"fmt"
"log/slog"
"os"
"os/signal"
"syscall"
"time"
"highload/internal/broker"
"highload/internal/config"
"highload/internal/database"
"highload/internal/database/sqlc"
"highload/internal/fault"
"highload/internal/observability"
"highload/internal/repository/postgres"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/trace"
)
func main() {
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
defer stop()
if err := run(ctx); err != nil {
slog.Error("relay startup failed", "error", err)
os.Exit(1)
}
}
func run(ctx context.Context) error {
cfg, err := config.LoadPipeline()
if err != nil {
return err
}
defer observability.Start(ctx, "messenger-relay")()
if cfg.BrokerURL == "" {
return fmt.Errorf("BROKER_URL_FILE must contain an AMQP URL")
}
url, err := config.LoadPostgresURL()
if err != nil {
return err
}
pool, err := database.NewPostgres(ctx, url)
if err != nil {
return fmt.Errorf("PostgreSQL unavailable")
}
defer pool.Close()
queries := sqlc.New(pool)
repo := postgres.NewAttachmentRepository(queries, pool)
sampled := make(chan struct{})
go func() {
defer close(sampled)
sampleQueue(ctx, queries, observability.WatchQueue(broker.Queue))
}()
recovered := make(chan struct{})
go func() {
defer close(recovered)
for ctx.Err() == nil {
if n, err := repo.EnqueueExpiredLeases(ctx, 32); err != nil && ctx.Err() == nil {
slog.Warn("lease recovery failed", "error", err)
} else if n > 0 {
slog.Info("expired leases enqueued", "count", n)
}
select {
case <-ctx.Done():
case <-time.After(250 * time.Millisecond):
}
}
}()
for ctx.Err() == nil {
if err := publish(ctx, repo, cfg.BrokerURL); err != nil && ctx.Err() == nil {
slog.Warn("relay reconnecting", "error", err)
}
select {
case <-ctx.Done():
case <-time.After(500 * time.Millisecond):
}
}
<-recovered
<-sampled
return nil
}
func publish(ctx context.Context, repo *postgres.AttachmentRepository, url string) error {
conn, ch, err := broker.Connect(url)
if err != nil {
return err
}
defer broker.Close(conn)
publisher, err := broker.NewPublisher(ch)
if err != nil {
return err
}
for ctx.Err() == nil {
sent, err := repo.PublishPending(ctx, func(txCtx context.Context, event sqlc.Outbox) error {
if err := fault.Pause(txCtx, "before_publish", event.JobID.String()); err != nil {
return err
}
if err := publishEvent(txCtx, publisher, event); err != nil {
return err
}
return fault.Pause(txCtx, "after_publish", event.JobID.String())
})
if err != nil {
return err
}
if !sent {
if conn.IsClosed() {
return fmt.Errorf("RabbitMQ connection closed")
}
select {
case <-ctx.Done():
case <-time.After(150 * time.Millisecond):
}
}
}
return nil
}
func sampleQueue(ctx context.Context, queries *sqlc.Queries, observe func(observability.QueueSample)) {
ticker := time.NewTicker(2 * time.Second)
defer ticker.Stop()
for ctx.Err() == nil {
sampledAt := time.Now()
queryCtx, cancel := context.WithTimeout(ctx, 800*time.Millisecond)
row, err := queries.GetQueueStats(queryCtx)
cancel()
if err == nil {
observe(observability.QueueSample{Depth: row.Depth, OldestSeconds: row.OldestSeconds, At: sampledAt})
}
select {
case <-ctx.Done():
case <-ticker.C:
}
}
}
func publishEvent(ctx context.Context, publisher *broker.Publisher, event sqlc.Outbox) (err error) {
var payload map[string]json.RawMessage
if err := json.Unmarshal(event.Payload, &payload); err != nil {
return err
}
if payload == nil {
return fmt.Errorf("outbox payload must be an object")
}
var traceparent, tracestate string
_ = json.Unmarshal(payload["traceparent"], &traceparent)
_ = json.Unmarshal(payload["tracestate"], &tracestate)
ctx = observability.JobContext(ctx, traceparent, tracestate)
ctx, span := observability.Tracer.Start(ctx, "rabbitmq.publish",
trace.WithSpanKind(trace.SpanKindProducer), trace.WithAttributes(
attribute.String("messaging.system", "rabbitmq"),
attribute.String("messaging.destination.name", broker.Queue),
attribute.String("messaging.operation.name", "publish"),
attribute.String("messaging.message.id", event.JobID.String()),
))
defer func() { observability.End(span, err) }()
carrier := observability.Carrier(ctx)
for _, key := range []string{"traceparent", "tracestate"} {
if value := carrier.Get(key); value != "" {
payload[key], _ = json.Marshal(value)
} else {
delete(payload, key)
}
}
body, err := json.Marshal(payload)
if err != nil {
return err
}
confirmCtx, cancel := context.WithTimeout(ctx, 5*time.Second)
defer cancel()
return publisher.Publish(confirmCtx, event.ID.String(), body)
}
+61
View File
@@ -0,0 +1,61 @@
package main
import (
"context"
"fmt"
"log/slog"
"os"
"os/signal"
"sync"
"syscall"
"highload/internal/config"
"highload/internal/database"
"highload/internal/database/sqlc"
"highload/internal/observability"
"highload/internal/pipeline"
"highload/internal/repository/postgres"
"highload/internal/storage/s3"
)
func main() {
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
defer stop()
if err := run(ctx); err != nil {
slog.Error("worker startup failed", "error", err)
os.Exit(1)
}
}
func run(ctx context.Context) error {
cfg, err := config.LoadPipeline()
if err != nil {
return err
}
defer observability.Start(ctx, "messenger-worker")()
observability.WatchWorker(cfg.Concurrency)
if cfg.BrokerURL == "" {
return fmt.Errorf("BROKER_URL_FILE must contain an AMQP URL")
}
url, storageConfig, err := config.LoadAttachmentCleanup()
if err != nil {
return err
}
pool, err := database.NewPostgres(ctx, url)
if err != nil {
return fmt.Errorf("PostgreSQL unavailable")
}
defer pool.Close()
objects, err := s3.NewStorage(ctx, &storageConfig)
if err != nil {
return fmt.Errorf("S3 configuration invalid")
}
worker := pipeline.Worker{Repo: postgres.NewAttachmentRepository(sqlc.New(pool), pool), Objects: objects, Config: cfg}
slog.Info("worker started", "concurrency", cfg.Concurrency, "prefetch_per_consumer", cfg.Prefetch)
var lanes sync.WaitGroup
for range cfg.Concurrency {
lanes.Go(func() { worker.Serve(ctx) })
}
lanes.Wait()
return nil
}
+376
View File
@@ -0,0 +1,376 @@
name: highload-lab6
services:
api:
build:
context: .
dockerfile: Dockerfile
scale: 2
cpus: "2.0"
mem_limit: 512m
expose:
- "8080"
env_file:
- .env
environment:
SESSION_STORE: redis
REDIS_ADDR: redis:6379
OTEL_SERVICE_NAME: messenger-api
secrets:
- database_url
- s3_access_key_id
- s3_secret_access_key
depends_on:
postgres:
condition: service_healthy
redis:
condition: service_healthy
migrate:
condition: service_completed_successfully
s3-init:
condition: service_completed_successfully
restart: unless-stopped
stop_grace_period: 20s
read_only: true
tmpfs:
- /tmp:size=32m,mode=1777
cap_drop:
- ALL
security_opt:
- no-new-privileges:true
attachment-cleanup:
build:
context: .
dockerfile: Dockerfile
target: cleanup
command: ["cleanup", "-dry-run=${LAB6_CLEANUP_DRY_RUN:-false}"]
cpus: "0.5"
mem_limit: 128m
env_file:
- .env
environment:
BROKER_URL_FILE: ""
secrets:
- database_url
- s3_access_key_id
- s3_secret_access_key
depends_on:
postgres:
condition: service_healthy
migrate:
condition: service_completed_successfully
s3-init:
condition: service_completed_successfully
restart: unless-stopped
stop_grace_period: 20s
read_only: true
cap_drop:
- ALL
security_opt:
- no-new-privileges:true
relay:
build:
context: .
dockerfile: Dockerfile
target: relay
env_file: [.env]
environment:
BROKER_URL_FILE: /run/secrets/broker_url
OTEL_SERVICE_NAME: messenger-relay
secrets: [database_url, broker_url]
depends_on:
migrate:
condition: service_completed_successfully
rabbitmq:
condition: service_healthy
cpus: "0.5"
mem_limit: 128m
read_only: true
cap_drop: [ALL]
security_opt: [no-new-privileges:true]
restart: unless-stopped
worker:
build:
context: .
dockerfile: Dockerfile.worker
env_file: [.env]
environment:
BROKER_URL_FILE: /run/secrets/broker_url
OTEL_SERVICE_NAME: messenger-worker
secrets: [database_url, broker_url, s3_access_key_id, s3_secret_access_key]
depends_on:
migrate:
condition: service_completed_successfully
s3-init:
condition: service_completed_successfully
rabbitmq:
condition: service_healthy
cpus: "2.0"
mem_limit: 1536m
read_only: true
tmpfs:
- /tmp:size=768m,mode=1777
cap_drop: [ALL]
security_opt: [no-new-privileges:true]
restart: unless-stopped
stop_grace_period: 15s
rabbitmq:
image: rabbitmq:4.2.5-alpine@sha256:d4a3806070111cc69130a6542aa4049b612789852ef11fc3862abc94631b2572
hostname: rabbitmq
entrypoint: ["/bin/sh", "-ec"]
command:
- |
export RABBITMQ_DEFAULT_PASS="$$(cat /run/secrets/rabbitmq_password)"
exec docker-entrypoint.sh rabbitmq-server
environment:
RABBITMQ_DEFAULT_USER: messenger
RABBITMQ_DEFAULT_VHOST: messenger
secrets: [rabbitmq_password]
volumes:
- rabbitmq_data:/var/lib/rabbitmq
healthcheck:
test: ["CMD", "rabbitmq-diagnostics", "-q", "check_running"]
interval: 5s
timeout: 10s
retries: 12
start_period: 60s
mem_limit: 768m
cpus: "1.0"
restart: unless-stopped
stop_grace_period: 30s
lb:
image: haproxy:3.2.24-alpine@sha256:1aca32ee932b96d2a75f1882909325839dd23c138ba9c6fd26e4d88179f30f13
cpus: "1.0"
mem_limit: 256m
ports:
- "127.0.0.1:${LAB6_HTTP_PORT:-8086}:8080"
volumes:
- ./infra/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro
depends_on:
api:
condition: service_healthy
restart: unless-stopped
read_only: true
cap_drop:
- ALL
security_opt:
- no-new-privileges:true
migrate:
build:
context: .
dockerfile: Dockerfile.migrate
cpus: "1.0"
mem_limit: 256m
environment:
DATABASE_URL_FILE: /run/secrets/database_url
secrets:
- database_url
depends_on:
postgres:
condition: service_healthy
restart: "no"
stop_grace_period: 20s
read_only: true
cap_drop:
- ALL
security_opt:
- no-new-privileges:true
postgres:
image: postgres:18@sha256:5a5a84b19854a9ffaa54082c166ff4ec27473a361e496e5ea167f298f2da9722
cpus: "2.0"
mem_limit: 512m
environment:
POSTGRES_DB: messenger
POSTGRES_USER: messenger
POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password
secrets:
- postgres_password
ports:
- "127.0.0.1:${LAB6_POSTGRES_PORT:-15486}:5432"
volumes:
- postgres_data:/var/lib/postgresql
healthcheck:
test: ["CMD-SHELL", "pg_isready -h 127.0.0.1 -U messenger -d messenger"]
interval: 5s
timeout: 5s
retries: 5
restart: unless-stopped
stop_grace_period: 60s
redis:
image: redis:7.4-alpine@sha256:858f009f9709ce576febc734aa78b8f6d624b82571f9ddb6bda4377c833b3499
cpus: "0.5"
mem_limit: 128m
restart: unless-stopped
command:
- redis-server
- --appendonly
- "yes"
- --appendfsync
- everysec
volumes:
- redis_data:/data
healthcheck:
test: ["CMD", "redis-cli", "ping"]
interval: 5s
timeout: 3s
retries: 5
s3:
image: rustfs/rustfs:1.0.4@sha256:ed9210d6411d77ffceb70d94b964a3cb7d225983368bc25bebe01e995293dcc2
cpus: "1.0"
mem_limit: 512m
environment:
RUSTFS_VOLUMES: /data
RUSTFS_ACCESS_KEY_FILE: /run/secrets/rustfs_root_access_key
RUSTFS_SECRET_KEY_FILE: /run/secrets/rustfs_root_secret_key
RUSTFS_CONSOLE_ENABLE: "false"
secrets:
- rustfs_root_access_key
- rustfs_root_secret_key
volumes:
- rustfs_data:/data
healthcheck:
test: ["CMD-SHELL", "curl -fsS http://127.0.0.1:9000/health/ready >/dev/null"]
interval: 5s
timeout: 3s
retries: 12
start_period: 30s
restart: unless-stopped
cap_drop:
- ALL
security_opt:
- no-new-privileges:true
s3-init:
image: rustfs/rc:v0.1.30@sha256:1296510ec27800668d26a8bdca6251e587086187a9368c7b80134a169e8a3bb8
cpus: "0.5"
mem_limit: 128m
entrypoint: ["/bin/sh", "/bootstrap.sh"]
environment:
HOME: /tmp
S3_BUCKET: messenger-attachments
S3_REGION: us-east-1
secrets:
- rustfs_root_access_key
- rustfs_root_secret_key
- s3_access_key_id
- s3_secret_access_key
volumes:
- ./infra/s3/bootstrap.sh:/bootstrap.sh:ro
depends_on:
s3:
condition: service_healthy
restart: "no"
read_only: true
tmpfs:
- /tmp:uid=100,gid=101,mode=0700
cap_drop:
- ALL
security_opt:
- no-new-privileges:true
otel-collector:
image: otel/opentelemetry-collector-contrib:0.131.0
command: ["--config=/etc/otelcol-contrib/config.yaml"]
environment:
GOMEMLIMIT: 120MiB
volumes:
- ./infra/otel/collector.yaml:/etc/otelcol-contrib/config.yaml:ro
mem_limit: 256m
cpus: "0.75"
restart: unless-stopped
prometheus:
image: prom/prometheus:v3.5.0
command:
- --config.file=/etc/prometheus/prometheus.yml
- --storage.tsdb.path=/prometheus
- --storage.tsdb.retention.time=24h
- --storage.tsdb.retention.size=512MB
ports: ["127.0.0.1:${LAB6_PROMETHEUS_PORT:-9096}:9090"]
volumes:
- ./infra/otel/prometheus.yaml:/etc/prometheus/prometheus.yml:ro
- ./infra/otel/alerts.yaml:/etc/prometheus/alerts.yaml:ro
- prometheus_data:/prometheus
mem_limit: 256m
cpus: "0.75"
restart: unless-stopped
loki:
image: grafana/loki:3.5.0
command: ["-config.file=/etc/loki/lab.yaml"]
ports: ["127.0.0.1:${LAB6_LOKI_PORT:-3106}:3100"]
volumes:
- ./infra/otel/loki.yaml:/etc/loki/lab.yaml:ro
- loki_data:/loki
mem_limit: 384m
cpus: "0.75"
restart: unless-stopped
tempo:
image: grafana/tempo:2.8.1
command: ["-config.file=/etc/tempo/lab.yaml"]
ports: ["127.0.0.1:${LAB6_TEMPO_PORT:-3206}:3200"]
volumes:
- ./infra/otel/tempo.yaml:/etc/tempo/lab.yaml:ro
- tempo_data:/var/tempo
mem_limit: 384m
cpus: "0.75"
restart: unless-stopped
grafana:
image: grafana/grafana:12.3.1
environment:
GF_AUTH_ANONYMOUS_ENABLED: "true"
GF_AUTH_ANONYMOUS_ORG_ROLE: Viewer
GF_AUTH_BASIC_ENABLED: "false"
GF_AUTH_DISABLE_LOGIN_FORM: "true"
GF_SECURITY_DISABLE_INITIAL_ADMIN_CREATION: "true"
GF_ANALYTICS_REPORTING_ENABLED: "false"
GF_ANALYTICS_CHECK_FOR_UPDATES: "false"
GF_ANALYTICS_CHECK_FOR_PLUGIN_UPDATES: "false"
GF_PLUGINS_PREINSTALL_DISABLED: "true"
ports: ["127.0.0.1:${LAB6_GRAFANA_PORT:-3006}:3000"]
volumes:
- ./infra/otel/grafana/provisioning:/etc/grafana/provisioning:ro
- ./infra/otel/grafana/dashboards:/var/lib/grafana/dashboards:ro
- grafana_data:/var/lib/grafana
mem_limit: 256m
cpus: "0.75"
restart: unless-stopped
secrets:
broker_url:
file: ./secrets/broker_url
rabbitmq_password:
file: ./secrets/rabbitmq_password
database_url:
file: ./secrets/database_url
postgres_password:
file: ./secrets/postgres_password
rustfs_root_access_key:
file: ./secrets/rustfs_root_access_key
rustfs_root_secret_key:
file: ./secrets/rustfs_root_secret_key
s3_access_key_id:
file: ./secrets/s3_access_key_id
s3_secret_access_key:
file: ./secrets/s3_secret_access_key
volumes:
rabbitmq_data:
postgres_data:
redis_data:
rustfs_data:
prometheus_data:
loki_data:
tempo_data:
grafana_data:
+79
View File
@@ -0,0 +1,79 @@
-- +goose Up
CREATE TABLE users
(
id uuid PRIMARY KEY NOT NULL DEFAULT gen_random_uuid(),
username VARCHAR(32) UNIQUE NOT NULL,
display_name VARCHAR(100) NOT NULL,
password_hash TEXT NOT NULL,
created_at TIMESTAMPTZ NOT NULL,
CONSTRAINT users_username_format_check CHECK (username COLLATE "C" ~ '^[a-z0-9_]{3,32}$'),
CONSTRAINT users_display_name_length_check CHECK (char_length(display_name) BETWEEN 1 AND 100),
CONSTRAINT users_display_name_not_blank_check CHECK (char_length(btrim(display_name)) > 0),
CONSTRAINT users_password_hash_not_empty_check CHECK (char_length(password_hash) > 0)
);
CREATE TABLE chats
(
id UUID PRIMARY KEY NOT NULL DEFAULT gen_random_uuid(),
title VARCHAR(100) NOT NULL,
created_at TIMESTAMPTZ NOT NULL,
CONSTRAINT chats_title_length_check CHECK (char_length(title) BETWEEN 1 AND 100 ),
CONSTRAINT chats_title_not_blank_check CHECK (char_length(btrim(title)) > 0)
);
CREATE TABLE chat_members
(
chat_id UUID NOT NULL REFERENCES chats (id),
user_id UUID NOT NULL REFERENCES users (id),
PRIMARY KEY (chat_id, user_id)
);
CREATE INDEX idx_user_chat_id ON chat_members (user_id, chat_id);
CREATE TABLE messages
(
id UUID PRIMARY KEY NOT NULL DEFAULT gen_random_uuid(),
chat_id UUID NOT NULL,
sender_id UUID NOT NULL,
text TEXT NOT NULL,
created_at TIMESTAMPTZ NOT NULL,
FOREIGN KEY (chat_id, sender_id)
REFERENCES chat_members (chat_id, user_id),
CONSTRAINT messages_text_length_check CHECK (char_length(text) BETWEEN 1 AND 4000),
CONSTRAINT messages_text_not_blank_check CHECK (char_length(btrim(text)) > 0)
);
CREATE INDEX idx_messages_chat_created_id ON messages (chat_id, created_at, id);
CREATE TABLE sessions
(
token_hash BYTEA PRIMARY KEY NOT NULL,
user_id UUID NOT NULL REFERENCES users (id),
created_at TIMESTAMPTZ NOT NULL,
expires_at TIMESTAMPTZ NOT NULL,
revoked_at TIMESTAMPTZ NULL,
CONSTRAINT sessions_expiry_after_creation_check CHECK (expires_at > created_at)
);
CREATE INDEX idx_session_user_id ON sessions (user_id);
-- +goose Down
DROP TABLE IF EXISTS sessions;
DROP TABLE IF EXISTS messages;
DROP TABLE IF EXISTS chat_members;
DROP TABLE IF EXISTS chats;
DROP TABLE IF EXISTS users;
@@ -0,0 +1,20 @@
-- +goose Up
ALTER TABLE users
ALTER COLUMN created_at SET DEFAULT now();
ALTER TABLE chats
ALTER COLUMN created_at SET DEFAULT now();
ALTER TABLE messages
ALTER COLUMN created_at SET DEFAULT now();
ALTER TABLE sessions
ALTER COLUMN created_at SET DEFAULT now();
-- +goose Down
ALTER TABLE sessions
ALTER COLUMN created_at DROP DEFAULT;
ALTER TABLE messages
ALTER COLUMN created_at DROP DEFAULT;
ALTER TABLE chats
ALTER COLUMN created_at DROP DEFAULT;
ALTER TABLE users
ALTER COLUMN created_at DROP DEFAULT;
@@ -0,0 +1,17 @@
-- +goose Up
DROP TABLE IF EXISTS sessions;
-- +goose Down
-- Restore the schema at version 2; deleted session data cannot be recovered.
CREATE TABLE sessions
(
token_hash BYTEA PRIMARY KEY NOT NULL,
user_id UUID NOT NULL REFERENCES users (id),
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
expires_at TIMESTAMPTZ NOT NULL,
revoked_at TIMESTAMPTZ NULL,
CONSTRAINT sessions_expiry_after_creation_check CHECK (expires_at > created_at)
);
CREATE INDEX idx_session_user_id ON sessions (user_id);
@@ -0,0 +1,23 @@
-- +goose Up
CREATE TABLE message_idempotency
(
user_id UUID NOT NULL REFERENCES users(id),
chat_id UUID NOT NULL REFERENCES chats(id),
idempotency_key VARCHAR(128) NOT NULL,
message_id UUID NOT NULL REFERENCES messages(id),
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
expires_at TIMESTAMPTZ NOT NULL DEFAULT (now() + interval '24 hours'),
PRIMARY KEY (user_id, chat_id, idempotency_key),
UNIQUE (message_id),
CONSTRAINT message_idempotency_key_format_check
CHECK (idempotency_key COLLATE "C" ~ '^[A-Za-z0-9._:-]{1,128}$'),
CONSTRAINT message_idempotency_expiry_check
CHECK (expires_at > created_at)
);
-- +goose Down
DROP TABLE IF EXISTS message_idempotency;
+54
View File
@@ -0,0 +1,54 @@
-- +goose Up
CREATE TABLE attachments
(
id UUID PRIMARY KEY NOT NULL DEFAULT gen_random_uuid(),
chat_id UUID NOT NULL,
owner_id UUID NOT NULL,
filename text NOT NULL,
object_key text UNIQUE NOT NULL,
content_type VARCHAR(10) NOT NULL,
size_bytes bigint NOT NULL,
sha256 text NOT NULL,
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
updated_at TIMESTAMPTZ NOT NULL DEFAULT now(),
CONSTRAINT fk_attachments_chat_member FOREIGN KEY (chat_id, owner_id)
REFERENCES chat_members(chat_id, user_id),
CONSTRAINT attachments_sha256_format_check CHECK (sha256 COLLATE "C" ~ '^[0-9a-f]{64}$'),
CONSTRAINT attachments_content_type_check CHECK (content_type IN ('image/png', 'image/jpeg')),
CONSTRAINT attachments_filename_check CHECK (
char_length(filename) BETWEEN 1 AND 255
AND filename !~ '[/\\]'
),
CONSTRAINT attachments_size_bytes_check CHECK (size_bytes BETWEEN 1 AND 10485760),
CONSTRAINT attachments_id_chat_unique UNIQUE (id, chat_id),
CONSTRAINT attachments_object_key_length_check CHECK (char_length(object_key) > 0)
);
ALTER TABLE messages
ADD CONSTRAINT messages_id_chat_unique UNIQUE (id, chat_id);
CREATE TABLE messages_attachments
(
message_id UUID NOT NULL ,
attachment_id UUID NOT NULL ,
chat_id UUID NOT NULL,
position SMALLINT NOT NULL,
CONSTRAINT message_attachments_pk PRIMARY KEY (message_id, attachment_id),
CONSTRAINT message_attachments_position_unique UNIQUE (message_id, position),
CONSTRAINT message_attachments_position_check CHECK (position BETWEEN 0 AND 9),
CONSTRAINT message_attachments_message_chat_fk
FOREIGN KEY (message_id, chat_id) REFERENCES messages(id, chat_id),
CONSTRAINT message_attachments_attachment_chat_fk
FOREIGN KEY (attachment_id, chat_id) REFERENCES attachments(id, chat_id)
);
-- +goose Down
DROP TABLE IF EXISTS messages_attachments;
DROP TABLE IF EXISTS attachments;
ALTER TABLE messages DROP CONSTRAINT IF EXISTS messages_id_chat_unique;
@@ -0,0 +1,18 @@
-- +goose Up
ALTER TABLE messages
DROP CONSTRAINT messages_text_length_check,
DROP CONSTRAINT messages_text_not_blank_check;
ALTER TABLE messages
ADD CONSTRAINT messages_text_length_check
CHECK (char_length(text) <= 4000);
-- +goose Down
ALTER TABLE messages
DROP CONSTRAINT messages_text_length_check;
ALTER TABLE messages
ADD CONSTRAINT messages_text_length_check
CHECK (char_length(text) BETWEEN 1 AND 4000),
ADD CONSTRAINT messages_text_not_blank_check
CHECK (char_length(btrim(text)) > 0);
+144
View File
@@ -0,0 +1,144 @@
-- +goose Up
ALTER TABLE attachments
ADD COLUMN status TEXT NOT NULL DEFAULT 'queued',
ADD COLUMN error_code TEXT,
ADD COLUMN staging_key TEXT,
ADD COLUMN pipeline_version INT NOT NULL DEFAULT 1,
ALTER COLUMN object_key DROP NOT NULL;
-- Retain the original download path while the copied synchronous API is in use.
UPDATE attachments SET staging_key = object_key;
ALTER TABLE attachments
ADD CONSTRAINT attachments_status_check
CHECK (status IN ('queued', 'processing', 'ready', 'failed')),
ADD CONSTRAINT attachments_pipeline_check CHECK (pipeline_version > 0),
ADD CONSTRAINT attachments_staging_key_check CHECK (length(staging_key) > 0),
ADD CONSTRAINT attachments_object_state_check CHECK (
(status NOT IN ('queued', 'processing') OR staging_key IS NOT NULL)
AND (status <> 'ready' OR object_key IS NOT NULL)
),
ADD CONSTRAINT attachments_error_check CHECK (
(status = 'failed') = (error_code IS NOT NULL)
AND (error_code IS NULL OR error_code COLLATE "C" ~ '^[a-z][a-z0-9_]{0,49}$')
);
CREATE UNIQUE INDEX attachments_staging_key_unique ON attachments (staging_key)
WHERE staging_key IS NOT NULL;
CREATE TABLE attachment_jobs (
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
attachment_id UUID NOT NULL REFERENCES attachments(id),
pipeline_version INT NOT NULL CHECK (pipeline_version > 0),
attempts_started INT NOT NULL DEFAULT 0,
max_attempts INT NOT NULL DEFAULT 5,
next_attempt_at TIMESTAMPTZ DEFAULT now(),
lease_generation INT NOT NULL DEFAULT 0 CHECK (lease_generation >= 0),
lease_until TIMESTAMPTZ,
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
updated_at TIMESTAMPTZ NOT NULL DEFAULT now(),
UNIQUE (attachment_id, pipeline_version),
CHECK (attempts_started >= 0 AND max_attempts > 0 AND attempts_started <= max_attempts),
CHECK (lease_until IS NULL OR next_attempt_at IS NULL)
);
CREATE INDEX attachment_jobs_expired_lease_idx ON attachment_jobs (lease_until, id)
WHERE lease_until IS NOT NULL;
CREATE TABLE attachment_variants (
attachment_id UUID NOT NULL,
pipeline_version INT NOT NULL,
name TEXT NOT NULL,
object_key TEXT NOT NULL UNIQUE CHECK (length(object_key) > 0),
content_type TEXT NOT NULL,
size_bytes BIGINT NOT NULL CHECK (size_bytes > 0),
width INT NOT NULL CHECK (width > 0),
height INT NOT NULL CHECK (height > 0),
sha256 TEXT NOT NULL CHECK (sha256 COLLATE "C" ~ '^[0-9a-f]{64}$'),
PRIMARY KEY (attachment_id, pipeline_version, name),
FOREIGN KEY (attachment_id, pipeline_version)
REFERENCES attachment_jobs(attachment_id, pipeline_version),
CHECK (
(name IN ('thumbnail', 'crop') AND content_type = 'image/jpeg')
OR (name = 'optimized' AND content_type = 'image/webp')
)
);
CREATE TABLE attachment_attempts (
job_id UUID NOT NULL REFERENCES attachment_jobs(id),
lease_generation INT NOT NULL CHECK (lease_generation > 0),
attempt INT NOT NULL CHECK (attempt > 0),
started_at TIMESTAMPTZ NOT NULL,
finished_at TIMESTAMPTZ,
outcome TEXT CHECK (outcome IN ('ready', 'retry', 'failed', 'lease_expired')),
error_code TEXT CHECK (error_code COLLATE "C" ~ '^[a-z][a-z0-9_]{0,49}$'),
PRIMARY KEY (job_id, lease_generation),
UNIQUE (job_id, attempt),
CHECK ((outcome IS NULL) = (finished_at IS NULL))
);
CREATE TABLE attachment_replays (
request_id UUID PRIMARY KEY,
job_id UUID NOT NULL REFERENCES attachment_jobs(id),
operator_id TEXT NOT NULL CHECK (length(btrim(operator_id)) > 0),
reason TEXT NOT NULL CHECK (length(btrim(reason)) > 0),
extra_attempts INT NOT NULL CHECK (extra_attempts BETWEEN 1 AND 5),
created_at TIMESTAMPTZ NOT NULL DEFAULT now()
);
CREATE TABLE outbox (
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
job_id UUID NOT NULL REFERENCES attachment_jobs(id),
payload JSONB NOT NULL,
available_at TIMESTAMPTZ NOT NULL DEFAULT now(),
sent_at TIMESTAMPTZ,
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
dedup_key TEXT UNIQUE NOT NULL CHECK (length(dedup_key) > 0)
);
CREATE INDEX outbox_pending_idx ON outbox (available_at, id) WHERE sent_at IS NULL;
-- +goose StatementBegin
CREATE FUNCTION attachment_job_payload(uuid, uuid, integer, integer, timestamptz)
RETURNS jsonb LANGUAGE sql STABLE
AS $$
SELECT jsonb_build_object(
'schema_version', 1,
'type', 'attachment.process.v1',
'job_id', $1,
'attachment_id', $2,
'pipeline_version', $3,
'attempt', $4,
'created_at', to_char($5 AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS.US"Z"')
);
$$;
-- +goose StatementEnd
WITH jobs AS (
INSERT INTO attachment_jobs (attachment_id, pipeline_version)
SELECT id, pipeline_version FROM attachments
RETURNING *
)
INSERT INTO outbox (job_id, payload, dedup_key)
SELECT id, attachment_job_payload(id, attachment_id, pipeline_version, 1, created_at),
'initial/' || id
FROM jobs;
-- +goose Down
-- Restore the legacy reference when the staging original is the only copy.
-- SET NOT NULL aborts the rollback if an original no longer exists.
UPDATE attachments SET object_key = staging_key WHERE object_key IS NULL;
ALTER TABLE attachments ALTER COLUMN object_key SET NOT NULL;
DROP FUNCTION attachment_job_payload(uuid, uuid, integer, integer, timestamptz);
DROP TABLE outbox;
DROP TABLE attachment_replays;
DROP TABLE attachment_attempts;
DROP TABLE attachment_variants;
DROP TABLE attachment_jobs;
ALTER TABLE attachments
DROP COLUMN status,
DROP COLUMN error_code,
DROP COLUMN staging_key,
DROP COLUMN pipeline_version;
@@ -0,0 +1,27 @@
-- +goose Up
ALTER TABLE attachment_jobs ADD COLUMN published_original_key TEXT
CHECK (length(published_original_key) > 0);
UPDATE attachment_jobs AS j
SET published_original_key = a.object_key
FROM attachments AS a
WHERE a.id = j.attachment_id AND a.pipeline_version = j.pipeline_version
AND a.status = 'ready';
CREATE INDEX attachment_jobs_published_original_idx ON attachment_jobs (published_original_key)
WHERE published_original_key IS NOT NULL;
ALTER TABLE attachment_attempts
DROP CONSTRAINT attachment_attempts_outcome_check,
ADD CONSTRAINT attachment_attempts_outcome_check
CHECK (outcome IN ('ready', 'retry', 'failed', 'lease_expired', 'superseded'));
-- +goose Down
UPDATE attachment_attempts SET outcome = 'failed' WHERE outcome = 'superseded';
ALTER TABLE attachment_attempts
DROP CONSTRAINT attachment_attempts_outcome_check,
ADD CONSTRAINT attachment_attempts_outcome_check
CHECK (outcome IN ('ready', 'retry', 'failed', 'lease_expired'));
ALTER TABLE attachment_jobs DROP COLUMN published_original_key;
+9
View File
@@ -0,0 +1,9 @@
package migrations
import "embed"
// Files contains the SQL shipped with this exact migrator build.
// Add schema changes as new numbered files; do not edit applied migrations.
//
//go:embed *.sql
var Files embed.FS
+11
View File
@@ -0,0 +1,11 @@
-- name: CreateAttachmentVariant :exec
INSERT INTO attachment_variants (
attachment_id, pipeline_version, name, object_key, content_type, size_bytes, width, height, sha256
) VALUES (
$1, $2, $3, $4, $5, $6, $7, $8, $9
);
-- name: ListAttachmentVariants :many
SELECT * FROM attachment_variants
WHERE attachment_id = $1 AND pipeline_version = $2
ORDER BY name;
+69
View File
@@ -0,0 +1,69 @@
-- name: CreateAttachment :one
-- Compatibility for the synchronous API until CreateQueuedAttachment is wired in.
WITH legacy AS (
INSERT INTO attachments
(chat_id, owner_id, filename, object_key, staging_key, content_type, size_bytes, sha256)
VALUES (sqlc.arg(chat_id), sqlc.arg(owner_id), sqlc.arg(filename),
sqlc.arg(object_key)::text, sqlc.arg(object_key)::text,
sqlc.arg(content_type), sqlc.arg(size_bytes), sqlc.arg(sha256))
RETURNING *
), job AS (
INSERT INTO attachment_jobs (attachment_id, pipeline_version)
SELECT id, pipeline_version FROM legacy
RETURNING *
), event AS (
INSERT INTO outbox (job_id, payload, dedup_key)
SELECT id, attachment_job_payload(id, attachment_id, pipeline_version, 1, created_at),
'initial/' || id
FROM job
RETURNING job_id
)
SELECT legacy.* FROM legacy CROSS JOIN event;
-- name: GetAttachmentForUser :one
SELECT a.*
FROM attachments AS a
JOIN chat_members AS cm ON cm.chat_id = a.chat_id
WHERE a.id = sqlc.arg(attachment_id)
AND cm.user_id = sqlc.arg(user_id);
-- name: GetAttachmentsForMessages :many
SELECT message_id, attachment_id, position
FROM messages_attachments
WHERE message_id = ANY(sqlc.arg(message_ids)::uuid[])
ORDER BY message_id, position;
-- name: AttachmentExistsByObjectKey :one
SELECT EXISTS (
SELECT 1 FROM attachments
WHERE object_key = sqlc.arg(object_key)::text OR staging_key = sqlc.arg(object_key)::text
UNION ALL
SELECT 1 FROM attachment_variants WHERE object_key = sqlc.arg(object_key)::text
UNION ALL
SELECT 1 FROM attachment_jobs WHERE published_original_key = sqlc.arg(object_key)::text
);
-- name: TryLockAttachmentObjectKey :one
SELECT pg_try_advisory_xact_lock(
hashtextextended(sqlc.arg(object_key)::text, 0)
)::boolean AS locked;
-- name: ListAttachmentKeys :many
SELECT id, object_key, created_at
FROM attachments
WHERE object_key > sqlc.arg(after_key)
ORDER BY object_key
LIMIT sqlc.arg(page_size);
-- name: GetAttachmentByID :one
SELECT * FROM attachments WHERE id = sqlc.arg(attachment_id);
-- name: CreateQueuedAttachment :one
INSERT INTO attachments (
id, chat_id, owner_id, filename, content_type, size_bytes, sha256, staging_key, status
) VALUES (
$1, $2, $3, $4, $5, $6, $7, $8, 'queued'
) RETURNING *;
+207
View File
@@ -0,0 +1,207 @@
-- name: CreateAttachmentJob :one
INSERT INTO attachment_jobs (attachment_id, pipeline_version, next_attempt_at, max_attempts, created_at)
VALUES (sqlc.arg(attachment_id), sqlc.arg(pipeline_version),
sqlc.arg(available_at)::timestamptz, sqlc.arg(max_attempts), clock_timestamp())
RETURNING *;
-- name: GetQueueStats :one
SELECT count(*)::bigint AS depth,
COALESCE(EXTRACT(EPOCH FROM (statement_timestamp() - min(j.created_at))), 0)::double precision AS oldest_seconds
FROM attachment_jobs AS j JOIN attachments AS a ON a.id = j.attachment_id
WHERE a.status = 'queued' AND a.pipeline_version = j.pipeline_version;
-- name: ClaimJob :one
WITH claimed AS (
UPDATE attachment_jobs AS j
SET lease_generation = j.lease_generation + 1,
lease_until = sqlc.arg(new_lease_until)::timestamptz,
attempts_started = j.attempts_started + 1,
next_attempt_at = NULL,
updated_at = clock_timestamp()
FROM attachments AS a
WHERE j.id = sqlc.arg(job_id)
AND a.id = j.attachment_id AND a.pipeline_version = j.pipeline_version
AND j.attempts_started < j.max_attempts
AND sqlc.arg(new_lease_until)::timestamptz > clock_timestamp()
AND (
(a.status = 'queued' AND j.lease_until IS NULL AND j.next_attempt_at <= clock_timestamp())
OR (a.status = 'processing' AND j.lease_until <= clock_timestamp())
)
RETURNING j.*
), expired AS (
UPDATE attachment_attempts AS attempt
SET outcome = 'lease_expired', error_code = 'lease_expired', finished_at = claimed.updated_at
FROM claimed
WHERE attempt.job_id = claimed.id
AND attempt.lease_generation = claimed.lease_generation - 1 AND attempt.finished_at IS NULL
), recorded AS (
INSERT INTO attachment_attempts (job_id, lease_generation, attempt, started_at)
SELECT id, lease_generation, attempts_started, updated_at FROM claimed
RETURNING job_id
), processing AS (
UPDATE attachments AS a
SET status = 'processing', updated_at = claimed.updated_at
FROM claimed
WHERE a.id = claimed.attachment_id
RETURNING a.id
)
SELECT claimed.* FROM claimed JOIN recorded ON recorded.job_id = claimed.id
JOIN processing ON processing.id = claimed.attachment_id;
-- name: LockActiveAttachmentJob :one
-- Hold this lock through variant insertion and CompleteJob. Roll back on a zero-row completion.
SELECT j.*
FROM attachment_jobs AS j JOIN attachments AS a ON a.id = j.attachment_id
WHERE j.id = sqlc.arg(job_id)
AND j.lease_generation = sqlc.arg(lease_generation)
AND j.lease_until > clock_timestamp()
AND a.status = 'processing' AND a.pipeline_version = j.pipeline_version
FOR UPDATE OF j;
-- name: CompleteJob :execrows
WITH completed AS (
UPDATE attachment_jobs AS j
SET lease_until = NULL, next_attempt_at = NULL, updated_at = clock_timestamp(),
published_original_key = sqlc.arg(object_key)::text
FROM attachments AS a
WHERE j.id = sqlc.arg(job_id)
AND j.lease_generation = sqlc.arg(lease_generation) AND j.lease_until > clock_timestamp()
AND a.id = j.attachment_id AND a.pipeline_version = j.pipeline_version AND a.status = 'processing'
AND (SELECT count(*) FROM attachment_variants AS v
WHERE v.attachment_id = a.id AND v.pipeline_version = j.pipeline_version) = 3
RETURNING j.*
), recorded AS (
UPDATE attachment_attempts AS attempt
SET outcome = 'ready', finished_at = completed.updated_at
FROM completed
WHERE attempt.job_id = completed.id AND attempt.lease_generation = completed.lease_generation
)
UPDATE attachments AS a
SET status = 'ready', object_key = sqlc.arg(object_key)::text, updated_at = completed.updated_at
FROM completed
WHERE a.id = completed.attachment_id;
-- name: ScheduleRetry :execrows
WITH scheduled AS (
UPDATE attachment_jobs AS j
SET lease_until = NULL, next_attempt_at = sqlc.arg(available_at)::timestamptz,
updated_at = clock_timestamp()
FROM attachments AS a
WHERE j.id = sqlc.arg(job_id)
AND j.lease_generation = sqlc.arg(lease_generation) AND j.lease_until > clock_timestamp()
AND a.id = j.attachment_id AND a.pipeline_version = j.pipeline_version AND a.status = 'processing'
AND j.attempts_started < j.max_attempts
AND sqlc.arg(available_at)::timestamptz > clock_timestamp()
RETURNING j.*
), recorded AS (
UPDATE attachment_attempts AS attempt
SET outcome = 'retry', error_code = sqlc.arg(error_code)::text, finished_at = scheduled.updated_at
FROM scheduled
WHERE attempt.job_id = scheduled.id AND attempt.lease_generation = scheduled.lease_generation
), event AS (
INSERT INTO outbox (job_id, payload, available_at, dedup_key)
SELECT id, attachment_job_payload(id, attachment_id, pipeline_version, attempts_started + 1, updated_at)
|| jsonb_strip_nulls(jsonb_build_object(
'traceparent', NULLIF(sqlc.arg(traceparent)::text, ''),
'tracestate', NULLIF(sqlc.arg(tracestate)::text, '')
)),
next_attempt_at, 'retry/' || id || '/' || (attempts_started + 1)
FROM scheduled
)
UPDATE attachments AS a
SET status = 'queued', updated_at = scheduled.updated_at
FROM scheduled WHERE a.id = scheduled.attachment_id;
-- name: FailJob :execrows
WITH failed AS (
UPDATE attachment_jobs AS j
SET lease_until = NULL, next_attempt_at = NULL, updated_at = clock_timestamp()
FROM attachments AS a
WHERE j.id = sqlc.arg(job_id)
AND j.lease_generation = sqlc.arg(lease_generation)
AND (
j.lease_until > clock_timestamp()
OR (j.lease_until <= clock_timestamp() AND j.attempts_started = j.max_attempts
AND sqlc.arg(error_code)::text = 'retry_exhausted')
)
AND a.id = j.attachment_id AND a.pipeline_version = j.pipeline_version AND a.status = 'processing'
RETURNING j.*
), recorded AS (
UPDATE attachment_attempts AS attempt
SET outcome = 'failed', error_code = sqlc.arg(error_code)::text, finished_at = failed.updated_at
FROM failed
WHERE attempt.job_id = failed.id AND attempt.lease_generation = failed.lease_generation
)
UPDATE attachments AS a
SET status = 'failed', error_code = sqlc.arg(error_code)::text, updated_at = failed.updated_at
FROM failed WHERE a.id = failed.attachment_id;
-- name: ReplayFailed :execrows
WITH replayed AS (
UPDATE attachment_jobs AS j
SET max_attempts = j.attempts_started + sqlc.arg(extra_attempts)::int,
next_attempt_at = clock_timestamp(), updated_at = clock_timestamp()
FROM attachments AS a
WHERE j.id = sqlc.arg(job_id)
AND a.id = j.attachment_id AND a.pipeline_version = j.pipeline_version
AND a.status = 'failed' AND a.staging_key IS NOT NULL
AND j.lease_until IS NULL AND j.next_attempt_at IS NULL
AND NOT EXISTS (
SELECT 1 FROM attachment_replays AS r WHERE r.request_id = sqlc.arg(request_id)::uuid
)
RETURNING j.*
), recorded AS (
INSERT INTO attachment_replays (request_id, job_id, operator_id, reason, extra_attempts)
SELECT sqlc.arg(request_id)::uuid, id, sqlc.arg(operator_id), sqlc.arg(reason), sqlc.arg(extra_attempts)
FROM replayed
), event AS (
INSERT INTO outbox (job_id, payload, available_at, dedup_key)
SELECT id, attachment_job_payload(id, attachment_id, pipeline_version, attempts_started + 1, updated_at),
next_attempt_at, 'replay/' || id || '/' || sqlc.arg(request_id)::uuid
FROM replayed
)
UPDATE attachments AS a
SET status = 'queued', error_code = NULL, updated_at = replayed.updated_at
FROM replayed WHERE a.id = replayed.attachment_id;
-- name: GetAttachmentJobState :one
SELECT
j.id, j.attachment_id, j.pipeline_version,
CASE WHEN a.pipeline_version = j.pipeline_version THEN a.status ELSE 'superseded' END::text AS status,
j.lease_generation,
j.lease_until,
coalesce(j.lease_until <= clock_timestamp(), false)::boolean AS lease_expired,
j.attempts_started,
j.max_attempts,
j.next_attempt_at
FROM attachment_jobs AS j
JOIN attachments AS a ON a.id = j.attachment_id
WHERE j.id = sqlc.arg(job_id)::uuid;
-- name: ListFailedAttachmentJobs :many
SELECT j.id, j.attachment_id, j.attempts_started, a.error_code, a.updated_at
FROM attachment_jobs j
JOIN attachments a ON a.id = j.attachment_id AND a.pipeline_version = j.pipeline_version
WHERE a.status = 'failed'
ORDER BY a.updated_at DESC, j.id
LIMIT sqlc.arg(batch_size);
-- name: LockAttachmentJobForPipeline :one
SELECT j.*
FROM attachment_jobs j
WHERE j.attachment_id = $1 AND j.pipeline_version = $2
FOR UPDATE;
-- name: LockAttachmentJobForReplay :one
SELECT j.*
FROM attachment_jobs j WHERE j.id = $1
FOR UPDATE;
-- name: ClearStagingReference :execrows
UPDATE attachments
SET object_key = CASE WHEN status = 'failed' AND object_key = staging_key THEN NULL ELSE object_key END,
staging_key = NULL
WHERE id = sqlc.arg(id) AND staging_key = sqlc.arg(staging_key);
-- name: GetAttachmentByStagingKey :one
SELECT * FROM attachments WHERE staging_key = sqlc.arg(staging_key)::text;
+3
View File
@@ -0,0 +1,3 @@
-- name: CreateChatMember :exec
INSERT INTO chat_members(chat_id, user_id)
VALUES ($1, $2);
+34
View File
@@ -0,0 +1,34 @@
-- name: CreateChat :one
INSERT INTO chats (title)
VALUES ($1)
RETURNING id, title, created_at;
-- name: GetChatByID :one
SELECT c.id, c.title, c.created_at, array_agg(cm.user_id)::uuid[] AS member_ids
FROM chats c
JOIN chat_members cm ON cm.chat_id = c.id
WHERE c.id = $1
GROUP BY c.id;
-- name: GetChatsByUserID :many
SELECT c.id, c.title, c.created_at, array_agg(cm.user_id)::uuid[] AS member_ids
FROM chats c
JOIN chat_members cm ON cm.chat_id = c.id
WHERE EXISTS (SELECT 1
FROM chat_members membership
WHERE membership.chat_id = c.id
AND membership.user_id = sqlc.arg(user_id))
AND (
sqlc.narg(after_created_at)::timestamptz IS NULL
OR (c.created_at, c.id) >
(
sqlc.narg(after_created_at)::timestamptz,
sqlc.narg(after_id)::uuid
)
)
GROUP BY c.id
ORDER BY c.created_at ASC, c.id ASC
LIMIT sqlc.arg(limit_count);
+22
View File
@@ -0,0 +1,22 @@
-- name: InsertMessageIdempotency :execrows
INSERT INTO message_idempotency(user_id, chat_id, idempotency_key, message_id)
VALUES ($1, $2, $3, $4)
ON CONFLICT (user_id, chat_id, idempotency_key) DO NOTHING;
-- name: GetIdempotentMessage :one
SELECT m.id, m.chat_id, m.sender_id, m.text, m.created_at
FROM message_idempotency AS i
JOIN messages AS m ON m.id = i.message_id
WHERE i.user_id = $1
AND i.chat_id = $2
AND i.idempotency_key = $3;
-- name: LockMessageRequest :exec
SELECT pg_advisory_xact_lock(hashtextextended(
'message/' || (sqlc.arg(user_id)::uuid)::text || '/' ||
(sqlc.arg(chat_id)::uuid)::text || '/' || sqlc.arg(idempotency_key)::text, 0));
-- name: DeleteExpiredMessageIdempotency :exec
DELETE FROM message_idempotency
WHERE user_id = $1 AND chat_id = $2 AND idempotency_key = $3 AND expires_at <= now();
+19
View File
@@ -0,0 +1,19 @@
-- name: CreateMessage :one
INSERT INTO messages (chat_id, sender_id, text)
VALUES ($1, $2, $3)
RETURNING *;
-- name: GetMessagesByChatID :many
SELECT id, chat_id, sender_id, text, created_at
FROM messages
WHERE chat_id = sqlc.arg(chat_id)
AND (
sqlc.narg(after_created_at)::timestamptz IS NULL
OR (created_at, id) >
(sqlc.narg(after_created_at)::timestamptz,
sqlc.narg(after_id)::uuid
)
)
ORDER BY created_at ASC, id ASC
LIMIT sqlc.arg(limit_count);
+16
View File
@@ -0,0 +1,16 @@
-- name: CreateMessagesAttachments :execrows
INSERT INTO messages_attachments (message_id, attachment_id, chat_id, position)
SELECT $1, $2, $3, $4
FROM attachments
WHERE id = $2 AND chat_id = $3;
-- name: GetMessagesAttachments :many
SELECT attachment_id, position
FROM messages_attachments
WHERE message_id = sqlc.arg(message_id)
ORDER BY position ASC;
-- name: LockAttachmentsForMessage :many
SELECT id, status FROM attachments
WHERE chat_id = sqlc.arg(chat_id) AND id = ANY(sqlc.arg(attachment_ids)::uuid[])
ORDER BY id FOR SHARE;
+50
View File
@@ -0,0 +1,50 @@
-- name: CreateInitialOutboxEvent :exec
INSERT INTO outbox (job_id, payload, available_at, dedup_key)
SELECT id, attachment_job_payload(id, attachment_id, pipeline_version, 1, created_at)
|| jsonb_strip_nulls(jsonb_build_object(
'traceparent', NULLIF(sqlc.arg(traceparent)::text, ''),
'tracestate', NULLIF(sqlc.arg(tracestate)::text, '')
)),
next_attempt_at, 'initial/' || id
FROM attachment_jobs WHERE id = sqlc.arg(job_id)::uuid;
-- name: EnqueueExpiredLeases :many
WITH expired AS (
SELECT j.*, 'recovery/' || j.id || '/' || j.lease_generation AS recovery_key
FROM attachment_jobs AS j
JOIN attachments AS a ON a.id = j.attachment_id AND a.pipeline_version = j.pipeline_version
WHERE a.status = 'processing' AND j.lease_until <= clock_timestamp()
AND NOT EXISTS (
SELECT 1 FROM outbox AS o WHERE o.dedup_key = 'recovery/' || j.id || '/' || j.lease_generation
)
ORDER BY j.lease_until, j.id
LIMIT sqlc.arg(batch_size)
FOR UPDATE OF j SKIP LOCKED
)
INSERT INTO outbox (job_id, payload, dedup_key)
SELECT e.id, attachment_job_payload(e.id, e.attachment_id, e.pipeline_version,
LEAST(e.attempts_started + 1, e.max_attempts), clock_timestamp())
|| COALESCE((
SELECT jsonb_strip_nulls(jsonb_build_object(
'traceparent', o.payload->>'traceparent', 'tracestate', o.payload->>'tracestate'
))
FROM outbox AS o WHERE o.job_id = e.id
ORDER BY o.created_at DESC, o.id DESC LIMIT 1
), '{}'::jsonb),
e.recovery_key
FROM expired AS e
ON CONFLICT (dedup_key) DO NOTHING
RETURNING *;
-- name: NextPendingOutboxEvent :one
SELECT id, job_id, payload, available_at, sent_at, created_at, dedup_key
FROM outbox
WHERE sent_at IS NULL AND available_at <= statement_timestamp()
ORDER BY available_at, id
LIMIT 1
FOR UPDATE SKIP LOCKED;
-- name: MarkOutboxEventSent :execrows
UPDATE outbox SET sent_at = clock_timestamp()
WHERE id = sqlc.arg(id) AND sent_at IS NULL;
+21
View File
@@ -0,0 +1,21 @@
-- name: CreateUser :one
INSERT INTO users (username, display_name, password_hash)
VALUES ($1, $2, $3)
RETURNING id, username, display_name, password_hash, created_at;
-- name: GetUserByID :one
SELECT id, username, display_name, password_hash, created_at
FROM users
WHERE id = $1;
-- name: GetUserByUsername :one
SELECT id, username, display_name, password_hash, created_at
FROM users
WHERE username = $1;
+30
View File
@@ -0,0 +1,30 @@
-- name: SupersedeAttachmentJob :exec
WITH superseded AS (
UPDATE attachment_jobs
SET lease_until = NULL, next_attempt_at = NULL, updated_at = clock_timestamp()
WHERE id = sqlc.arg(job_id)::uuid
RETURNING id, updated_at
)
UPDATE attachment_attempts AS attempt
SET outcome = 'superseded', error_code = 'superseded', finished_at = superseded.updated_at
FROM superseded
WHERE attempt.job_id = superseded.id AND attempt.finished_at IS NULL;
-- name: QueueAttachmentUpgrade :one
UPDATE attachments
SET pipeline_version = 2, status = 'queued', staging_key = sqlc.arg(source_key)::text,
object_key = NULL, error_code = NULL, updated_at = clock_timestamp()
WHERE id = sqlc.arg(id)::uuid AND pipeline_version = 1
RETURNING *;
-- name: GetPipelineJobID :one
SELECT id FROM attachment_jobs
WHERE attachment_id = sqlc.arg(attachment_id)::uuid AND pipeline_version = sqlc.arg(pipeline_version)::int;
-- name: GetPublishedOriginalKey :one
SELECT published_original_key FROM attachment_jobs
WHERE attachment_id = sqlc.arg(attachment_id)::uuid AND pipeline_version = sqlc.arg(pipeline_version)::int
AND published_original_key IS NOT NULL;
-- name: IsPublishedOriginalKey :one
SELECT EXISTS (SELECT 1 FROM attachment_jobs WHERE published_original_key = sqlc.arg(object_key)::text);
+39
View File
@@ -0,0 +1,39 @@
# Приёмка ЛР 6 на «4», 16 ноября 2026
Все девять проверок получили PASS по итогам отдельных прогонов на
`highload-lab6-student`. Данные и секреты синтетические, тома выделены
при первом запуске. API 28106, Prometheus 29196, Tempo 23206, Loki 23106,
Grafana 23006. Docker Engine 28.4.0 на aarch64, 8 CPU и 3,83 GiB RAM.
Использован Python 3.14.6. Команды и инварианты есть в [инструкции](../../student-tests/lab6/README.md).
[Сводка](summary.json) указывает успешный архив для каждого сценария.
| Проверка | Наблюдение |
| --- | --- |
| Smoke и Grafana | `make check` и `make test` прошли с включённой телеметрией. У provisioning есть три datasource. Запросы опубликованных панелей успешно исполнил Prometheus. После обновления dashboard проверено также выражение с нулём при отсутствии 5xx. |
| Сквозная трасса | 43 реальных спана в Tempo. Входящий W3C-parent ведёт к API upload, дальше к двум разным producer → consumer парам. Первая попытка дала `s3_unavailable`, вторая привела ту же job к ready. Видны дочерние S3 и PostgreSQL. Обе outbox-записи сохранили один trace ID и `tracestate`. Исходник проверен побайтно. |
| Логи и приватность | Loki показал совпадающие trace/span IDs для API и worker и фактический request ID API. Проверено 58 записей в контрольной выборке Loki, трасса загрузки и четыре трассы регистрации, входов и сообщения. Контрольные секретные значения отсутствовали. |
| Метрики | Согласованные buckets/count/sum для HTTP, jobs и time-to-ready. Для смешанной бизнес-выборки p50/p95/p99 HTTP составили 2,20 / 209,09 / 1000 мс. Prometheus использовал HTTP-гистограмму без health. Восемь новых чатов дали прирост счётчика на одном шаблоне пути. Присутствуют queue, DB pool, worker saturation, CPU и память. |
| Worker остановлен | Три `202` оставили глубину 3. Первая выборка после порога показала возраст 18,59 с. `GrowingQueueLag` стал firing через 39,82 с от остановки. После запуска все три исходника сохранились и стали ready, глубина 0, resolved через 9,04 с. |
| Чтение PostgreSQL заблокировано | Под `AccessExclusiveLock` на `public.users` API за LB ответил 503 на 23 проверочных запроса. Readiness оставался доступен. Trace 503 содержит PostgreSQL-ошибку, Loki показывает связанный API-лог. `HighAPIErrorRate` стал firing через 32,73 с от блокировки, resolved через 23,81 с после её снятия. |
| Loki остановлен на 34 с | 278 чтений и 8 записей успешно подтверждены. Все 8 сообщений остались в истории. Счётчик неэкспортированных логов вырос на 96, впервые через 8,69 с. Очередь достигла 16 из 16, пиковый RSS 88,55 MiB при лимите 256 MiB. Collector сохранил тот же процесс. После запуска Loki видны предаварийный и новый коррелированные логи. |
Лимиты режима с Loki указаны в конфигурации и измерены через Prometheus.
Очередь 16 пакетов, retry до 5 с, memory limiter 96 MiB. Подтверждён
[сохранённый лог и точки монтирования](retention.json). Проверка истории через
физический рестарт Prometheus и Tempo в этот прогон не входила. Короткие
искусственные окна не подтверждают выполнение SLO за 24 часа.
Первые прогоны дали полезные сбои. Первую проверку smoke остановила неполная
тестовая копия. Два сценария алертов остановил конфликт полей JSONL. В следующем
прогоне база GrowingQueueLag ещё не успела успокоиться после retry. Тест теперь
дожидается фактического inactive. В прогретом опыте Loki RSS подошёл к прежнему
лимиту 192 MiB. Автор поднял лимит контейнера до 256 MiB, сохранив ограничения
очереди и повторов. В первой попытке с новым лимитом одна API-реплика вернула
503 за 1,09 с одновременно со стартом второго Compose. Причинность с Loki
не установлена. [Поля API/LB для 503](loki-503-api-lb.txt) сохранены. Повтор
с теми же строгими проверками прошёл после остановки второго стенда.
[Сквозной прогон и DB](final.jsonl.gz) · [Worker](worker-rerun.jsonl.gz) ·
[Актуальный dashboard и smoke](loki-final.jsonl.gz) · [Успешный опыт Loki](loki-quiet.jsonl.gz) ·
[Первичные результаты](acceptance-resumed.jsonl.gz) · [Образы](image-ids.txt) ·
[Исходники и актуальные конфиги](source-sha256-latest.txt).
Binary file not shown.
Binary file not shown.
+5
View File
@@ -0,0 +1,5 @@
highload-lab6-api:latest sha256:f20080b5068ac0caf9c9ff5d495413bfe0e8230f2cfd920d2bfb080380b34447
highload-lab6-attachment-cleanup:latest sha256:e814e8c21686a66cb87921130182c80d83f1b82414758ed421dd965793dc8159
highload-lab6-relay:latest sha256:89bee09ca623bc36b9ba001c6beefbdbb5a2ebc5a86fc9a20101bdafdea91a40
highload-lab6-worker:latest sha256:dad64c91ff7869ef650df2cdc312397d071a881557a25a5cf2b4101a3f89f07c
highload-lab6-migrate:latest sha256:b424420c3d12a67c0f867393146fab6c8a193ce49c3d5e0fabf6934ff8915904
+2
View File
@@ -0,0 +1,2 @@
[16/Nov/2026:06:34:22.540] api api_replicas/api2 0/0/43/1247/1290 503 318 - - ---- 1/1/0/0/0 0/0 "GET /api/v1/users/me HTTP/1.1"
2026-11-16T06:34:23.832489586Z 2026/11/16 06:34:23 INFO http request method=GET route=/api/v1/users/me status=503 duration=1.092129709s
Binary file not shown.
Binary file not shown.
+25
View File
@@ -0,0 +1,25 @@
{
"loki_log_stored_before_restart_and_read_after": [
{
"timestamp_ns": "1794810962234573465",
"event": "http.request",
"trace_id": "d4ad4c8543c10b091e760dbfe866a291",
"span_id": "5b0971b0e3ce60ae",
"request_id": "55de0f79-ffb2-4d17-ab69-555362ed7f47"
}
],
"named_volume_mounts": {
"prometheus": [
"/prometheus"
],
"loki": [
"/loki"
],
"tempo": [
"/var/tempo"
],
"grafana": [
"/var/lib/grafana"
]
}
}
@@ -0,0 +1,134 @@
2658e84d6a062552303e04e41e45bd0b5fa1f6a75e90158f616d60fedbef3b3d cmd/attachment-cleanup/main.go
3c462c48f6373570e19d9b63b87662e7ee8a135acbb80c97d0052d5610937354 cmd/migrate/roundtrip_test.go
f7438f00679b09125cdefa813ad5a49a8ca88004b3b1535b7112700536ab5000 cmd/migrate/async_attachments_test.go
ac1707fdcbbfccb1a9ed678da66365c9bc9e3526ea751b7749ad512bf3a6c752 cmd/migrate/postgres_test.go
6e9d413a86f53cf5f2e83d5ed0308af9b2c33e69fcb3d061b19bf480e352f257 cmd/migrate/main.go
a88f2252d70caf644904331182dc3afd4df48b825a0329a82482798ec90b5d46 cmd/migrate/main_test.go
8545c84b30af7c7e58889e1102e3e3669307294e1d99733ba153235803b06ebe cmd/api/main.go
75c0ca950b85eaee3c0802e3001c61ba6436874b3be12cd38a5feb907b91528e cmd/api/poolstats.go
662d72dccac5061170bea781de5e6801fc93c4c8ad54db16ee145631eed4735e cmd/jobs/main.go
e9cf006fdce4dd646571bebc51b6ec9d96f13b70a92af463de761a0d2069772b cmd/worker/main.go
49fdc6844992a49c33dea95dfa5f55bb9684f8f35387485bcff5db9d359464a3 cmd/relay/main.go
ad56ecd0e8b88e0665cdc144aaa5d934ea7f48047cd702ce6f8832b6cf0bded3 db/migrations/00006_allow_attachment_only_messages.sql
a42e52ab05bc89b557a5bc6b3f0215883d99c4a039a8b32d6c19e66b2f769508 db/migrations/embed.go
e248c25d6af34cfed78976e349be4bee3d1987cd4cdfb8293c7aef07a9b2a92b db/migrations/00007_async_attachments.sql
b90ec5afff3a3680440e6034bfa12b1ded9fa1c1d5a85e4374e363306905092f db/migrations/00008_versioned_pipeline.sql
033d8884cecb50b5067453a97f2d7c51e51331a1911d9ddf8687437afa59443c db/migrations/00005_attachments.sql
1b666cee95b9eb1210b0a303de95994bd3c24628e9d9fdb1e1f3318edb756c3d db/migrations/00003_drop_session_table.sql
e7b3dd5a056a702dfde7acba6666430477db0c7e03ae03fb270e145d6303b9b2 db/migrations/00004_message_idempotency.sql
e0680838294b93f916d727a3a20d83aca7c3f74077202c9696b1fc73cda14649 db/migrations/00002_add_created_at_defaults.sql
48db014273f78be0f6eb97fc521cccc54f851dc99e3688d480d3a6ff7b9b6bbe db/migrations/00001_init_schema.sql
c80b6e50c321ff94317545ea2f9fdee1dfd69f564d8638f7b111c8ec091ef03a db/queries/chat_members.sql
0d73a6d46f4d4a1b0af37b750a208c07a38249f76974b41748f32c1949f990df db/queries/users.sql
5436a4d87e9a9e8a63a8845651dedebd17729ccdec6c6739ed209bed27dc3bd3 db/queries/messages_attachments.sql
cf38c8c9eee8d62983c05f5bf03f4804cf2a101dae19851ed6e1770e5a0ebc01 db/queries/message_idempotence.sql
003b2cd200d6c37ed74b7f8b77a6643e301c39662b90bfc367e8534784c77afd db/queries/attachment_variants.sql
f6f6382f27d698b1c2af2305d45699226ef10806b938b3dca78ec9087a622b7b db/queries/messages.sql
a73568ebdc2d8f12845bdacf19a7e30754d6eede2013027e61b315b0a96e6432 db/queries/versioned_pipeline.sql
8c52c762da78eb3a3eb7402b17ec87bd7fc2e1387f6ddb8e297b4f2d75beca7b db/queries/chats.sql
81b9797bf5b42f53d8712d0a153edb34544c18aafb4a928603b522c76e928a68 db/queries/outbox.sql
2e2c3cd3b5cf3a43318209d1757be412d38fbe3b3db8d0714668cde69f577471 db/queries/attachments.sql
58d317ac9e5f2f522b88339f098728d124728e231987c6ff8d5251a37bc33b42 db/queries/attachments_jobs.sql
566d4bfba921ddcfa55e4731eb1e00d4dd794fcf88ae07497f3fe6d557068901 internal/database/postgres.go
5b85a60bb0d1cb156c61bcd3c518c41c955f53c1d12da265b5e3a5dd7a397602 internal/database/redis.go
07391d01cd0480b65cbd8de0941a8ea0797d491b38e537ea9ffe878fc80b3b5d internal/database/sqlc/attachment_variants.sql.go
493a2ba9132786a83c28fbf86295502b624aadebff4672dcffc65dad37689513 internal/database/sqlc/models.go
3ca71a7f007a39987cae838da64303419dd804d2d1b6227676f47177971d76cf internal/database/sqlc/db.go
3dccf4b0ffa918ec5a3b66026b3dab4027456b499486359881000d184944802c internal/database/sqlc/chats.sql.go
92c35b6e7da73e7ed00b274e5e9acc44bdd00c9e285a2b12f74cca8fcdfab54d internal/database/sqlc/attachments.sql.go
a73ab2511cb912fd68db0e5b3b42c2d8e2fe73a6950f080e7104f8474e8c1e36 internal/database/sqlc/message_idempotence.sql.go
9836433d9645e6dd3bd66273ce15a53691f79093016c416fb24ffd085e62831e internal/database/sqlc/messages.sql.go
61d538f3e664d88248fa8d77403e7e7a72cce444e839b665590fb489d59d9dbb internal/database/sqlc/outbox.sql.go
ee1814d651bbdfe62f0685db0d25315c44d408f145f27cf4a631f023c5f8e624 internal/database/sqlc/versioned_pipeline.sql.go
ef74459641b72c53a531e76a830a644036dfa8534a19c393ff5c6e1994b0c1cb internal/database/sqlc/messages_attachments.sql.go
b954abd5b5964df39c20d857a60105513af84c8df68ce1ac9421da541a7800f4 internal/database/sqlc/users.sql.go
2706d561d8dce6e30baddd9921bda9c11ae2b237653839426aba734bf993cb7b internal/database/sqlc/attachments_jobs.sql.go
9e8cd177f6f888c72d19857fd4748fbb2020770c1af772a463fe3d1b813b37ee internal/database/sqlc/chat_members.sql.go
f6669338f64209afcb5fa5f7f255cb6c9155a812c29f5d8fe9330906d85d1957 internal/pipeline/worker.go
128222674808cbad6936757c9d5a413393ba13567d954787e9172f5d77bea735 internal/pipeline/cleanup.go
bee65136137d97bea2240411c96a93cd945b1b9145452416f7a83733a1fd28e7 internal/repository/user.go
c1654fea2ba47734a5403d6f6cbb2fe70fbe7e6e37166ae75f338d3490981de1 internal/repository/redis/session.go
ab56571d1d0b7d13b7d7ff868916f7e506c5a54c29e6bb211b3021ad4bf601ab internal/repository/redis/errors.go
2f92e2808317850ca5873cc26f8e6c4d83104e8fc957a89304fc74ba2f64cbf2 internal/repository/message.go
071285ff4d8033a6512a7256583a1cb32de782d6fd9f1696fac109927f7db699 internal/repository/session.go
eb6770e5cdd96b16a6e9d2116af89a6dd466dfbca4f4c179fe06c30c487e53c4 internal/repository/postgres/user.go
5cc80f3837bad98bb162c20fdf0e9cc801dc14767b18a0ba69a5a1f5ff50bf51 internal/repository/postgres/versioned_pipeline.go
b37ffbae0c4d8f60e7d237c3c7d730cae4cf554d34446eef283e6e10ceddbba8 internal/repository/postgres/pipeline_cleanup.go
ae82a95dbd43a872cf3e32e428e699f767008d07a18ecc041f7601eae2d7ad86 internal/repository/postgres/message.go
53ca3c2143cd98daefa924fc4fa4df1d9cf87a120332e42727a4159b87343ea8 internal/repository/postgres/pipeline.go
04dc48cffea2f6adad7659e60649d0f6ecd724b1ccb82f7fea78eace8624d097 internal/repository/postgres/utils.go
683b6a5a20e0e37305d178700fb6e2e5f47b1b81acaded210b5d3eaa32e29fb1 internal/repository/postgres/attachment.go
d757c579e5b6599ab9366aef46fd59f9652247f3d947615aa46c919746b65ea5 internal/repository/postgres/chat.go
580e5e599f7af8a403404c6850986e88efceda6393d21e772ac108be34004ba9 internal/repository/postgres/errors.go
62e75dbb4ed1cb966abbafc74530a676409ffa0b016a7943be5281cab1dce45c internal/repository/attachment.go
7ef8dca39ff371f16d351b073199c25f719db22c7a525ab76ea7aca948efa157 internal/repository/chat.go
f5afee6418a23fba5066eadd2946c99eed850d76f24e331193146165d48126bb internal/repository/attachment_tx.go
9e3f77340ff87e41edc87380f795759701980685478737e7e4e1f9737e2634fa internal/repository/errors.go
a5f1c30e58149bfddc645ca481a074b0024e82272a9598ec80dd46001bf98c2b internal/broker/rabbitmq.go
d181bce0f003749d813c1254df4c8ebcc1b99ff506b61efa6899da4a499e95f4 internal/transport/http/handler/chat_types.go
86fd65ed176e9a83dc27c1112ff8384bc5aa32a92410290fb505d715ca167775 internal/transport/http/handler/user.go
f175a570d47ca38ce627d0bc88b6f6a2adbf648345229fa961ea0144717744f1 internal/transport/http/handler/auth.go
540065ffc9090848fc25d560f1ad9734d4f0d45cdf0b5907a0f196f9efa8ede3 internal/transport/http/handler/message.go
1abf04652ea0b69bd87321b95e1a921d3c5ad3c04207232c2a9408bf722e550a internal/transport/http/handler/health.go
c94b2e6550037c1ccdd4e1e505965e7b4d85d037be180afe542f38a2b4af3001 internal/transport/http/handler/attachment.go
e5214ebf94ac9b8f37537e1a3fc844a2e8ae4e9b5aa159fe2b6e4da9ab2342f0 internal/transport/http/handler/chat.go
5ffcd182606a77d538efb5f1a51df23b8a324a605b5d226026048d34c409ebdb internal/transport/http/handler/auth_types.go
02ee3031e10fbb279e020f409fbd905ae2640aa306b2c558c3f8e6dcfe9efa80 internal/transport/http/handler/health_types.go
270a9504a3001b1dc02a1f62cdaed12d69af2471340674e50022c609643dcb34 internal/transport/http/handler/message_types.go
a87417ccc7b9ffb468e3ac1df20d5e4adc33d47f31509e37449647ea83706d11 internal/transport/http/handler/attachment_types.go
0c4450be649afe5edf4e8d3ac9baada79deb9e1423ace56b87aa227ebd0eb998 internal/transport/http/handler/user_types.go
413d7fb5f8573a808c2eb4e36becd29b4d7edffe5b096b26fc07ac22a0285430 internal/transport/http/middleware/auth.go
512da6bd2133da34c22a4f1f967b05c308f1ddcdae543a7b5910d178b4a378bc internal/transport/http/middleware/logging.go
f6db0e807659f30b05cae91f73c70d87f359760c57110e0d25b49a65bd0394a5 internal/transport/http/middleware/context.go
59c092950b69f5390427d0c6c1f90d2c33630d87a42bef363aed704960cb3a22 internal/transport/http/httputils/helper.go
6617ceb365903935382cb9592a96a60956d4edbd7c4132530721ddb6ad0e93af internal/transport/http/router.go
21f5d30de769a1817b2d77fc68f9e915d512862316bbb37c7f09baff90cf9407 internal/config/config.go
5a917a0a844d6578ba7e24ae892ae74ed55ea848759e624b94b4356dd7fda996 internal/config/pipeline.go
8f74bef9216a2d12aa734e04fa786eac6b31f1378fe42f729fe1b97495c10487 internal/security/password.go
5d7b5bd8e332104c057d0bb0adf6dce5a3a5bef37d2ba086b69af0cb31ed6a95 internal/observability/sdk.go
32fc41d7bc89d084178f0deae70f1298a896f133ab46f7fb62394ac9f34741e5 internal/observability/signals.go
283c352bfde58afa30ec26997128ac6d6ceebc5e057ef3c112bc6d32f57ab18c internal/observability/propagation.go
c52d3f36aa750b6dcba2b720e342fc1fef69eb99130be697baf93df171c507c4 internal/observability/gauges.go
f8ed8e5486e9278f18a2beb6abb8952aa15d946d2691ecbea5226db9c54e0e71 internal/storage/s3/client.go
6f1f5a7dbd93f65f8101ba80b6fa17bab964744d237746b3ae4ac3650cc9b52c internal/storage/s3/errors.go
ef50236baefe3e3aea8ca64264ae3c2e5fc89c245b9b1d7cfd207d354f0be2c9 internal/storage/object_cleanup_storage.go
15ca1257b9c42f64a42f19a4bbb487e9d2950aed2fdf87959fe6c2d763998b64 internal/storage/object_writer.go
21a5622cd2655663fc24ee485013c1a917279990fd128a5d406b59f5f0ee67cd internal/storage/errors.go
c541dac69672bcd7b89439ae45d63c3e835218f700be7c48c9130a5beeeddbdd internal/imageproc/processor.go
336bbb6ffd99659e77a4f21c0eed49d8a20dd2b79da8fcc045558fdb34acb3ef internal/model/attachment_job.go
4bfd2d78e72f1c80b12541f2abc8500613e8f4d101575bd5d74fac22b52e3617 internal/model/user.go
4cebc4963e2eb104e85a1df6e8a93d6745c9ae42425df2d04bf6319e1d17b12d internal/model/message.go
b455d730e4e3efc95c6d1fe74ab43d86153e4bf863c09efd36bfe57efc669bc2 internal/model/session.go
7ee03b8a3473600e01b83a61369c7212a7cfab4e1a3f982e297ee36dcf9fc9be internal/model/attachment.go
e0213f8360ea106b3627aa50cc87710153ff8e353475b42c2becdb9a32592660 internal/model/chat.go
542051ef3eeff15d340979a615a7e0da3d21ed7ed93bea2547ab52d8e303b5e2 internal/service/user.go
71b8960640866e515e9060159af750c418d97a6373d2616c3b59b764172aa54c internal/service/auth.go
c4e66bb48f7cd26de6fedf4ca7cf751c815dcab30e9442de61f7fe000d3c2367 internal/service/attachment_cleanup.go
9556a18a5679d7e44b5cdcf329488185d2b90cadb3180af926df4bb6ab3f9d0d internal/service/message.go
390320fa296d9637630e6ac0d90c1229002b578a29186cad414896d5f4d1b8a6 internal/service/helper.go
93d786691f371a5e6ad8acb4da90141351d8f4f76df7ebdf70857d7e104a91d9 internal/service/attachment.go
bc7c8bea0066d8bbd926f78d2c100f0bd9a48c29fb542c5cfa3f2ae415299ef5 internal/service/chat.go
f7dbf60b14f9e82cc018bdd29482d71786a30c32bfb5d1ed1808342c9fd36cdb internal/service/attachment_reconcile.go
fb5aa4f7be76af14ab81629eceb02e1c3a1bafd8bfeb036e7ee5ccd3940d64da internal/service/errors.go
c607c83cfeedd8e29a678023b133299dcb5ff754362717d43667db3d84641601 internal/fault/pause.go
baa4b0b6fa4bd2a76b194b672ef2ca8575858ef6abc7ddfbede3eec3534bc8ce infra/s3/bootstrap.sh
895a178e7354de61a5d40a2fe18b7b7c6d1d3d5faf0e5a5faa72783e7b983cb8 infra/haproxy.cfg
ca1f38f2f75f41d1eab7ff2200f17af9fb0999e32eb3c8fd32861966810dfd8d infra/otel/loki.yaml
7773dc948acfde585519bee38c555a8b19710265ed93ce9ff9ad429892b89df7 infra/otel/alerts.yaml
448a5ad110746d41d93aaeb5b2cd6b070cf765427e8ef77563a661e09233f0e0 infra/otel/prometheus.yaml
dc6598890e91c1992c0fa536eddc9f5bc2e786cfc327665551729983e0a54b24 infra/otel/runbook.md
f14185fb7c029a3d35388c19072e56a6881aebb1b83e3a937a44e71fa642acb6 infra/otel/grafana/provisioning/datasources/messenger.yaml
673ac92a82c26caa47723a3c5359204417c85330983ff13f4eb9c5fcab013af1 infra/otel/grafana/provisioning/dashboards/messenger.yaml
9937aba6adb530da8a0928d37918847bbaef100a445db07855ed5467fd1b0281 infra/otel/grafana/dashboards/messenger.json
4a8937430ba638c32dc8d97c36e93ac19f0b3785099f2df1e22b234d109b0f95 infra/otel/README.md
427f54f25412124eb82dfe169b3cb2ee7e018bc4256c7b839a6814722df06c12 infra/otel/collector.yaml
3185ded54a1221207ecec0ee3c496b3d45614eebab63471e948a32a51a41d8c2 infra/otel/tempo.yaml
0e804fa51debba4c38d3cc84abf4704897665e0335b272060363f03aadd6354f infra/README.md
b75281f51a4c0d83904bf15c509cc79c6e8b177d05c6c11a412b6ac16d59cf85 infra/imagemagick/policy.xml
1410205b161cbbed45dd024c37ec45b550751a54a4065cecf23a81e4a755d837 compose.yaml
7584dc5927d3324e0f415b913ad8f756e9baa863fe02e369bffc34d777eaa9f9 Dockerfile
fd7ac9491c0c3c663b083c41a762fb38bc2d212fa4a2ed3e6a0702af72108709 Dockerfile.example
84b41bac810e3add9996a3a0120b3e611f2e32adbea4e447a580cbc0f88d6c4b Dockerfile.migrate
a5b5c9aea21573f57e604428196c5ce3b7b5c0e15ecae81c134b3b6f8950ec26 Dockerfile.worker
a59216b0859592fa49353942fa4ebd16d9e098ecdd41147f8f6b52fe1804835d go.mod
b01795b85f5d8c4a0a48925e956e2bc747bb37a96b5e2f8531476a842ad5d7b9 go.sum
+241
View File
@@ -0,0 +1,241 @@
{
"results": {
"scoped-compose": {
"status": "PASS",
"evidence": "loki-quiet.jsonl.gz",
"t_s": 4.056418
},
"backend-readiness": {
"status": "PASS",
"evidence": "loki-quiet.jsonl.gz",
"t_s": 19.38088
},
"public-smoke": {
"status": "PASS",
"evidence": "loki-final.jsonl.gz",
"t_s": 7.548447
},
"provisioning": {
"status": "PASS",
"evidence": "loki-final.jsonl.gz",
"t_s": 7.668063
},
"trace-retry": {
"status": "PASS",
"evidence": "final.jsonl.gz",
"t_s": 71.889912
},
"metrics": {
"status": "PASS",
"evidence": "final.jsonl.gz",
"t_s": 79.172357
},
"worker-outage": {
"status": "PASS",
"evidence": "worker-rerun.jsonl.gz",
"t_s": 68.043544
},
"db-outage": {
"status": "PASS",
"evidence": "final.jsonl.gz",
"t_s": 137.340855
},
"loki-outage": {
"status": "PASS",
"evidence": "loki-quiet.jsonl.gz",
"t_s": 74.885014
}
},
"alerts": [
{
"event": "alert_transition",
"phase": "db-outage",
"t_s": 113.021074,
"name": "HighAPIErrorRate",
"state": "firing",
"seconds": 32.72850495799503
},
{
"event": "alert_transition",
"phase": "db-outage",
"t_s": 137.206116,
"name": "HighAPIErrorRate",
"state": "resolved",
"seconds": 23.813540999995894
},
{
"event": "alert_transition",
"phase": "worker-outage",
"t_s": 58.282739,
"name": "GrowingQueueLag",
"state": "firing",
"seconds": 39.821040167007595
},
{
"event": "alert_transition",
"phase": "worker-outage",
"t_s": 68.043475,
"name": "GrowingQueueLag",
"state": "resolved",
"seconds": 9.0445497079927
}
],
"db": {
"event": "db_errors",
"phase": "db-outage",
"t_s": 113.025066,
"count": 23,
"elapsed_s": [
1.0122738749923883,
1.0136010000132956,
1.0066660839947872,
1.0647313329973258,
1.0637415000092005,
1.0244076670060167,
1.0293545829918003,
1.0129166250117123,
1.0139167909946991,
1.011540708001121,
1.0182600000116508,
1.0135567909892416,
1.010816124995472,
1.0129806670011021,
1.0097749999986263,
1.0129036669968627,
1.0104306249995716,
1.0163643749983748,
1.0270365830074297,
1.0105538750067353,
1.0157577079953626,
1.0105855829897337,
1.023447416999261
],
"kind": "users-table exclusive lock with healthy SQL connectivity"
},
"histograms": {
"messenger_http_request_duration_seconds": {
"0.5": 0.002197183098591549,
"0.95": 0.20909090909090886,
"0.99": 0.9999999999999993
},
"messenger_job_duration_seconds": {
"0.5": 0.375,
"0.95": 98.99999999999999,
"0.99": 115.8
},
"messenger_attachment_time_to_ready_seconds": {
"0.5": 1.5,
"0.95": 104.99999999999999,
"0.99": 117.0
}
},
"trace": {
"trace_id": "117fb860af9afead75e086dd224dc3f3",
"span_count": 43,
"roles": {
"server": [
"033181f9ec0d505e"
],
"producer": [
"92738b419e089622",
"9e3979899a512f6b"
],
"consumer_error": [
"0ce981115c578fdb"
],
"consumer_success": [
"e4eaf51e02978a35"
],
"s3": [
"d0fe4c64c8802873",
"fbf208abc9acad68",
"8fb1c65ab0adee8b",
"6e0093008e42bfd5",
"630ed562ef2ff0ff"
],
"postgres": [
"edad82ff768e568d",
"399d1cc7721a8d62",
"9fcb2e4ec4fa5213",
"591f6fa126cc2a86",
"3318e29280f91e83",
"cbcd5e82838679e4",
"fc3a0681f8ba3a32",
"a37814110380ba1a",
"9f816e5ca0bbddc5",
"e565df310156f969",
"f37888f8e0cbcf08",
"67730935285e41d5",
"bb6218ee1ab0d63f",
"5ca43f6e214a1096"
],
"failed_s3": [
"4912ecf2e97dd446"
]
}
},
"privacy": {
"event": "loki_correlation",
"phase": "trace-retry",
"t_s": 71.791617,
"matched": {
"messenger-api": {
"timestamp_ns": "1794810562106030543",
"trace_id": "117fb860af9afead75e086dd224dc3f3",
"span_id": "033181f9ec0d505e",
"request_id": "930ba702-f6ca-485f-83a2-eb2737b2d48c",
"severity": "INFO",
"event": "http.request"
},
"messenger-worker": {
"timestamp_ns": "1794810625211766795",
"trace_id": "117fb860af9afead75e086dd224dc3f3",
"span_id": "e4eaf51e02978a35",
"request_id": null,
"severity": "INFO",
"event": "job.finished"
}
},
"examined": 58,
"privacy_canaries": [
"basic_credentials",
"bearer_token",
"filename",
"image_body",
"message_text",
"registration_password",
"rejected_basic_credentials"
]
},
"loki": {
"event": "loki_outage_verified",
"phase": "loki-outage",
"t_s": 74.884933,
"phase_seconds": 34,
"time_to_loss_s": 8.685361416995875,
"initial_dropped": 24.0,
"dropped_delta": 96.0,
"peak_rss_bytes": 92856320.0,
"peak_queue_size": 16.0,
"hard_limit_bytes": 268435456,
"successful_reads": 278,
"preserved_messages": 8,
"recovered_trace_id": "c69b20b4396c82e6384eee0e74426c58",
"collector": {
"id": "06e8839ea7ce66b2eb98cda8b183e1c31a671a634964fbdea1e7e871955c1465",
"image": "otel/opentelemetry-collector-contrib:0.131.0",
"status": "running",
"started_at": "2026-11-16T06:33:52.029445876Z",
"restarts": 0,
"oom": false,
"memory": 268435456,
"cpu_nanos": 750000000,
"ports": {
"4317/tcp": null,
"4318/tcp": null,
"55678/tcp": null,
"55679/tcp": null
}
}
}
}
Binary file not shown.
+30
View File
@@ -0,0 +1,30 @@
{
"status": "PASS",
"services": [
"prometheus",
"loki",
"tempo",
"grafana"
],
"old_container_ids": {
"prometheus": "e063f54b27c1",
"loki": "179ecc857b80",
"tempo": "e4b0e7e2a3a0",
"grafana": "a6db80031e16"
},
"new_container_ids": {
"prometheus": "ab619a6a89e8",
"loki": "f69a8b780008",
"tempo": "1d8534706f8c",
"grafana": "b6c8635afdbb"
},
"historical_sample_unix": 1794811077.7996218,
"matched_before_and_after": {
"trace_id": "a674a4438e9c0032c63b4f2756839598",
"request_id": "0971f967-996b-4484-bcd5-49c6a8813a03",
"server_span": "1801a6325cc09d20",
"loki_matches": 1,
"prom_completed_registration_counter": 1.0
},
"dashboard_uid": "lab6-messenger"
}
+66
View File
@@ -0,0 +1,66 @@
module highload
go 1.27.1
require (
github.com/aws/aws-sdk-go-v2 v1.48.0
github.com/aws/aws-sdk-go-v2/config v1.33.18
github.com/aws/aws-sdk-go-v2/credentials v1.21.0
github.com/aws/aws-sdk-go-v2/service/s3 v1.114.10
github.com/aws/smithy-go v1.28.9
github.com/google/uuid v1.6.0
github.com/jackc/pgx/v5 v5.11.0
github.com/pressly/goose/v3 v3.26.0
github.com/rabbitmq/amqp091-go v1.10.0
github.com/redis/go-redis/v9 v9.23.0
go.opentelemetry.io/otel v1.38.0
go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.14.0
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.38.0
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.38.0
go.opentelemetry.io/otel/log v0.14.0
go.opentelemetry.io/otel/metric v1.38.0
go.opentelemetry.io/otel/sdk v1.38.0
go.opentelemetry.io/otel/sdk/log v0.14.0
go.opentelemetry.io/otel/sdk/metric v1.38.0
go.opentelemetry.io/otel/trace v1.38.0
golang.org/x/crypto v0.57.0
)
require (
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.27 // indirect
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.9 // indirect
github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.12 // indirect
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.12 // indirect
github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.12 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.25 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.13 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.12 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.12 // indirect
github.com/aws/aws-sdk-go-v2/service/signin v1.13.1 // indirect
github.com/aws/aws-sdk-go-v2/service/sso v1.39.5 // indirect
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.44.5 // indirect
github.com/aws/aws-sdk-go-v2/service/sts v1.52.0 // indirect
github.com/cenkalti/backoff/v5 v5.0.3 // indirect
github.com/cespare/xxhash/v2 v2.3.0 // indirect
github.com/go-logr/logr v1.4.3 // indirect
github.com/go-logr/stdr v1.2.2 // indirect
github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.2 // indirect
github.com/jackc/pgpassfile v1.0.0 // indirect
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect
github.com/jackc/puddle/v2 v2.2.2 // indirect
github.com/mfridman/interpolate v0.0.2 // indirect
github.com/sethvargo/go-retry v0.3.0 // indirect
go.opentelemetry.io/auto/sdk v1.1.0 // indirect
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.38.0 // indirect
go.opentelemetry.io/proto/otlp v1.7.1 // indirect
go.uber.org/atomic v1.12.0 // indirect
go.uber.org/multierr v1.11.0 // indirect
golang.org/x/net v0.58.0 // indirect
golang.org/x/sync v0.23.0 // indirect
golang.org/x/sys v0.48.0 // indirect
golang.org/x/text v0.42.0 // indirect
google.golang.org/genproto/googleapis/api v0.0.0-20250825161204-c5933d9347a5 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20250825161204-c5933d9347a5 // indirect
google.golang.org/grpc v1.75.0 // indirect
google.golang.org/protobuf v1.36.8 // indirect
)
+162
View File
@@ -0,0 +1,162 @@
github.com/aws/aws-sdk-go-v2 v1.48.0 h1:tFU7EJVm7MfyaXLsXdYTZ0Vd6vwt9OljCyqq0gqk0Mc=
github.com/aws/aws-sdk-go-v2 v1.48.0/go.mod h1:oeKU5p+TQBnpeLmU42WZLWvzv+ORLn+74n4QVLnm/Z0=
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.27 h1:3dMP9eEmHVosFN1QiD24IJBKVY8m6iUfVYXqRCT118Q=
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.27/go.mod h1:KH4PKcqtjy1m0656eqQV2v53xE04ddJzuNloEKqlHkI=
github.com/aws/aws-sdk-go-v2/config v1.33.18 h1:WC0C4jM/tewXG8DWXnoxXnusJPZTryPazqk2k/zgd2s=
github.com/aws/aws-sdk-go-v2/config v1.33.18/go.mod h1:0kRpFgsnD59mC2bbbyOTVpaZ9RnXW/pFkqlfiwbgZaU=
github.com/aws/aws-sdk-go-v2/credentials v1.21.0 h1:OViwhqeipkQzSHuO2eQDUCdLrI6r3nzGxFUyM8GqnCs=
github.com/aws/aws-sdk-go-v2/credentials v1.21.0/go.mod h1:OffaVQod3fyZzhZJyPtw11WpHPF1uIHADc+/MYlXmbA=
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.9 h1:oH1E0tjN0g+ZRusbX1dsuat4FJ0z05l8hWaycSbDuNc=
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.9/go.mod h1:3ZaQGigIoaS45x9HgnLZ4sP475Fs7QoxU3IsZX/J3h4=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.12 h1:W0yUUzKBA8SofGQ0yZafEUuZq9UxfvijRgdc6vcRQPA=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.12/go.mod h1:xwKv6vwBNdpPFOxEm/ld4IODZK2s/1ksHvdBqLfH6Gs=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.12 h1:lqSi8qvAnKC1vfosImdmOsuiJLr/jdybZX5299beB6Q=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.12/go.mod h1:rqPQE2vqkW9YHx5+Gp55sx1wsTM4xtGQLl+TKMjxxrs=
github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.12 h1:uvX2yaL+aMjkmnrLPPlR/zwjoKFRMoNhhJfeGjfk6qc=
github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.12/go.mod h1:dicT+2eNhHHqGkxHO7LoXw3QghIUGuzgdmFqaSTopgI=
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.25 h1:rKwkY3Z+fvt2W0msjdNs6uNewPmXzPayQ961oyuWhes=
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.25/go.mod h1:dBuAadQKd1F4Q4ZOg8gnFmuSHMRE40Pe3IfFwbtC4lE=
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.13 h1:U5HZ8FpTr6tyvulf1nT11Nw4KG1CAsXkuQGdUrIjZ/E=
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.13/go.mod h1:dUfvCU1NaE+b3vqNENDiJncUQW7Djl5pmgUlcjesVqk=
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.12 h1:wzAqMZ2O4+ij9HQqIrk+P9PBGacE0wOTuWB7Mo97//I=
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.12/go.mod h1:/0TfyruByHsOlphzP26xDTRe2DKD0J1I1Uyy2TyP8EI=
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.12 h1:H0IaQayT3TLiGIHiNnleQZTbndDlx7LGZ0NWZ5mDwjA=
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.12/go.mod h1:YuJJNxaOjnknLIrscBmkLhIpnmg9LzHd25sQRKUboGU=
github.com/aws/aws-sdk-go-v2/service/s3 v1.114.10 h1:WBJb2VxRzzKnuN94YWqZPiu5a5zc3IleFqgZnmVCyvU=
github.com/aws/aws-sdk-go-v2/service/s3 v1.114.10/go.mod h1:8NhAIBVYkVHq0hIbbzujdoou0VkKwMdG+De9e5oX6qI=
github.com/aws/aws-sdk-go-v2/service/signin v1.13.1 h1:ssrDA6AKfXEc3h0Pf4EI4vQ8sN5Zd136SrU9Tjr0XPc=
github.com/aws/aws-sdk-go-v2/service/signin v1.13.1/go.mod h1:LdwHjKqPE9VCC7sTlWcOAizZx3ZJYjmfwC/EW2tIIAE=
github.com/aws/aws-sdk-go-v2/service/sso v1.39.5 h1:ANAoa+qt3/9kXnvQVgUyxcTzhzZf4DqaBS+M2DQrELU=
github.com/aws/aws-sdk-go-v2/service/sso v1.39.5/go.mod h1:vsMi+SSR7xGQXtV430LVi49qbv/R+U+upHHSp1W/+QI=
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.44.5 h1:uIOBLMhef4N16WguadoHKuBy9etdz+RKTeqzchXErAI=
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.44.5/go.mod h1:NwWmkFLtsTVyXQpbzPOlYLwEoANkdCFjJRtrNcbg/vY=
github.com/aws/aws-sdk-go-v2/service/sts v1.52.0 h1:IaBLmJWiIEtKkjXMgMbYGZk7WKXbYA8148wCMCXq9hU=
github.com/aws/aws-sdk-go-v2/service/sts v1.52.0/go.mod h1:MG6N8U9+lY4LqchrlHPqCOSr7hyxw70XU0Et2YR3eNA=
github.com/aws/smithy-go v1.28.9 h1:zkZ058OpNX0iGTK24b+ZUEQSMc11CBA6oO7xtKZfzkY=
github.com/aws/smithy-go v1.28.9/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc=
github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs=
github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c=
github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA=
github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0=
github.com/cenkalti/backoff/v5 v5.0.3 h1:ZN+IMa753KfX5hd8vVaMixjnqRZ3y8CuJKRKj1xcsSM=
github.com/cenkalti/backoff/v5 v5.0.3/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw=
github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI=
github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek=
github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps=
github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.2 h1:8Tjv8EJ+pM1xP8mK6egEbD1OgnVTyacbefKhmbLhIhU=
github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.2/go.mod h1:pkJQ2tZHJ0aFOVEEot6oZmaVEZcRme73eIFmhiVuRWs=
github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM=
github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg=
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo=
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM=
github.com/jackc/pgx/v5 v5.11.0 h1:IzBBtyK9AHqf98cctWFifYSci2hgQR/cd56wB4p+ogg=
github.com/jackc/pgx/v5 v5.11.0/go.mod h1:mal1tBGAFfLHvZzaYh77YS/eC6IX9OWbRV1QIIM0Jn4=
github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo=
github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4=
github.com/klauspost/cpuid/v2 v2.4.0 h1:S6Hrbc7+ywsr0r+RLapfGBHfyefhCTwEh3A0tV913Dw=
github.com/klauspost/cpuid/v2 v2.4.0/go.mod h1:19jmZ9mjzoF//ddRSUsv0zfBTJWh3QJh9FNxZTMrGxU=
github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
github.com/mfridman/interpolate v0.0.2 h1:pnuTK7MQIxxFz1Gr+rjSIx9u7qVjf5VOoM/u6BbAxPY=
github.com/mfridman/interpolate v0.0.2/go.mod h1:p+7uk6oE07mpE/Ik1b8EckO0O4ZXiGAfshKBWLUM9Xg=
github.com/ncruces/go-strftime v0.1.9 h1:bY0MQC28UADQmHmaF5dgpLmImcShSi2kHU9XLdhx/f4=
github.com/ncruces/go-strftime v0.1.9/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/pressly/goose/v3 v3.26.0 h1:KJakav68jdH0WDvoAcj8+n61WqOIaPGgH0bJWS6jpmM=
github.com/pressly/goose/v3 v3.26.0/go.mod h1:4hC1KrritdCxtuFsqgs1R4AU5bWtTAf+cnWvfhf2DNY=
github.com/rabbitmq/amqp091-go v1.10.0 h1:STpn5XsHlHGcecLmMFCtg7mqq0RnD+zFr4uzukfVhBw=
github.com/rabbitmq/amqp091-go v1.10.0/go.mod h1:Hy4jKW5kQART1u+JkDTF9YYOQUHXqMuhrgxOEeS7G4o=
github.com/redis/go-redis/v9 v9.23.0 h1:/TTVdJa8BRNsybwwGGnmiYktHNlUXtbOQxlWpHBPL3s=
github.com/redis/go-redis/v9 v9.23.0/go.mod h1:EWP2UUk+XDaI1s5nQB5mablZN6vccXlmBQTyfnF87D0=
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE=
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
github.com/sethvargo/go-retry v0.3.0 h1:EEt31A35QhrcRZtrYFDTBg91cqZVnFL2navjDrah2SE=
github.com/sethvargo/go-retry v0.3.0/go.mod h1:mNX17F0C/HguQMyMyJxcnU471gOZGxCLyYaFyAZraas=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.12.1 h1:EuwCh5fleGS7H32xRwO3wRGT7DxrDhLAT6FF8MpWDWE=
github.com/stretchr/testify v1.12.1/go.mod h1:MDEgiDPPsNp5cuIrHPPCyornHKgEVbtFUmoNlxoYthg=
github.com/zeebo/xxh3 v1.1.0 h1:s7DLGDK45Dyfg7++yxI0khrfwq9661w9EN78eP/UZVs=
github.com/zeebo/xxh3 v1.1.0/go.mod h1:IisAie1LELR4xhVinxWS5+zf1lA4p0MW4T+w+W07F5s=
go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA=
go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A=
go.opentelemetry.io/otel v1.38.0 h1:RkfdswUDRimDg0m2Az18RKOsnI8UDzppJAtj01/Ymk8=
go.opentelemetry.io/otel v1.38.0/go.mod h1:zcmtmQ1+YmQM9wrNsTGV/q/uyusom3P8RxwExxkZhjM=
go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.14.0 h1:QQqYw3lkrzwVsoEX0w//EhH/TCnpRdEenKBOOEIMjWc=
go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.14.0/go.mod h1:gSVQcr17jk2ig4jqJ2DX30IdWH251JcNAecvrqTxH1s=
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.38.0 h1:Oe2z/BCg5q7k4iXC3cqJxKYg0ieRiOqF0cecFYdPTwk=
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.38.0/go.mod h1:ZQM5lAJpOsKnYagGg/zV2krVqTtaVdYdDkhMoX6Oalg=
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.38.0 h1:GqRJVj7UmLjCVyVJ3ZFLdPRmhDUp2zFmQe3RHIOsw24=
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.38.0/go.mod h1:ri3aaHSmCTVYu2AWv44YMauwAQc0aqI9gHKIcSbI1pU=
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.38.0 h1:aTL7F04bJHUlztTsNGJ2l+6he8c+y/b//eR0jjjemT4=
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.38.0/go.mod h1:kldtb7jDTeol0l3ewcmd8SDvx3EmIE7lyvqbasU3QC4=
go.opentelemetry.io/otel/log v0.14.0 h1:2rzJ+pOAZ8qmZ3DDHg73NEKzSZkhkGIua9gXtxNGgrM=
go.opentelemetry.io/otel/log v0.14.0/go.mod h1:5jRG92fEAgx0SU/vFPxmJvhIuDU9E1SUnEQrMlJpOno=
go.opentelemetry.io/otel/metric v1.38.0 h1:Kl6lzIYGAh5M159u9NgiRkmoMKjvbsKtYRwgfrA6WpA=
go.opentelemetry.io/otel/metric v1.38.0/go.mod h1:kB5n/QoRM8YwmUahxvI3bO34eVtQf2i4utNVLr9gEmI=
go.opentelemetry.io/otel/sdk v1.38.0 h1:l48sr5YbNf2hpCUj/FoGhW9yDkl+Ma+LrVl8qaM5b+E=
go.opentelemetry.io/otel/sdk v1.38.0/go.mod h1:ghmNdGlVemJI3+ZB5iDEuk4bWA3GkTpW+DOoZMYBVVg=
go.opentelemetry.io/otel/sdk/log v0.14.0 h1:JU/U3O7N6fsAXj0+CXz21Czg532dW2V4gG1HE/e8Zrg=
go.opentelemetry.io/otel/sdk/log v0.14.0/go.mod h1:imQvII+0ZylXfKU7/wtOND8Hn4OpT3YUoIgqJVksUkM=
go.opentelemetry.io/otel/sdk/log/logtest v0.14.0 h1:Ijbtz+JKXl8T2MngiwqBlPaHqc4YCaP/i13Qrow6gAM=
go.opentelemetry.io/otel/sdk/log/logtest v0.14.0/go.mod h1:dCU8aEL6q+L9cYTqcVOk8rM9Tp8WdnHOPLiBgp0SGOA=
go.opentelemetry.io/otel/sdk/metric v1.38.0 h1:aSH66iL0aZqo//xXzQLYozmWrXxyFkBJ6qT5wthqPoM=
go.opentelemetry.io/otel/sdk/metric v1.38.0/go.mod h1:dg9PBnW9XdQ1Hd6ZnRz689CbtrUp0wMMs9iPcgT9EZA=
go.opentelemetry.io/otel/trace v1.38.0 h1:Fxk5bKrDZJUH+AMyyIXGcFAPah0oRcT+LuNtJrmcNLE=
go.opentelemetry.io/otel/trace v1.38.0/go.mod h1:j1P9ivuFsTceSWe1oY+EeW3sc+Pp42sO++GHkg4wwhs=
go.opentelemetry.io/proto/otlp v1.7.1 h1:gTOMpGDb0WTBOP8JaO72iL3auEZhVmAQg4ipjOVAtj4=
go.opentelemetry.io/proto/otlp v1.7.1/go.mod h1:b2rVh6rfI/s2pHWNlB7ILJcRALpcNDzKhACevjI+ZnE=
go.uber.org/atomic v1.12.0 h1:BvcXdFKuviU4fTL/f+SxdQ5qJX/Jix8pAkgdUcb3XOE=
go.uber.org/atomic v1.12.0/go.mod h1:I6c4cg+6HCxRjfjSsYtApoFILnpc0CGUdGkXVqbYVNk=
go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=
go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw=
go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg=
golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M=
golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA=
golang.org/x/exp v0.0.0-20250620022241-b7579e27df2b h1:M2rDM6z3Fhozi9O7NWsxAkg/yqS/lQJ6PmkyIV3YP+o=
golang.org/x/exp v0.0.0-20250620022241-b7579e27df2b/go.mod h1:3//PLf8L/X+8b4vuAfHzxeRUl04Adcb341+IGKfnqS8=
golang.org/x/net v0.58.0 h1:ynWG7rqYi4ccpTEuPZ2QGWHktVEM9DMCj9yzDE0Q7To=
golang.org/x/net v0.58.0/go.mod h1:YwCddHnFlT7eLQqVprV19OnhLGtc5xOKgE0RyqgfWAU=
golang.org/x/sync v0.23.0 h1:KameEIfc1IkluZyXWLn39Wd4tURc6GbCiISGiZm2bQk=
golang.org/x/sync v0.23.0/go.mod h1:sUUOizhqBxiL6pEWpqNLUiaJn1ShEbZ6BBqskPbjZm0=
golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo=
golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og=
golang.org/x/text v0.42.0 h1:JbOZXgfeCPU9gacVtYliJqOhD+zhrEqK4LfdpmlUZqI=
golang.org/x/text v0.42.0/go.mod h1:ojzP1Z+2QtioaF8DTtO8K5q7JWVVYwZKenzujK0Zd0E=
gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk=
gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E=
google.golang.org/genproto/googleapis/api v0.0.0-20250825161204-c5933d9347a5 h1:BIRfGDEjiHRrk0QKZe3Xv2ieMhtgRGeLcZQ0mIVn4EY=
google.golang.org/genproto/googleapis/api v0.0.0-20250825161204-c5933d9347a5/go.mod h1:j3QtIyytwqGr1JUDtYXwtMXWPKsEa5LtzIFN1Wn5WvE=
google.golang.org/genproto/googleapis/rpc v0.0.0-20250825161204-c5933d9347a5 h1:eaY8u2EuxbRv7c3NiGK0/NedzVsCcV6hDuU5qPX5EGE=
google.golang.org/genproto/googleapis/rpc v0.0.0-20250825161204-c5933d9347a5/go.mod h1:M4/wBTSeyLxupu3W3tJtOgB14jILAS/XWPSSa3TAlJc=
google.golang.org/grpc v1.75.0 h1:+TW+dqTd2Biwe6KKfhE5JpiYIBWq865PhKGSXiivqt4=
google.golang.org/grpc v1.75.0/go.mod h1:JtPAzKiq4v1xcAB2hydNlWI2RnF85XXcV0mhKXr2ecQ=
google.golang.org/protobuf v1.36.8 h1:xHScyCOEuuwZEc6UtSOvPbAT4zRh0xcNRYekJwfqyMc=
google.golang.org/protobuf v1.36.8/go.mod h1:fuxRtAxBytpl4zzqUh6/eyUujkJdNiuEkXntxiD/uRU=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
modernc.org/libc v1.66.3 h1:cfCbjTUcdsKyyZZfEUKfoHcP3S0Wkvz3jgSzByEWVCQ=
modernc.org/libc v1.66.3/go.mod h1:XD9zO8kt59cANKvHPXpx7yS2ELPheAey0vjIuZOhOU8=
modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU=
modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg=
modernc.org/memory v1.11.0 h1:o4QC8aMQzmcwCK3t3Ux/ZHmwFPzE6hf2Y5LbkRs+hbI=
modernc.org/memory v1.11.0/go.mod h1:/JP4VbVC+K5sU2wZi9bHoq2MAkCnrt2r98UGeSK7Mjw=
modernc.org/sqlite v1.38.2 h1:Aclu7+tgjgcQVShZqim41Bbw9Cho0y/7WzYptXqkEek=
modernc.org/sqlite v1.38.2/go.mod h1:cPTJYSlgg3Sfg046yBShXENNtPrWrDX8bsbAQBzgQ5E=
+28
View File
@@ -0,0 +1,28 @@
global
log stdout format raw local0
stats socket ipv4@127.0.0.1:9999 level admin
defaults
mode http
log global
option httplog
timeout connect 2s
timeout client 30s
timeout server 30s
timeout http-request 10s
resolvers docker
nameserver docker 127.0.0.11:53
accepted_payload_size 8192
hold valid 1s
frontend api
bind *:8080
default_backend api_replicas
backend api_replicas
balance roundrobin
timeout check 3s
option httpchk GET /health/ready
http-check expect status 200
server-template api 4 api:8080 check inter 1s fall 2 rise 2 resolvers docker init-addr none
+16
View File
@@ -0,0 +1,16 @@
<?xml version="1.0" encoding="UTF-8"?>
<policymap>
<policy domain="resource" name="thread" value="1"/>
<policy domain="resource" name="memory" value="256MiB"/>
<policy domain="resource" name="map" value="256MiB"/>
<policy domain="resource" name="disk" value="512MiB"/>
<policy domain="resource" name="time" value="45"/>
<policy domain="resource" name="list-length" value="2"/>
<policy domain="delegate" rights="none" pattern="*"/>
<policy domain="filter" rights="none" pattern="*"/>
<policy domain="path" rights="none" pattern="@*"/>
<policy domain="module" rights="none" pattern="*"/>
<policy domain="module" rights="read | write" pattern="{JPEG,PNG,WEBP}"/>
<policy domain="coder" rights="none" pattern="*"/>
<policy domain="coder" rights="read | write" pattern="{JPEG,JPG,PNG,WEBP}"/>
</policymap>
+11 -4
View File
@@ -1,7 +1,14 @@
# Конфигурация наблюдаемости — реализовать # Конфигурация наблюдаемости
Добавьте `collector.yaml` с OTLP receivers, processors и exporters выбранной версии Collector. Укажите явные bind endpoints, внутренние сети, memory limits/batch и поведение при backpressure по уровню. Сопоставьте все три pipeline с реально развёрнутыми backend. `collector.yaml` принимает OTLP/HTTP от API, relay и worker и отправляет
сигналы в Prometheus, Loki и Tempo. `prometheus.yaml` настраивает сбор
метрик приложения и самого Collector. `alerts.yaml` содержит два правила
с последовательностью диагностики в [runbook](runbook.md).
Добавьте конфигурации storage, Grafana provisioning/datasources, provisioning/dashboards и импортируемые dashboard JSON. На 4 — alert rules и runbooks. Пустой dashboard или debug exporter, который только печатает данные в stdout, не заменяют хранилища/доказательства из `contracts/observability.md`. `grafana/provisioning` подключает три источника и регистрирует dashboard
[Messenger observability](grafana/dashboards/messenger.json). JSON можно
импортировать в Grafana, где доступны те же UID источников. Compose
публикует интерфейсы чтения только на loopback. Приёмник OTLP и scrape
endpoints находятся во внутренней сети.
Не публикуйте OTLP/metrics/административные ports в общую сеть без необходимости. В lab7 реализуйте TLS/mTLS по trust matrix; экспорт Authorization и message bodies запрещён. Команды запуска и описание метрик приведены в [отчёте](../../REPORT.md).
+27
View File
@@ -0,0 +1,27 @@
groups:
- name: messenger
rules:
- alert: HighAPIErrorRate
expr: |
(
sum(rate(messenger_http_requests_total{route!~"/health/(live|ready)",status_class="5xx"}[30s]))
/ clamp_min(sum(rate(messenger_http_requests_total{route!~"/health/(live|ready)"}[30s])), 0.001)
) > 0.10
and sum(increase(messenger_http_requests_total{route!~"/health/(live|ready)"}[30s])) >= 5
for: 15s
labels:
severity: warning
annotations:
summary: Business API 5xx ratio above 10 percent
runbook: See infra/otel/runbook.md, PostgreSQL degradation.
- alert: GrowingQueueLag
expr: |
max by (queue) (messenger_queue_oldest_age_seconds) > 15
and on (queue) max by (queue) (messenger_queue_depth) >= 1
and on (queue) min by (queue) (messenger_queue_sample_age_seconds) < 8
for: 15s
labels:
severity: warning
annotations:
summary: A durable queued job has waited longer than 15 seconds
runbook: See infra/otel/runbook.md, stopped worker.
+68
View File
@@ -0,0 +1,68 @@
receivers:
otlp:
protocols:
http:
endpoint: 0.0.0.0:4318
processors:
memory_limiter:
check_interval: 1s
limit_mib: 96
spike_limit_mib: 24
batch:
timeout: 1s
send_batch_size: 64
send_batch_max_size: 128
exporters:
prometheus:
endpoint: 0.0.0.0:9464
metric_expiration: 1m
resource_to_telemetry_conversion:
enabled: true
otlphttp/loki:
endpoint: http://loki:3100/otlp
timeout: 2s
sending_queue:
enabled: true
num_consumers: 1
queue_size: 16
retry_on_failure:
initial_interval: 1s
max_interval: 2s
max_elapsed_time: 5s
otlphttp/tempo:
endpoint: http://tempo:4318
timeout: 2s
sending_queue:
enabled: true
num_consumers: 1
queue_size: 16
retry_on_failure:
initial_interval: 1s
max_interval: 2s
max_elapsed_time: 5s
service:
telemetry:
metrics:
readers:
- pull:
exporter:
prometheus:
host: 0.0.0.0
port: 8888
without_units: true
pipelines:
metrics:
receivers: [otlp]
processors: [memory_limiter, batch]
exporters: [prometheus]
traces:
receivers: [otlp]
processors: [memory_limiter, batch]
exporters: [otlphttp/tempo]
logs:
receivers: [otlp]
processors: [memory_limiter, batch]
exporters: [otlphttp/loki]
@@ -0,0 +1,572 @@
{
"id": null,
"uid": "lab6-messenger",
"title": "Messenger observability",
"tags": [
"lab6"
],
"timezone": "browser",
"schemaVersion": 39,
"version": 1,
"refresh": "5s",
"time": {
"from": "now-15m",
"to": "now"
},
"panels": [
{
"id": 1,
"type": "timeseries",
"title": "API requests per second",
"description": "Per-second rate across both API replicas, excluding health checks. Window 1 minute.",
"datasource": {
"type": "prometheus",
"uid": "lab-prometheus"
},
"gridPos": {
"h": 7,
"w": 12,
"x": 0,
"y": 0
},
"fieldConfig": {
"defaults": {
"unit": "reqps",
"min": 0
},
"overrides": []
},
"targets": [
{
"refId": "A",
"expr": "sum(rate(messenger_http_requests_total{route!~\"/health/(live|ready)\"}[1m]))",
"legendFormat": "All business requests"
},
{
"refId": "B",
"expr": "sum(rate(messenger_http_requests_total{route!~\"/health/(live|ready)\",status_class=\"5xx\"}[1m]))",
"legendFormat": "5xx"
}
]
},
{
"id": 2,
"type": "timeseries",
"title": "API error ratio",
"description": "Server errors divided by all business responses. An empty window has no ratio.",
"datasource": {
"type": "prometheus",
"uid": "lab-prometheus"
},
"gridPos": {
"h": 7,
"w": 12,
"x": 12,
"y": 0
},
"fieldConfig": {
"defaults": {
"unit": "percentunit",
"min": 0,
"max": 1
},
"overrides": []
},
"targets": [
{
"refId": "A",
"expr": "(sum(rate(messenger_http_requests_total{route!~\"/health/(live|ready)\",status_class=\"5xx\"}[1m])) or 0 * sum(rate(messenger_http_requests_total{route!~\"/health/(live|ready)\"}[1m]))) / (sum(rate(messenger_http_requests_total{route!~\"/health/(live|ready)\"}[1m])) > 0)",
"legendFormat": "5xx / requests"
}
]
},
{
"id": 3,
"type": "timeseries",
"title": "API response duration",
"description": "p50, p95 and p99 from cumulative histogram buckets, aggregated across replicas.",
"datasource": {
"type": "prometheus",
"uid": "lab-prometheus"
},
"gridPos": {
"h": 7,
"w": 12,
"x": 0,
"y": 7
},
"fieldConfig": {
"defaults": {
"unit": "s",
"min": 0
},
"overrides": []
},
"targets": [
{
"refId": "A",
"expr": "histogram_quantile(0.5, sum by (le) (rate(messenger_http_request_duration_seconds_bucket{route!~\"/health/(live|ready)\"}[1m])))",
"legendFormat": "p50"
},
{
"refId": "B",
"expr": "histogram_quantile(0.95, sum by (le) (rate(messenger_http_request_duration_seconds_bucket{route!~\"/health/(live|ready)\"}[1m])))",
"legendFormat": "p95"
},
{
"refId": "C",
"expr": "histogram_quantile(0.99, sum by (le) (rate(messenger_http_request_duration_seconds_bucket{route!~\"/health/(live|ready)\"}[1m])))",
"legendFormat": "p99"
}
]
},
{
"id": 4,
"type": "timeseries",
"title": "Worker job outcomes per second",
"description": "Rate of actual attempts, grouped by outcome. Duplicate deliveries without a claim are not attempts.",
"datasource": {
"type": "prometheus",
"uid": "lab-prometheus"
},
"gridPos": {
"h": 7,
"w": 12,
"x": 12,
"y": 7
},
"fieldConfig": {
"defaults": {
"unit": "ops",
"min": 0
},
"overrides": []
},
"targets": [
{
"refId": "A",
"expr": "sum by (outcome) (rate(messenger_jobs_total[1m]))",
"legendFormat": "{{outcome}}"
}
]
},
{
"id": 5,
"type": "timeseries",
"title": "Worker processing duration",
"description": "p50, p95 and p99 from the duration histogram for claimed attempts.",
"datasource": {
"type": "prometheus",
"uid": "lab-prometheus"
},
"gridPos": {
"h": 7,
"w": 12,
"x": 0,
"y": 14
},
"fieldConfig": {
"defaults": {
"unit": "s",
"min": 0
},
"overrides": []
},
"targets": [
{
"refId": "A",
"expr": "histogram_quantile(0.5, sum by (le) (rate(messenger_job_duration_seconds_bucket[1m])))",
"legendFormat": "p50"
},
{
"refId": "B",
"expr": "histogram_quantile(0.95, sum by (le) (rate(messenger_job_duration_seconds_bucket[1m])))",
"legendFormat": "p95"
},
{
"refId": "C",
"expr": "histogram_quantile(0.99, sum by (le) (rate(messenger_job_duration_seconds_bucket[1m])))",
"legendFormat": "p99"
}
]
},
{
"id": 6,
"type": "timeseries",
"title": "Attachment time to ready",
"description": "From durable job creation to the commit that exposes ready. Includes time queued and any retries.",
"datasource": {
"type": "prometheus",
"uid": "lab-prometheus"
},
"gridPos": {
"h": 7,
"w": 12,
"x": 12,
"y": 14
},
"fieldConfig": {
"defaults": {
"unit": "s",
"min": 0
},
"overrides": []
},
"targets": [
{
"refId": "A",
"expr": "histogram_quantile(0.5, sum by (le) (rate(messenger_attachment_time_to_ready_seconds_bucket[1m])))",
"legendFormat": "p50"
},
{
"refId": "B",
"expr": "histogram_quantile(0.95, sum by (le) (rate(messenger_attachment_time_to_ready_seconds_bucket[1m])))",
"legendFormat": "p95"
},
{
"refId": "C",
"expr": "histogram_quantile(0.99, sum by (le) (rate(messenger_attachment_time_to_ready_seconds_bucket[1m])))",
"legendFormat": "p99"
}
]
},
{
"id": 7,
"type": "timeseries",
"title": "Queue depth",
"description": "Durable queued jobs on the current pipeline version, including pending outbox and delayed retries.",
"datasource": {
"type": "prometheus",
"uid": "lab-prometheus"
},
"gridPos": {
"h": 7,
"w": 12,
"x": 0,
"y": 21
},
"fieldConfig": {
"defaults": {
"unit": "short",
"min": 0
},
"overrides": []
},
"targets": [
{
"refId": "A",
"expr": "max by (queue) (messenger_queue_depth)",
"legendFormat": "{{queue}}"
}
]
},
{
"id": 8,
"type": "timeseries",
"title": "Oldest queued job age",
"description": "Elapsed time since the oldest job still in the durable queue was accepted.",
"datasource": {
"type": "prometheus",
"uid": "lab-prometheus"
},
"gridPos": {
"h": 7,
"w": 12,
"x": 12,
"y": 21
},
"fieldConfig": {
"defaults": {
"unit": "s",
"min": 0
},
"overrides": []
},
"targets": [
{
"refId": "A",
"expr": "max by (queue) (messenger_queue_oldest_age_seconds)",
"legendFormat": "{{queue}}"
}
]
},
{
"id": 9,
"type": "timeseries",
"title": "PostgreSQL pool used and capacity",
"description": "Checked-out connections and configured pool capacity, summed per service.",
"datasource": {
"type": "prometheus",
"uid": "lab-prometheus"
},
"gridPos": {
"h": 7,
"w": 12,
"x": 0,
"y": 28
},
"fieldConfig": {
"defaults": {
"unit": "short",
"min": 0
},
"overrides": []
},
"targets": [
{
"refId": "A",
"expr": "sum by (service_name,state) (messenger_db_pool_connections)",
"legendFormat": "{{service_name}} / {{state}}"
}
]
},
{
"id": 10,
"type": "timeseries",
"title": "PostgreSQL pool waiting",
"description": "Acquisition calls in progress. Instantaneous gauge, so very short waits can fall between scrapes.",
"datasource": {
"type": "prometheus",
"uid": "lab-prometheus"
},
"gridPos": {
"h": 7,
"w": 12,
"x": 12,
"y": 28
},
"fieldConfig": {
"defaults": {
"unit": "short",
"min": 0
},
"overrides": []
},
"targets": [
{
"refId": "A",
"expr": "sum by (service_name) (messenger_db_pool_waiting)",
"legendFormat": "{{service_name}}"
}
]
},
{
"id": 11,
"type": "timeseries",
"title": "Worker busy and capacity",
"description": "Active claimed jobs and total processing lanes.",
"datasource": {
"type": "prometheus",
"uid": "lab-prometheus"
},
"gridPos": {
"h": 7,
"w": 12,
"x": 0,
"y": 35
},
"fieldConfig": {
"defaults": {
"unit": "short",
"min": 0
},
"overrides": []
},
"targets": [
{
"refId": "A",
"expr": "sum(messenger_worker_busy)",
"legendFormat": "Busy"
},
{
"refId": "B",
"expr": "sum(messenger_worker_capacity)",
"legendFormat": "Capacity"
}
]
},
{
"id": 12,
"type": "timeseries",
"title": "Worker saturation",
"description": "Fraction of processing lanes occupied by claimed attempts.",
"datasource": {
"type": "prometheus",
"uid": "lab-prometheus"
},
"gridPos": {
"h": 7,
"w": 12,
"x": 12,
"y": 35
},
"fieldConfig": {
"defaults": {
"unit": "percentunit",
"min": 0,
"max": 1
},
"overrides": []
},
"targets": [
{
"refId": "A",
"expr": "sum(messenger_worker_busy) / (sum(messenger_worker_capacity) > 0)",
"legendFormat": "Busy / capacity"
}
]
},
{
"id": 13,
"type": "timeseries",
"title": "Process CPU",
"description": "CPU seconds per wall second. 1.0 is one full CPU core.",
"datasource": {
"type": "prometheus",
"uid": "lab-prometheus"
},
"gridPos": {
"h": 7,
"w": 12,
"x": 0,
"y": 42
},
"fieldConfig": {
"defaults": {
"unit": "none",
"min": 0
},
"overrides": []
},
"targets": [
{
"refId": "A",
"expr": "rate(messenger_process_cpu_time_seconds_total[1m])",
"legendFormat": "{{service_name}} / {{service_instance_id}}"
}
]
},
{
"id": 14,
"type": "timeseries",
"title": "Process resident memory",
"description": "RSS per application process, read from Linux statm.",
"datasource": {
"type": "prometheus",
"uid": "lab-prometheus"
},
"gridPos": {
"h": 7,
"w": 12,
"x": 12,
"y": 42
},
"fieldConfig": {
"defaults": {
"unit": "bytes",
"min": 0
},
"overrides": []
},
"targets": [
{
"refId": "A",
"expr": "messenger_process_memory_bytes",
"legendFormat": "{{service_name}} / {{service_instance_id}}"
}
]
},
{
"id": 15,
"type": "timeseries",
"title": "Firing alerts",
"description": "Firing state reported by Prometheus. Inactive rules disappear from this series.",
"datasource": {
"type": "prometheus",
"uid": "lab-prometheus"
},
"gridPos": {
"h": 7,
"w": 12,
"x": 0,
"y": 49
},
"fieldConfig": {
"defaults": {
"unit": "short",
"min": 0
},
"overrides": []
},
"targets": [
{
"refId": "A",
"expr": "ALERTS{alertname=~\"HighAPIErrorRate|GrowingQueueLag\",alertstate=\"firing\"}",
"legendFormat": "{{alertname}} {{queue}}"
}
]
},
{
"id": 16,
"type": "timeseries",
"title": "Collector exporter queue",
"description": "Batches pending delivery to telemetry storage and the configured queue limit.",
"datasource": {
"type": "prometheus",
"uid": "lab-prometheus"
},
"gridPos": {
"h": 7,
"w": 12,
"x": 12,
"y": 49
},
"fieldConfig": {
"defaults": {
"unit": "short",
"min": 0
},
"overrides": []
},
"targets": [
{
"refId": "A",
"expr": "otelcol_exporter_queue_size",
"legendFormat": "{{exporter}} queued"
},
{
"refId": "B",
"expr": "otelcol_exporter_queue_capacity",
"legendFormat": "{{exporter}} capacity"
}
]
},
{
"id": 17,
"type": "logs",
"title": "Correlated API and worker logs",
"description": "Expand a log line and follow its TraceID to Tempo.",
"datasource": {
"type": "loki",
"uid": "lab-loki"
},
"gridPos": {
"h": 9,
"w": 24,
"x": 0,
"y": 56
},
"targets": [
{
"refId": "A",
"expr": "{service_name=~\"messenger-api|messenger-worker\"}"
}
],
"options": {
"showTime": true,
"sortOrder": "Descending",
"wrapLogMessage": false
}
}
]
}
@@ -0,0 +1,9 @@
apiVersion: 1
providers:
- name: Messenger
orgId: 1
folder: Lab 6
type: file
updateIntervalSeconds: 10
options:
path: /var/lib/grafana/dashboards
@@ -0,0 +1,38 @@
apiVersion: 1
datasources:
- name: Prometheus
uid: lab-prometheus
type: prometheus
access: proxy
url: http://prometheus:9090
isDefault: true
editable: false
jsonData:
timeInterval: 5s
- name: Loki
uid: lab-loki
type: loki
access: proxy
url: http://loki:3100
editable: false
jsonData:
derivedFields:
- name: TraceID
matcherRegex: '"trace_id":"([0-9a-f]{32})"'
datasourceUid: lab-tempo
url: '$${__value.raw}'
- name: Tempo
uid: lab-tempo
type: tempo
access: proxy
url: http://tempo:3200
editable: false
jsonData:
tracesToLogsV2:
datasourceUid: lab-loki
spanStartTimeShift: -30s
spanEndTimeShift: 30s
tags:
- key: service.name
value: service_name
filterByTraceID: true
+30
View File
@@ -0,0 +1,30 @@
auth_enabled: false
server:
http_listen_port: 3100
common:
instance_addr: 127.0.0.1
path_prefix: /loki
storage:
filesystem:
chunks_directory: /loki/chunks
rules_directory: /loki/rules
replication_factor: 1
ring:
kvstore:
store: inmemory
schema_config:
configs:
- from: 2024-01-01
store: tsdb
object_store: filesystem
schema: v13
index:
prefix: index_
period: 24h
compactor:
working_directory: /loki/compactor
retention_enabled: true
delete_request_store: filesystem
limits_config:
allow_structured_metadata: true
retention_period: 24h
+13
View File
@@ -0,0 +1,13 @@
global:
scrape_interval: 5s
evaluation_interval: 5s
rule_files: [/etc/prometheus/alerts.yaml]
scrape_configs:
- job_name: messenger-otlp
static_configs:
- targets: [otel-collector:9464]
- job_name: otel-collector
static_configs:
- targets: [otel-collector:8888]
+69
View File
@@ -0,0 +1,69 @@
# Диагностика ЛР 6
Команды рассчитаны на локальный стенд с синтетическими данными. Выполняйте
сценарии по одному. После остальных проверок запустите автоматическую приёмку
с тестовым Compose override по [инструкции](../../student-tests/lab6/README.md).
## Где смотреть
- [Dashboard Messenger](http://127.0.0.1:3006/d/lab6-messenger) показывает историю
HTTP, worker, очереди, пула, CPU/RAM и доступные логи.
- [Правила Prometheus](http://127.0.0.1:9096/alerts) показывают состояние каждого
алерта. Переход `firing → inactive` означает, что условие сбросилось.
- В Grafana Explore выберите Loki. По ответу API и его `X-Request-Id` найдите
запись `{service_name="messenger-api"} | json | request_id="<id>"`.
В записи откройте поле `TraceID` или вставьте значение `trace_id` в поиск Tempo.
- В Tempo проверьте дочерние операции `postgres.query`, `postgres.acquire`
и `s3.get`/`s3.put`. Для worker используйте
`{service_name="messenger-worker"} | json | trace_id="<id>"` в Loki.
## PostgreSQL degradation
1. Создайте синтетического пользователя и получите сессию, пока БД отвечает.
2. В отдельном терминале откройте сессию
`docker compose exec -it postgres psql -U messenger -d messenger`. Выполните:
```sql
BEGIN;
LOCK TABLE public.users IN ACCESS EXCLUSIVE MODE;
```
3. Не закрывая транзакцию, 45 секунд посылайте небольшим постоянным потоком
авторизованные `GET /api/v1/users/me`. Проверьте `/health/ready` параллельно.
Если LB доступен, запросы с ошибкой и здоровая readiness указывают на
проблему внутри рабочего пути. `HighAPIErrorRate` должен стать `firing`.
4. Найдите по `X-Request-Id` структурный лог ответа и откройте трассу. Проверьте
длительность и ошибку дочернего чтения `postgres.query` с операцией
`GetUserByID`. CPU и pool gauges помогают отличить блокировку от насыщения.
5. В том же терминале PostgreSQL выполните `ROLLBACK;`. Продолжайте делать
успешные запросы до опустошения 30-секундного окна и сброса алерта.
## Stopped worker
1. Выполните `docker compose stop worker`. Оставьте RabbitMQ, relay, DB и API
включёнными.
2. Через публичный API загрузите три исправных небольших PNG в свой чат.
Ожидайте `202 Accepted`, рост `messenger_queue_depth` и возраста самой
старой задачи. Подождите, пока `GrowingQueueLag` станет `firing`.
3. В Tempo найдите producer. Пока worker остановлен, за producer для этих
задач ещё нет consumer spans. Проверьте также состояние worker в Compose.
4. Выполните `docker compose start worker`. Убедитесь, что все три вложения
стали `ready`, глубина и возраст вернулись к нулю, а алерт стал `inactive`.
## Остановка Loki
1. Запишите текущее значение `otelcol_exporter_send_failed_log_records_total`
и `otelcol_exporter_enqueue_failed_log_records_total` для
`exporter="otlphttp/loki"` в Prometheus. Отсутствующий счётчик до первой
ошибки считайте нулём.
2. Выполните `docker compose stop loki`. Проверьте readiness, проведите серию
синтетических записей и прочитайте их через API. Сохраните максимум
`otelcol_exporter_queue_size`, вместимость
`otelcol_exporter_queue_capacity` для Loki и
`otelcol_process_memory_rss` за время опыта.
3. Выполните `docker compose start loki`. Дождитесь `/ready` на порту 3106.
Отправьте новый рабочий запрос, найдите его в Loki по `X-Request-Id`.
Сравните счётчики после обработки очереди и проверьте, что Collector
сохранил тот же контейнер без рестарта. `send_failed` и `enqueue_failed`
считают записи в разных точках, первая величина относится к уже принятым
экспортёром пачкам, вторая к записям, которые экспортёр не принял.
+18
View File
@@ -0,0 +1,18 @@
server:
http_listen_port: 3200
distributor:
receivers:
otlp:
protocols:
http:
endpoint: 0.0.0.0:4318
compactor:
compaction:
block_retention: 24h
storage:
trace:
backend: local
local:
path: /var/tempo/blocks
wal:
path: /var/tempo/wal
+65
View File
@@ -0,0 +1,65 @@
#!/bin/sh
set -eu
for name in rustfs_root_access_key rustfs_root_secret_key s3_access_key_id s3_secret_access_key; do
if [ ! -s "/run/secrets/$name" ]; then
echo "missing S3 secret: $name" >&2
exit 1
fi
done
root_key=$(cat /run/secrets/rustfs_root_access_key)
root_secret=$(cat /run/secrets/rustfs_root_secret_key)
app_key=$(cat /run/secrets/s3_access_key_id)
app_secret=$(cat /run/secrets/s3_secret_access_key)
if [ "$root_key" = "$app_key" ]; then
echo 'API and RustFS root access keys must differ' >&2
exit 1
fi
# These values become path and JSON fragments below.
case "$S3_BUCKET" in
messenger-attachments) ;;
*) echo 'unsupported S3_BUCKET in bootstrap' >&2; exit 1 ;;
esac
rc=/usr/bin/rc
policy=messenger-attachments-api-v1
"$rc" alias set root http://s3:9000 "$root_key" "$root_secret" \
--region "$S3_REGION" --bucket-lookup path >/dev/null
"$rc" bucket create "root/$S3_BUCKET" --ignore-existing
# A new bucket is private by default; this also removes a stale public policy.
"$rc" bucket anonymous set private "root/$S3_BUCKET"
cat > /tmp/api-policy.json <<EOF
{
"Version": "2012-10-17",
"Statement": [
{
"Effect": "Allow",
"Action": ["s3:ListBucket", "s3:GetBucketLocation"],
"Resource": "arn:aws:s3:::$S3_BUCKET"
},
{
"Effect": "Allow",
"Action": ["s3:PutObject", "s3:GetObject", "s3:DeleteObject"],
"Resource": "arn:aws:s3:::$S3_BUCKET/*"
}
]
}
EOF
if ! "$rc" admin policy info root/ "$policy" >/dev/null 2>&1; then
"$rc" admin policy create root/ "$policy" /tmp/api-policy.json
fi
if ! "$rc" admin user info root/ "$app_key" >/dev/null 2>&1; then
"$rc" admin user add root/ "$app_key" "$app_secret" >/dev/null
fi
"$rc" admin policy attach root/ "$policy" --user "$app_key" >/dev/null
"$rc" alias set app http://s3:9000 "$app_key" "$app_secret" \
--region "$S3_REGION" --bucket-lookup path >/dev/null
"$rc" bucket list "app/$S3_BUCKET" >/dev/null
echo "S3 bucket and API identity ready: $S3_BUCKET"
+119
View File
@@ -0,0 +1,119 @@
package broker
import (
"context"
"errors"
"net"
"time"
amqp "github.com/rabbitmq/amqp091-go"
)
const (
Exchange = "images"
Queue = "image.jobs"
Route = "process"
dead = "image.invalid"
)
type boundedConn struct{ net.Conn }
func (c boundedConn) Write(p []byte) (int, error) {
if err := c.SetWriteDeadline(time.Now().Add(5 * time.Second)); err != nil {
return 0, err
}
return c.Conn.Write(p)
}
// Connect supplies a freshly declared channel after a broker connection loss.
func Connect(url string) (*amqp.Connection, *amqp.Channel, error) {
dial := amqp.DefaultDial(5 * time.Second)
conn, err := amqp.DialConfig(url, amqp.Config{
Heartbeat: 10 * time.Second, Locale: "en_US",
Dial: func(network, addr string) (net.Conn, error) {
c, err := dial(network, addr)
if err != nil {
return nil, err
}
return boundedConn{c}, nil
},
})
if err != nil {
// Connection errors may include credentials in a URL.
return nil, nil, errors.New("RabbitMQ connection unavailable")
}
ch, err := conn.Channel()
if err == nil {
err = declare(ch)
}
if err != nil {
Close(conn)
return nil, nil, err
}
return conn, ch, nil
}
func Close(conn *amqp.Connection) {
_ = conn.CloseDeadline(time.Now().Add(time.Second))
}
func declare(ch *amqp.Channel) error {
if err := ch.ExchangeDeclare(Exchange, "direct", true, false, false, false, nil); err != nil {
return err
}
if _, err := ch.QueueDeclare(dead, true, false, false, false, nil); err != nil {
return err
}
if _, err := ch.QueueDeclare(Queue, true, false, false, false, amqp.Table{
"x-queue-type": "classic", "x-dead-letter-exchange": "", "x-dead-letter-routing-key": dead,
}); err != nil {
return err
}
return ch.QueueBind(Queue, Route, Exchange, false, nil)
}
// Publisher has exactly one in-flight publish. A failed/ambiguous publish
// invalidates its channel before the next event can be attempted.
type Publisher struct {
ch *amqp.Channel
returns <-chan amqp.Return
confirms <-chan amqp.Confirmation
}
func NewPublisher(ch *amqp.Channel) (*Publisher, error) {
if err := ch.Confirm(false); err != nil {
return nil, err
}
return &Publisher{
ch: ch, returns: ch.NotifyReturn(make(chan amqp.Return, 1)),
confirms: ch.NotifyPublish(make(chan amqp.Confirmation, 1)),
}, nil
}
func (p *Publisher) Publish(ctx context.Context, id string, payload []byte) error {
seq := p.ch.GetNextPublishSeqNo()
if err := p.ch.PublishWithContext(ctx, Exchange, Route, true, false, amqp.Publishing{
ContentType: "application/json", DeliveryMode: amqp.Persistent, MessageId: id, Body: payload,
}); err != nil {
return err
}
select {
case <-p.returns:
return errors.New("outbox message was not routed")
case confirm, ok := <-p.confirms:
if !ok || confirm.DeliveryTag != seq || !confirm.Ack {
return errors.New("outbox publish was not confirmed")
}
// RabbitMQ sends basic.return before its confirm for an unrouted
// mandatory publish. The client's frame reader enqueues both in order.
// With a single in-flight message any queued return is for this ID.
select {
case <-p.returns:
return errors.New("outbox message was not routed")
default:
return nil
}
case <-ctx.Done():
return ctx.Err()
}
}
+180
View File
@@ -0,0 +1,180 @@
package config
import (
"fmt"
"os"
"strconv"
"strings"
"time"
)
type Database struct {
PostgresURL string
RedisAddr string
}
type S3 struct {
Endpoint string
Region string
Bucket string
AccessKeyID string
SecretAccessKey string
}
type Config struct {
JobMaxAttempts int
HTTP struct {
Host string
Port string
}
Database Database
Session struct {
TTL time.Duration
Store string
}
S3 S3
AppEnv string
CacheURL string
}
func Load() (*Config, error) {
config := &Config{}
sessionTTL, err := getSessionTTL()
if err != nil {
return nil, err
}
config.Session.TTL = sessionTTL
config.JobMaxAttempts, err = EnvInt("JOB_MAX_ATTEMPTS", 5, 1, 20)
if err != nil {
return nil, err
}
config.HTTP.Host = os.Getenv("APP_HOST")
if config.HTTP.Host == "" {
config.HTTP.Host = "0.0.0.0"
}
config.HTTP.Port = os.Getenv("APP_PORT")
if config.HTTP.Port == "" {
config.HTTP.Port = "8080"
}
config.AppEnv = os.Getenv("APP_ENV")
if config.AppEnv == "" {
config.AppEnv = "lab"
}
store := os.Getenv("SESSION_STORE")
config.Session.Store = strings.TrimSpace(string(store))
if config.Session.Store == "" {
return nil, fmt.Errorf("empty session store data")
}
config.Database, err = getDatabaseURL()
if err != nil {
return nil, err
}
config.S3, err = getS3Data()
if err != nil {
return nil, err
}
return config, nil
}
func getSessionTTL() (time.Duration, error) {
value, ok := os.LookupEnv("SESSION_TTL_SECONDS")
if !ok {
return time.Hour, nil
}
seconds, err := strconv.ParseInt(value, 10, 64)
if err != nil || seconds <= 0 || seconds > int64((1<<63-1)/int64(time.Second)) {
return 0, fmt.Errorf("invalid SESSION_TTL_SECONDS: %q", value)
}
return time.Duration(seconds) * time.Second, nil
}
func LoadAttachmentCleanup() (postgresURL string, objectStorage S3, err error) {
postgresURL, err = LoadPostgresURL()
if err != nil {
return "", S3{}, err
}
objectStorage, err = getS3Data()
return postgresURL, objectStorage, err
}
func LoadPostgresURL() (string, error) {
postgresData, err := os.ReadFile(os.Getenv("DATABASE_URL_FILE"))
if err != nil {
return "", fmt.Errorf("read database URL: %w", err)
}
postgresURL := strings.TrimSpace(string(postgresData))
if postgresURL == "" {
return "", fmt.Errorf("empty database URL")
}
return postgresURL, nil
}
func getDatabaseURL() (Database, error) {
postgresURL, err := LoadPostgresURL()
if err != nil {
return Database{}, err
}
db := Database{PostgresURL: postgresURL}
redisData := os.Getenv("REDIS_ADDR")
db.RedisAddr = strings.TrimSpace(string(redisData))
if db.RedisAddr == "" {
return db, fmt.Errorf("redis addres is empty")
}
return db, nil
}
func getS3Data() (S3, error) {
s3 := S3{
Endpoint: os.Getenv("S3_ENDPOINT"),
Region: os.Getenv("S3_REGION"),
Bucket: os.Getenv("S3_BUCKET"),
}
if s3.Endpoint == "" {
return s3, fmt.Errorf("empty s3 endpoint")
}
if s3.Region == "" {
return s3, fmt.Errorf("empty s3 region")
}
if s3.Bucket == "" {
return s3, fmt.Errorf("emtpy s3 bucket")
}
accessKeyIDPath := os.Getenv("S3_ACCESS_KEY_ID_FILE")
accessKeyID, err := os.ReadFile(accessKeyIDPath)
if err != nil {
return s3, fmt.Errorf("empty s3 access_key_id")
}
s3.AccessKeyID = strings.TrimSpace(string(accessKeyID))
secretAccessKeyPath := os.Getenv("S3_SECRET_ACCESS_KEY_FILE")
secretAccessKey, err := os.ReadFile(secretAccessKeyPath)
if err != nil {
return s3, fmt.Errorf("empty s3 secret_access_key_file")
}
s3.SecretAccessKey = strings.TrimSpace(string(secretAccessKey))
return s3, nil
}
+77
View File
@@ -0,0 +1,77 @@
package config
import (
"fmt"
"os"
"strconv"
"strings"
"time"
)
type Pipeline struct {
BrokerURL string
Concurrency int
Prefetch int
Timeout time.Duration
Lease time.Duration
Retention time.Duration
CleanupGrace time.Duration
CleanupPeriod time.Duration
RetryBase time.Duration
}
func EnvInt(name string, fallback, minimum, maximum int) (int, error) {
value := os.Getenv(name)
if value == "" {
return fallback, nil
}
n, err := strconv.Atoi(value)
if err != nil || n < minimum || n > maximum {
return 0, fmt.Errorf("%s must be between %d and %d", name, minimum, maximum)
}
return n, nil
}
func LoadPipeline() (Pipeline, error) {
var cfg Pipeline
path := os.Getenv("BROKER_URL_FILE")
if path != "" {
data, err := os.ReadFile(path)
if err != nil {
return cfg, fmt.Errorf("cannot read broker URL file: %w", err)
}
cfg.BrokerURL = strings.TrimSpace(string(data))
}
duration := func(dst *time.Duration, name string, fallback, minimum, maximum int) error {
n, err := EnvInt(name, fallback, minimum, maximum)
*dst = time.Duration(n) * time.Second
return err
}
var err error
if cfg.Concurrency, err = EnvInt("WORKER_CONCURRENCY", 1, 1, 4); err != nil {
return cfg, err
}
if cfg.Prefetch, err = EnvInt("WORKER_PREFETCH", 1, 1, 16); err != nil {
return cfg, err
}
for _, d := range []struct {
dst *time.Duration
name string
fallback, minimum, maximum int
}{
{&cfg.Timeout, "JOB_TIMEOUT_SECONDS", 60, 1, 600},
{&cfg.Lease, "JOB_LEASE_SECONDS", 90, 2, 900},
{&cfg.Retention, "STAGING_RETENTION_SECONDS", 86400, 1, 604800},
{&cfg.CleanupGrace, "CLEANUP_GRACE_SECONDS", 600, 1, 604800},
{&cfg.CleanupPeriod, "CLEANUP_INTERVAL_SECONDS", 5, 1, 600},
{&cfg.RetryBase, "JOB_RETRY_BASE_SECONDS", 2, 1, 15},
} {
if err := duration(d.dst, d.name, d.fallback, d.minimum, d.maximum); err != nil {
return cfg, err
}
}
if cfg.Lease < cfg.Timeout+time.Second {
return cfg, fmt.Errorf("JOB_LEASE_SECONDS must exceed JOB_TIMEOUT_SECONDS")
}
return cfg, nil
}
+84
View File
@@ -0,0 +1,84 @@
package database
import (
"context"
"errors"
"strings"
"sync/atomic"
"highload/internal/observability"
"github.com/jackc/pgx/v5"
"github.com/jackc/pgx/v5/pgconn"
"github.com/jackc/pgx/v5/pgxpool"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/trace"
)
func NewPostgres(ctx context.Context, databaseURL string) (*pgxpool.Pool, error) {
config, err := pgxpool.ParseConfig(databaseURL)
if err != nil {
return nil, err
}
tracer := &postgresTrace{}
config.ConnConfig.Tracer = tracer
pool, err := pgxpool.NewWithConfig(ctx, config)
if err != nil {
return nil, err
}
if err = pool.Ping(ctx); err != nil {
pool.Close()
return nil, err
}
observability.WatchPool(pool, &tracer.waiting)
return pool, nil
}
type postgresTrace struct{ waiting atomic.Int64 }
func (p *postgresTrace) TraceAcquireStart(ctx context.Context, _ *pgxpool.Pool, _ pgxpool.TraceAcquireStartData) context.Context {
p.waiting.Add(1)
if !trace.SpanContextFromContext(ctx).IsValid() {
return ctx
}
ctx, _ = observability.Tracer.Start(ctx, "postgres.acquire", trace.WithSpanKind(trace.SpanKindClient),
trace.WithAttributes(attribute.String("db.system.name", "postgresql")))
return ctx
}
func (p *postgresTrace) TraceAcquireEnd(ctx context.Context, _ *pgxpool.Pool, data pgxpool.TraceAcquireEndData) {
p.waiting.Add(-1)
observability.End(trace.SpanFromContext(ctx), data.Err)
}
func (*postgresTrace) TraceQueryStart(ctx context.Context, _ *pgx.Conn, data pgx.TraceQueryStartData) context.Context {
if !trace.SpanContextFromContext(ctx).IsValid() {
return ctx
}
operation := "statement"
if name, ok := strings.CutPrefix(data.SQL, "-- name: "); ok {
// sqlc embeds the static query name. SQL and arguments stay local.
operation, _, _ = strings.Cut(name, " ")
} else {
switch data.SQL {
case "begin", "commit", "rollback":
operation = data.SQL
}
}
ctx, _ = observability.Tracer.Start(ctx, "postgres.query", trace.WithSpanKind(trace.SpanKindClient),
trace.WithAttributes(attribute.String("db.system.name", "postgresql"), attribute.String("db.operation.name", operation)))
return ctx
}
func (*postgresTrace) TraceQueryEnd(ctx context.Context, _ *pgx.Conn, data pgx.TraceQueryEndData) {
span := trace.SpanFromContext(ctx)
var pgErr *pgconn.PgError
if errors.As(data.Err, &pgErr) {
span.SetAttributes(attribute.String("db.response.status_code", pgErr.Code))
}
err := data.Err
if errors.Is(err, pgx.ErrNoRows) {
err = nil
}
observability.End(span, err)
}
+31
View File
@@ -0,0 +1,31 @@
package database
import (
"context"
"time"
"github.com/redis/go-redis/v9"
)
func NewRedis(ctx context.Context, addr string) (*redis.Client, error) {
options := &redis.Options{
Addr: addr,
DialTimeout: 3 * time.Second,
ReadTimeout: 3 * time.Second,
WriteTimeout: 2 * time.Second,
PoolSize: 10,
PoolTimeout: 2 * time.Second,
ContextTimeoutEnabled: true,
}
client := redis.NewClient(options)
if err := client.Ping(ctx).Err(); err != nil {
_ = client.Close()
return nil, err
}
return client, nil
}
@@ -0,0 +1,88 @@
// Code generated by sqlc. DO NOT EDIT.
// versions:
// sqlc v1.31.1
// source: attachment_variants.sql
package sqlc
import (
"context"
"github.com/jackc/pgx/v5/pgtype"
)
const createAttachmentVariant = `-- name: CreateAttachmentVariant :exec
INSERT INTO attachment_variants (
attachment_id, pipeline_version, name, object_key, content_type, size_bytes, width, height, sha256
) VALUES (
$1, $2, $3, $4, $5, $6, $7, $8, $9
)
`
type CreateAttachmentVariantParams struct {
AttachmentID pgtype.UUID
PipelineVersion int32
Name string
ObjectKey string
ContentType string
SizeBytes int64
Width int32
Height int32
Sha256 string
}
func (q *Queries) CreateAttachmentVariant(ctx context.Context, arg CreateAttachmentVariantParams) error {
_, err := q.db.Exec(ctx, createAttachmentVariant,
arg.AttachmentID,
arg.PipelineVersion,
arg.Name,
arg.ObjectKey,
arg.ContentType,
arg.SizeBytes,
arg.Width,
arg.Height,
arg.Sha256,
)
return err
}
const listAttachmentVariants = `-- name: ListAttachmentVariants :many
SELECT attachment_id, pipeline_version, name, object_key, content_type, size_bytes, width, height, sha256 FROM attachment_variants
WHERE attachment_id = $1 AND pipeline_version = $2
ORDER BY name
`
type ListAttachmentVariantsParams struct {
AttachmentID pgtype.UUID
PipelineVersion int32
}
func (q *Queries) ListAttachmentVariants(ctx context.Context, arg ListAttachmentVariantsParams) ([]AttachmentVariant, error) {
rows, err := q.db.Query(ctx, listAttachmentVariants, arg.AttachmentID, arg.PipelineVersion)
if err != nil {
return nil, err
}
defer rows.Close()
var items []AttachmentVariant
for rows.Next() {
var i AttachmentVariant
if err := rows.Scan(
&i.AttachmentID,
&i.PipelineVersion,
&i.Name,
&i.ObjectKey,
&i.ContentType,
&i.SizeBytes,
&i.Width,
&i.Height,
&i.Sha256,
); err != nil {
return nil, err
}
items = append(items, i)
}
if err := rows.Err(); err != nil {
return nil, err
}
return items, nil
}
+306
View File
@@ -0,0 +1,306 @@
// Code generated by sqlc. DO NOT EDIT.
// versions:
// sqlc v1.31.1
// source: attachments.sql
package sqlc
import (
"context"
"github.com/jackc/pgx/v5/pgtype"
)
const attachmentExistsByObjectKey = `-- name: AttachmentExistsByObjectKey :one
SELECT EXISTS (
SELECT 1 FROM attachments
WHERE object_key = $1::text OR staging_key = $1::text
UNION ALL
SELECT 1 FROM attachment_variants WHERE object_key = $1::text
UNION ALL
SELECT 1 FROM attachment_jobs WHERE published_original_key = $1::text
)
`
func (q *Queries) AttachmentExistsByObjectKey(ctx context.Context, objectKey string) (bool, error) {
row := q.db.QueryRow(ctx, attachmentExistsByObjectKey, objectKey)
var exists bool
err := row.Scan(&exists)
return exists, err
}
const createAttachment = `-- name: CreateAttachment :one
WITH legacy AS (
INSERT INTO attachments
(chat_id, owner_id, filename, object_key, staging_key, content_type, size_bytes, sha256)
VALUES ($1, $2, $3,
$4::text, $4::text,
$5, $6, $7)
RETURNING id, chat_id, owner_id, filename, object_key, content_type, size_bytes, sha256, created_at, updated_at, status, error_code, staging_key, pipeline_version
), job AS (
INSERT INTO attachment_jobs (attachment_id, pipeline_version)
SELECT id, pipeline_version FROM legacy
RETURNING id, attachment_id, pipeline_version, attempts_started, max_attempts, next_attempt_at, lease_generation, lease_until, created_at, updated_at, published_original_key
), event AS (
INSERT INTO outbox (job_id, payload, dedup_key)
SELECT id, attachment_job_payload(id, attachment_id, pipeline_version, 1, created_at),
'initial/' || id
FROM job
RETURNING job_id
)
SELECT legacy.id, legacy.chat_id, legacy.owner_id, legacy.filename, legacy.object_key, legacy.content_type, legacy.size_bytes, legacy.sha256, legacy.created_at, legacy.updated_at, legacy.status, legacy.error_code, legacy.staging_key, legacy.pipeline_version FROM legacy CROSS JOIN event
`
type CreateAttachmentParams struct {
ChatID pgtype.UUID
OwnerID pgtype.UUID
Filename string
ObjectKey string
ContentType string
SizeBytes int64
Sha256 string
}
type CreateAttachmentRow struct {
ID pgtype.UUID
ChatID pgtype.UUID
OwnerID pgtype.UUID
Filename string
ObjectKey pgtype.Text
ContentType string
SizeBytes int64
Sha256 string
CreatedAt pgtype.Timestamptz
UpdatedAt pgtype.Timestamptz
Status string
ErrorCode pgtype.Text
StagingKey pgtype.Text
PipelineVersion int32
}
// Compatibility for the synchronous API until CreateQueuedAttachment is wired in.
func (q *Queries) CreateAttachment(ctx context.Context, arg CreateAttachmentParams) (CreateAttachmentRow, error) {
row := q.db.QueryRow(ctx, createAttachment,
arg.ChatID,
arg.OwnerID,
arg.Filename,
arg.ObjectKey,
arg.ContentType,
arg.SizeBytes,
arg.Sha256,
)
var i CreateAttachmentRow
err := row.Scan(
&i.ID,
&i.ChatID,
&i.OwnerID,
&i.Filename,
&i.ObjectKey,
&i.ContentType,
&i.SizeBytes,
&i.Sha256,
&i.CreatedAt,
&i.UpdatedAt,
&i.Status,
&i.ErrorCode,
&i.StagingKey,
&i.PipelineVersion,
)
return i, err
}
const createQueuedAttachment = `-- name: CreateQueuedAttachment :one
INSERT INTO attachments (
id, chat_id, owner_id, filename, content_type, size_bytes, sha256, staging_key, status
) VALUES (
$1, $2, $3, $4, $5, $6, $7, $8, 'queued'
) RETURNING id, chat_id, owner_id, filename, object_key, content_type, size_bytes, sha256, created_at, updated_at, status, error_code, staging_key, pipeline_version
`
type CreateQueuedAttachmentParams struct {
ID pgtype.UUID
ChatID pgtype.UUID
OwnerID pgtype.UUID
Filename string
ContentType string
SizeBytes int64
Sha256 string
StagingKey pgtype.Text
}
func (q *Queries) CreateQueuedAttachment(ctx context.Context, arg CreateQueuedAttachmentParams) (Attachment, error) {
row := q.db.QueryRow(ctx, createQueuedAttachment,
arg.ID,
arg.ChatID,
arg.OwnerID,
arg.Filename,
arg.ContentType,
arg.SizeBytes,
arg.Sha256,
arg.StagingKey,
)
var i Attachment
err := row.Scan(
&i.ID,
&i.ChatID,
&i.OwnerID,
&i.Filename,
&i.ObjectKey,
&i.ContentType,
&i.SizeBytes,
&i.Sha256,
&i.CreatedAt,
&i.UpdatedAt,
&i.Status,
&i.ErrorCode,
&i.StagingKey,
&i.PipelineVersion,
)
return i, err
}
const getAttachmentByID = `-- name: GetAttachmentByID :one
SELECT id, chat_id, owner_id, filename, object_key, content_type, size_bytes, sha256, created_at, updated_at, status, error_code, staging_key, pipeline_version FROM attachments WHERE id = $1
`
func (q *Queries) GetAttachmentByID(ctx context.Context, attachmentID pgtype.UUID) (Attachment, error) {
row := q.db.QueryRow(ctx, getAttachmentByID, attachmentID)
var i Attachment
err := row.Scan(
&i.ID,
&i.ChatID,
&i.OwnerID,
&i.Filename,
&i.ObjectKey,
&i.ContentType,
&i.SizeBytes,
&i.Sha256,
&i.CreatedAt,
&i.UpdatedAt,
&i.Status,
&i.ErrorCode,
&i.StagingKey,
&i.PipelineVersion,
)
return i, err
}
const getAttachmentForUser = `-- name: GetAttachmentForUser :one
SELECT a.id, a.chat_id, a.owner_id, a.filename, a.object_key, a.content_type, a.size_bytes, a.sha256, a.created_at, a.updated_at, a.status, a.error_code, a.staging_key, a.pipeline_version
FROM attachments AS a
JOIN chat_members AS cm ON cm.chat_id = a.chat_id
WHERE a.id = $1
AND cm.user_id = $2
`
type GetAttachmentForUserParams struct {
AttachmentID pgtype.UUID
UserID pgtype.UUID
}
func (q *Queries) GetAttachmentForUser(ctx context.Context, arg GetAttachmentForUserParams) (Attachment, error) {
row := q.db.QueryRow(ctx, getAttachmentForUser, arg.AttachmentID, arg.UserID)
var i Attachment
err := row.Scan(
&i.ID,
&i.ChatID,
&i.OwnerID,
&i.Filename,
&i.ObjectKey,
&i.ContentType,
&i.SizeBytes,
&i.Sha256,
&i.CreatedAt,
&i.UpdatedAt,
&i.Status,
&i.ErrorCode,
&i.StagingKey,
&i.PipelineVersion,
)
return i, err
}
const getAttachmentsForMessages = `-- name: GetAttachmentsForMessages :many
SELECT message_id, attachment_id, position
FROM messages_attachments
WHERE message_id = ANY($1::uuid[])
ORDER BY message_id, position
`
type GetAttachmentsForMessagesRow struct {
MessageID pgtype.UUID
AttachmentID pgtype.UUID
Position int16
}
func (q *Queries) GetAttachmentsForMessages(ctx context.Context, messageIds []pgtype.UUID) ([]GetAttachmentsForMessagesRow, error) {
rows, err := q.db.Query(ctx, getAttachmentsForMessages, messageIds)
if err != nil {
return nil, err
}
defer rows.Close()
var items []GetAttachmentsForMessagesRow
for rows.Next() {
var i GetAttachmentsForMessagesRow
if err := rows.Scan(&i.MessageID, &i.AttachmentID, &i.Position); err != nil {
return nil, err
}
items = append(items, i)
}
if err := rows.Err(); err != nil {
return nil, err
}
return items, nil
}
const listAttachmentKeys = `-- name: ListAttachmentKeys :many
SELECT id, object_key, created_at
FROM attachments
WHERE object_key > $1
ORDER BY object_key
LIMIT $2
`
type ListAttachmentKeysParams struct {
AfterKey pgtype.Text
PageSize int32
}
type ListAttachmentKeysRow struct {
ID pgtype.UUID
ObjectKey pgtype.Text
CreatedAt pgtype.Timestamptz
}
func (q *Queries) ListAttachmentKeys(ctx context.Context, arg ListAttachmentKeysParams) ([]ListAttachmentKeysRow, error) {
rows, err := q.db.Query(ctx, listAttachmentKeys, arg.AfterKey, arg.PageSize)
if err != nil {
return nil, err
}
defer rows.Close()
var items []ListAttachmentKeysRow
for rows.Next() {
var i ListAttachmentKeysRow
if err := rows.Scan(&i.ID, &i.ObjectKey, &i.CreatedAt); err != nil {
return nil, err
}
items = append(items, i)
}
if err := rows.Err(); err != nil {
return nil, err
}
return items, nil
}
const tryLockAttachmentObjectKey = `-- name: TryLockAttachmentObjectKey :one
SELECT pg_try_advisory_xact_lock(
hashtextextended($1::text, 0)
)::boolean AS locked
`
func (q *Queries) TryLockAttachmentObjectKey(ctx context.Context, objectKey string) (bool, error) {
row := q.db.QueryRow(ctx, tryLockAttachmentObjectKey, objectKey)
var locked bool
err := row.Scan(&locked)
return locked, err
}
@@ -0,0 +1,556 @@
// Code generated by sqlc. DO NOT EDIT.
// versions:
// sqlc v1.31.1
// source: attachments_jobs.sql
package sqlc
import (
"context"
"github.com/jackc/pgx/v5/pgtype"
)
const claimJob = `-- name: ClaimJob :one
WITH claimed AS (
UPDATE attachment_jobs AS j
SET lease_generation = j.lease_generation + 1,
lease_until = $1::timestamptz,
attempts_started = j.attempts_started + 1,
next_attempt_at = NULL,
updated_at = clock_timestamp()
FROM attachments AS a
WHERE j.id = $2
AND a.id = j.attachment_id AND a.pipeline_version = j.pipeline_version
AND j.attempts_started < j.max_attempts
AND $1::timestamptz > clock_timestamp()
AND (
(a.status = 'queued' AND j.lease_until IS NULL AND j.next_attempt_at <= clock_timestamp())
OR (a.status = 'processing' AND j.lease_until <= clock_timestamp())
)
RETURNING j.id, j.attachment_id, j.pipeline_version, j.attempts_started, j.max_attempts, j.next_attempt_at, j.lease_generation, j.lease_until, j.created_at, j.updated_at, j.published_original_key
), expired AS (
UPDATE attachment_attempts AS attempt
SET outcome = 'lease_expired', error_code = 'lease_expired', finished_at = claimed.updated_at
FROM claimed
WHERE attempt.job_id = claimed.id
AND attempt.lease_generation = claimed.lease_generation - 1 AND attempt.finished_at IS NULL
), recorded AS (
INSERT INTO attachment_attempts (job_id, lease_generation, attempt, started_at)
SELECT id, lease_generation, attempts_started, updated_at FROM claimed
RETURNING job_id
), processing AS (
UPDATE attachments AS a
SET status = 'processing', updated_at = claimed.updated_at
FROM claimed
WHERE a.id = claimed.attachment_id
RETURNING a.id
)
SELECT claimed.id, claimed.attachment_id, claimed.pipeline_version, claimed.attempts_started, claimed.max_attempts, claimed.next_attempt_at, claimed.lease_generation, claimed.lease_until, claimed.created_at, claimed.updated_at, claimed.published_original_key FROM claimed JOIN recorded ON recorded.job_id = claimed.id
JOIN processing ON processing.id = claimed.attachment_id
`
type ClaimJobParams struct {
NewLeaseUntil pgtype.Timestamptz
JobID pgtype.UUID
}
type ClaimJobRow struct {
ID pgtype.UUID
AttachmentID pgtype.UUID
PipelineVersion int32
AttemptsStarted int32
MaxAttempts int32
NextAttemptAt pgtype.Timestamptz
LeaseGeneration int32
LeaseUntil pgtype.Timestamptz
CreatedAt pgtype.Timestamptz
UpdatedAt pgtype.Timestamptz
PublishedOriginalKey pgtype.Text
}
func (q *Queries) ClaimJob(ctx context.Context, arg ClaimJobParams) (ClaimJobRow, error) {
row := q.db.QueryRow(ctx, claimJob, arg.NewLeaseUntil, arg.JobID)
var i ClaimJobRow
err := row.Scan(
&i.ID,
&i.AttachmentID,
&i.PipelineVersion,
&i.AttemptsStarted,
&i.MaxAttempts,
&i.NextAttemptAt,
&i.LeaseGeneration,
&i.LeaseUntil,
&i.CreatedAt,
&i.UpdatedAt,
&i.PublishedOriginalKey,
)
return i, err
}
const clearStagingReference = `-- name: ClearStagingReference :execrows
UPDATE attachments
SET object_key = CASE WHEN status = 'failed' AND object_key = staging_key THEN NULL ELSE object_key END,
staging_key = NULL
WHERE id = $1 AND staging_key = $2
`
type ClearStagingReferenceParams struct {
ID pgtype.UUID
StagingKey pgtype.Text
}
func (q *Queries) ClearStagingReference(ctx context.Context, arg ClearStagingReferenceParams) (int64, error) {
result, err := q.db.Exec(ctx, clearStagingReference, arg.ID, arg.StagingKey)
if err != nil {
return 0, err
}
return result.RowsAffected(), nil
}
const completeJob = `-- name: CompleteJob :execrows
WITH completed AS (
UPDATE attachment_jobs AS j
SET lease_until = NULL, next_attempt_at = NULL, updated_at = clock_timestamp(),
published_original_key = $1::text
FROM attachments AS a
WHERE j.id = $2
AND j.lease_generation = $3 AND j.lease_until > clock_timestamp()
AND a.id = j.attachment_id AND a.pipeline_version = j.pipeline_version AND a.status = 'processing'
AND (SELECT count(*) FROM attachment_variants AS v
WHERE v.attachment_id = a.id AND v.pipeline_version = j.pipeline_version) = 3
RETURNING j.id, j.attachment_id, j.pipeline_version, j.attempts_started, j.max_attempts, j.next_attempt_at, j.lease_generation, j.lease_until, j.created_at, j.updated_at, j.published_original_key
), recorded AS (
UPDATE attachment_attempts AS attempt
SET outcome = 'ready', finished_at = completed.updated_at
FROM completed
WHERE attempt.job_id = completed.id AND attempt.lease_generation = completed.lease_generation
)
UPDATE attachments AS a
SET status = 'ready', object_key = $1::text, updated_at = completed.updated_at
FROM completed
WHERE a.id = completed.attachment_id
`
type CompleteJobParams struct {
ObjectKey string
JobID pgtype.UUID
LeaseGeneration int32
}
func (q *Queries) CompleteJob(ctx context.Context, arg CompleteJobParams) (int64, error) {
result, err := q.db.Exec(ctx, completeJob, arg.ObjectKey, arg.JobID, arg.LeaseGeneration)
if err != nil {
return 0, err
}
return result.RowsAffected(), nil
}
const createAttachmentJob = `-- name: CreateAttachmentJob :one
INSERT INTO attachment_jobs (attachment_id, pipeline_version, next_attempt_at, max_attempts, created_at)
VALUES ($1, $2,
$3::timestamptz, $4, clock_timestamp())
RETURNING id, attachment_id, pipeline_version, attempts_started, max_attempts, next_attempt_at, lease_generation, lease_until, created_at, updated_at, published_original_key
`
type CreateAttachmentJobParams struct {
AttachmentID pgtype.UUID
PipelineVersion int32
AvailableAt pgtype.Timestamptz
MaxAttempts int32
}
func (q *Queries) CreateAttachmentJob(ctx context.Context, arg CreateAttachmentJobParams) (AttachmentJob, error) {
row := q.db.QueryRow(ctx, createAttachmentJob,
arg.AttachmentID,
arg.PipelineVersion,
arg.AvailableAt,
arg.MaxAttempts,
)
var i AttachmentJob
err := row.Scan(
&i.ID,
&i.AttachmentID,
&i.PipelineVersion,
&i.AttemptsStarted,
&i.MaxAttempts,
&i.NextAttemptAt,
&i.LeaseGeneration,
&i.LeaseUntil,
&i.CreatedAt,
&i.UpdatedAt,
&i.PublishedOriginalKey,
)
return i, err
}
const failJob = `-- name: FailJob :execrows
WITH failed AS (
UPDATE attachment_jobs AS j
SET lease_until = NULL, next_attempt_at = NULL, updated_at = clock_timestamp()
FROM attachments AS a
WHERE j.id = $2
AND j.lease_generation = $3
AND (
j.lease_until > clock_timestamp()
OR (j.lease_until <= clock_timestamp() AND j.attempts_started = j.max_attempts
AND $1::text = 'retry_exhausted')
)
AND a.id = j.attachment_id AND a.pipeline_version = j.pipeline_version AND a.status = 'processing'
RETURNING j.id, j.attachment_id, j.pipeline_version, j.attempts_started, j.max_attempts, j.next_attempt_at, j.lease_generation, j.lease_until, j.created_at, j.updated_at, j.published_original_key
), recorded AS (
UPDATE attachment_attempts AS attempt
SET outcome = 'failed', error_code = $1::text, finished_at = failed.updated_at
FROM failed
WHERE attempt.job_id = failed.id AND attempt.lease_generation = failed.lease_generation
)
UPDATE attachments AS a
SET status = 'failed', error_code = $1::text, updated_at = failed.updated_at
FROM failed WHERE a.id = failed.attachment_id
`
type FailJobParams struct {
ErrorCode string
JobID pgtype.UUID
LeaseGeneration int32
}
func (q *Queries) FailJob(ctx context.Context, arg FailJobParams) (int64, error) {
result, err := q.db.Exec(ctx, failJob, arg.ErrorCode, arg.JobID, arg.LeaseGeneration)
if err != nil {
return 0, err
}
return result.RowsAffected(), nil
}
const getAttachmentByStagingKey = `-- name: GetAttachmentByStagingKey :one
SELECT id, chat_id, owner_id, filename, object_key, content_type, size_bytes, sha256, created_at, updated_at, status, error_code, staging_key, pipeline_version FROM attachments WHERE staging_key = $1::text
`
func (q *Queries) GetAttachmentByStagingKey(ctx context.Context, stagingKey string) (Attachment, error) {
row := q.db.QueryRow(ctx, getAttachmentByStagingKey, stagingKey)
var i Attachment
err := row.Scan(
&i.ID,
&i.ChatID,
&i.OwnerID,
&i.Filename,
&i.ObjectKey,
&i.ContentType,
&i.SizeBytes,
&i.Sha256,
&i.CreatedAt,
&i.UpdatedAt,
&i.Status,
&i.ErrorCode,
&i.StagingKey,
&i.PipelineVersion,
)
return i, err
}
const getAttachmentJobState = `-- name: GetAttachmentJobState :one
SELECT
j.id, j.attachment_id, j.pipeline_version,
CASE WHEN a.pipeline_version = j.pipeline_version THEN a.status ELSE 'superseded' END::text AS status,
j.lease_generation,
j.lease_until,
coalesce(j.lease_until <= clock_timestamp(), false)::boolean AS lease_expired,
j.attempts_started,
j.max_attempts,
j.next_attempt_at
FROM attachment_jobs AS j
JOIN attachments AS a ON a.id = j.attachment_id
WHERE j.id = $1::uuid
`
type GetAttachmentJobStateRow struct {
ID pgtype.UUID
AttachmentID pgtype.UUID
PipelineVersion int32
Status string
LeaseGeneration int32
LeaseUntil pgtype.Timestamptz
LeaseExpired bool
AttemptsStarted int32
MaxAttempts int32
NextAttemptAt pgtype.Timestamptz
}
func (q *Queries) GetAttachmentJobState(ctx context.Context, jobID pgtype.UUID) (GetAttachmentJobStateRow, error) {
row := q.db.QueryRow(ctx, getAttachmentJobState, jobID)
var i GetAttachmentJobStateRow
err := row.Scan(
&i.ID,
&i.AttachmentID,
&i.PipelineVersion,
&i.Status,
&i.LeaseGeneration,
&i.LeaseUntil,
&i.LeaseExpired,
&i.AttemptsStarted,
&i.MaxAttempts,
&i.NextAttemptAt,
)
return i, err
}
const getQueueStats = `-- name: GetQueueStats :one
SELECT count(*)::bigint AS depth,
COALESCE(EXTRACT(EPOCH FROM (statement_timestamp() - min(j.created_at))), 0)::double precision AS oldest_seconds
FROM attachment_jobs AS j JOIN attachments AS a ON a.id = j.attachment_id
WHERE a.status = 'queued' AND a.pipeline_version = j.pipeline_version
`
type GetQueueStatsRow struct {
Depth int64
OldestSeconds float64
}
func (q *Queries) GetQueueStats(ctx context.Context) (GetQueueStatsRow, error) {
row := q.db.QueryRow(ctx, getQueueStats)
var i GetQueueStatsRow
err := row.Scan(&i.Depth, &i.OldestSeconds)
return i, err
}
const listFailedAttachmentJobs = `-- name: ListFailedAttachmentJobs :many
SELECT j.id, j.attachment_id, j.attempts_started, a.error_code, a.updated_at
FROM attachment_jobs j
JOIN attachments a ON a.id = j.attachment_id AND a.pipeline_version = j.pipeline_version
WHERE a.status = 'failed'
ORDER BY a.updated_at DESC, j.id
LIMIT $1
`
type ListFailedAttachmentJobsRow struct {
ID pgtype.UUID
AttachmentID pgtype.UUID
AttemptsStarted int32
ErrorCode pgtype.Text
UpdatedAt pgtype.Timestamptz
}
func (q *Queries) ListFailedAttachmentJobs(ctx context.Context, batchSize int32) ([]ListFailedAttachmentJobsRow, error) {
rows, err := q.db.Query(ctx, listFailedAttachmentJobs, batchSize)
if err != nil {
return nil, err
}
defer rows.Close()
var items []ListFailedAttachmentJobsRow
for rows.Next() {
var i ListFailedAttachmentJobsRow
if err := rows.Scan(
&i.ID,
&i.AttachmentID,
&i.AttemptsStarted,
&i.ErrorCode,
&i.UpdatedAt,
); err != nil {
return nil, err
}
items = append(items, i)
}
if err := rows.Err(); err != nil {
return nil, err
}
return items, nil
}
const lockActiveAttachmentJob = `-- name: LockActiveAttachmentJob :one
SELECT j.id, j.attachment_id, j.pipeline_version, j.attempts_started, j.max_attempts, j.next_attempt_at, j.lease_generation, j.lease_until, j.created_at, j.updated_at, j.published_original_key
FROM attachment_jobs AS j JOIN attachments AS a ON a.id = j.attachment_id
WHERE j.id = $1
AND j.lease_generation = $2
AND j.lease_until > clock_timestamp()
AND a.status = 'processing' AND a.pipeline_version = j.pipeline_version
FOR UPDATE OF j
`
type LockActiveAttachmentJobParams struct {
JobID pgtype.UUID
LeaseGeneration int32
}
// Hold this lock through variant insertion and CompleteJob. Roll back on a zero-row completion.
func (q *Queries) LockActiveAttachmentJob(ctx context.Context, arg LockActiveAttachmentJobParams) (AttachmentJob, error) {
row := q.db.QueryRow(ctx, lockActiveAttachmentJob, arg.JobID, arg.LeaseGeneration)
var i AttachmentJob
err := row.Scan(
&i.ID,
&i.AttachmentID,
&i.PipelineVersion,
&i.AttemptsStarted,
&i.MaxAttempts,
&i.NextAttemptAt,
&i.LeaseGeneration,
&i.LeaseUntil,
&i.CreatedAt,
&i.UpdatedAt,
&i.PublishedOriginalKey,
)
return i, err
}
const lockAttachmentJobForPipeline = `-- name: LockAttachmentJobForPipeline :one
SELECT j.id, j.attachment_id, j.pipeline_version, j.attempts_started, j.max_attempts, j.next_attempt_at, j.lease_generation, j.lease_until, j.created_at, j.updated_at, j.published_original_key
FROM attachment_jobs j
WHERE j.attachment_id = $1 AND j.pipeline_version = $2
FOR UPDATE
`
type LockAttachmentJobForPipelineParams struct {
AttachmentID pgtype.UUID
PipelineVersion int32
}
func (q *Queries) LockAttachmentJobForPipeline(ctx context.Context, arg LockAttachmentJobForPipelineParams) (AttachmentJob, error) {
row := q.db.QueryRow(ctx, lockAttachmentJobForPipeline, arg.AttachmentID, arg.PipelineVersion)
var i AttachmentJob
err := row.Scan(
&i.ID,
&i.AttachmentID,
&i.PipelineVersion,
&i.AttemptsStarted,
&i.MaxAttempts,
&i.NextAttemptAt,
&i.LeaseGeneration,
&i.LeaseUntil,
&i.CreatedAt,
&i.UpdatedAt,
&i.PublishedOriginalKey,
)
return i, err
}
const lockAttachmentJobForReplay = `-- name: LockAttachmentJobForReplay :one
SELECT j.id, j.attachment_id, j.pipeline_version, j.attempts_started, j.max_attempts, j.next_attempt_at, j.lease_generation, j.lease_until, j.created_at, j.updated_at, j.published_original_key
FROM attachment_jobs j WHERE j.id = $1
FOR UPDATE
`
func (q *Queries) LockAttachmentJobForReplay(ctx context.Context, id pgtype.UUID) (AttachmentJob, error) {
row := q.db.QueryRow(ctx, lockAttachmentJobForReplay, id)
var i AttachmentJob
err := row.Scan(
&i.ID,
&i.AttachmentID,
&i.PipelineVersion,
&i.AttemptsStarted,
&i.MaxAttempts,
&i.NextAttemptAt,
&i.LeaseGeneration,
&i.LeaseUntil,
&i.CreatedAt,
&i.UpdatedAt,
&i.PublishedOriginalKey,
)
return i, err
}
const replayFailed = `-- name: ReplayFailed :execrows
WITH replayed AS (
UPDATE attachment_jobs AS j
SET max_attempts = j.attempts_started + $1::int,
next_attempt_at = clock_timestamp(), updated_at = clock_timestamp()
FROM attachments AS a
WHERE j.id = $2
AND a.id = j.attachment_id AND a.pipeline_version = j.pipeline_version
AND a.status = 'failed' AND a.staging_key IS NOT NULL
AND j.lease_until IS NULL AND j.next_attempt_at IS NULL
AND NOT EXISTS (
SELECT 1 FROM attachment_replays AS r WHERE r.request_id = $3::uuid
)
RETURNING j.id, j.attachment_id, j.pipeline_version, j.attempts_started, j.max_attempts, j.next_attempt_at, j.lease_generation, j.lease_until, j.created_at, j.updated_at, j.published_original_key
), recorded AS (
INSERT INTO attachment_replays (request_id, job_id, operator_id, reason, extra_attempts)
SELECT $3::uuid, id, $4, $5, $1
FROM replayed
), event AS (
INSERT INTO outbox (job_id, payload, available_at, dedup_key)
SELECT id, attachment_job_payload(id, attachment_id, pipeline_version, attempts_started + 1, updated_at),
next_attempt_at, 'replay/' || id || '/' || $3::uuid
FROM replayed
)
UPDATE attachments AS a
SET status = 'queued', error_code = NULL, updated_at = replayed.updated_at
FROM replayed WHERE a.id = replayed.attachment_id
`
type ReplayFailedParams struct {
ExtraAttempts int32
JobID pgtype.UUID
RequestID pgtype.UUID
OperatorID string
Reason string
}
func (q *Queries) ReplayFailed(ctx context.Context, arg ReplayFailedParams) (int64, error) {
result, err := q.db.Exec(ctx, replayFailed,
arg.ExtraAttempts,
arg.JobID,
arg.RequestID,
arg.OperatorID,
arg.Reason,
)
if err != nil {
return 0, err
}
return result.RowsAffected(), nil
}
const scheduleRetry = `-- name: ScheduleRetry :execrows
WITH scheduled AS (
UPDATE attachment_jobs AS j
SET lease_until = NULL, next_attempt_at = $1::timestamptz,
updated_at = clock_timestamp()
FROM attachments AS a
WHERE j.id = $2
AND j.lease_generation = $3 AND j.lease_until > clock_timestamp()
AND a.id = j.attachment_id AND a.pipeline_version = j.pipeline_version AND a.status = 'processing'
AND j.attempts_started < j.max_attempts
AND $1::timestamptz > clock_timestamp()
RETURNING j.id, j.attachment_id, j.pipeline_version, j.attempts_started, j.max_attempts, j.next_attempt_at, j.lease_generation, j.lease_until, j.created_at, j.updated_at, j.published_original_key
), recorded AS (
UPDATE attachment_attempts AS attempt
SET outcome = 'retry', error_code = $4::text, finished_at = scheduled.updated_at
FROM scheduled
WHERE attempt.job_id = scheduled.id AND attempt.lease_generation = scheduled.lease_generation
), event AS (
INSERT INTO outbox (job_id, payload, available_at, dedup_key)
SELECT id, attachment_job_payload(id, attachment_id, pipeline_version, attempts_started + 1, updated_at)
|| jsonb_strip_nulls(jsonb_build_object(
'traceparent', NULLIF($5::text, ''),
'tracestate', NULLIF($6::text, '')
)),
next_attempt_at, 'retry/' || id || '/' || (attempts_started + 1)
FROM scheduled
)
UPDATE attachments AS a
SET status = 'queued', updated_at = scheduled.updated_at
FROM scheduled WHERE a.id = scheduled.attachment_id
`
type ScheduleRetryParams struct {
AvailableAt pgtype.Timestamptz
JobID pgtype.UUID
LeaseGeneration int32
ErrorCode string
Traceparent string
Tracestate string
}
func (q *Queries) ScheduleRetry(ctx context.Context, arg ScheduleRetryParams) (int64, error) {
result, err := q.db.Exec(ctx, scheduleRetry,
arg.AvailableAt,
arg.JobID,
arg.LeaseGeneration,
arg.ErrorCode,
arg.Traceparent,
arg.Tracestate,
)
if err != nil {
return 0, err
}
return result.RowsAffected(), nil
}
@@ -0,0 +1,27 @@
// Code generated by sqlc. DO NOT EDIT.
// versions:
// sqlc v1.31.1
// source: chat_members.sql
package sqlc
import (
"context"
"github.com/jackc/pgx/v5/pgtype"
)
const createChatMember = `-- name: CreateChatMember :exec
INSERT INTO chat_members(chat_id, user_id)
VALUES ($1, $2)
`
type CreateChatMemberParams struct {
ChatID pgtype.UUID
UserID pgtype.UUID
}
func (q *Queries) CreateChatMember(ctx context.Context, arg CreateChatMemberParams) error {
_, err := q.db.Exec(ctx, createChatMember, arg.ChatID, arg.UserID)
return err
}
+118
View File
@@ -0,0 +1,118 @@
// Code generated by sqlc. DO NOT EDIT.
// versions:
// sqlc v1.31.1
// source: chats.sql
package sqlc
import (
"context"
"github.com/jackc/pgx/v5/pgtype"
)
const createChat = `-- name: CreateChat :one
INSERT INTO chats (title)
VALUES ($1)
RETURNING id, title, created_at
`
func (q *Queries) CreateChat(ctx context.Context, title string) (Chat, error) {
row := q.db.QueryRow(ctx, createChat, title)
var i Chat
err := row.Scan(&i.ID, &i.Title, &i.CreatedAt)
return i, err
}
const getChatByID = `-- name: GetChatByID :one
SELECT c.id, c.title, c.created_at, array_agg(cm.user_id)::uuid[] AS member_ids
FROM chats c
JOIN chat_members cm ON cm.chat_id = c.id
WHERE c.id = $1
GROUP BY c.id
`
type GetChatByIDRow struct {
ID pgtype.UUID
Title string
CreatedAt pgtype.Timestamptz
MemberIds []pgtype.UUID
}
func (q *Queries) GetChatByID(ctx context.Context, id pgtype.UUID) (GetChatByIDRow, error) {
row := q.db.QueryRow(ctx, getChatByID, id)
var i GetChatByIDRow
err := row.Scan(
&i.ID,
&i.Title,
&i.CreatedAt,
&i.MemberIds,
)
return i, err
}
const getChatsByUserID = `-- name: GetChatsByUserID :many
SELECT c.id, c.title, c.created_at, array_agg(cm.user_id)::uuid[] AS member_ids
FROM chats c
JOIN chat_members cm ON cm.chat_id = c.id
WHERE EXISTS (SELECT 1
FROM chat_members membership
WHERE membership.chat_id = c.id
AND membership.user_id = $1)
AND (
$2::timestamptz IS NULL
OR (c.created_at, c.id) >
(
$2::timestamptz,
$3::uuid
)
)
GROUP BY c.id
ORDER BY c.created_at ASC, c.id ASC
LIMIT $4
`
type GetChatsByUserIDParams struct {
UserID pgtype.UUID
AfterCreatedAt pgtype.Timestamptz
AfterID pgtype.UUID
LimitCount int32
}
type GetChatsByUserIDRow struct {
ID pgtype.UUID
Title string
CreatedAt pgtype.Timestamptz
MemberIds []pgtype.UUID
}
func (q *Queries) GetChatsByUserID(ctx context.Context, arg GetChatsByUserIDParams) ([]GetChatsByUserIDRow, error) {
rows, err := q.db.Query(ctx, getChatsByUserID,
arg.UserID,
arg.AfterCreatedAt,
arg.AfterID,
arg.LimitCount,
)
if err != nil {
return nil, err
}
defer rows.Close()
var items []GetChatsByUserIDRow
for rows.Next() {
var i GetChatsByUserIDRow
if err := rows.Scan(
&i.ID,
&i.Title,
&i.CreatedAt,
&i.MemberIds,
); err != nil {
return nil, err
}
items = append(items, i)
}
if err := rows.Err(); err != nil {
return nil, err
}
return items, nil
}
+32
View File
@@ -0,0 +1,32 @@
// Code generated by sqlc. DO NOT EDIT.
// versions:
// sqlc v1.31.1
package sqlc
import (
"context"
"github.com/jackc/pgx/v5"
"github.com/jackc/pgx/v5/pgconn"
)
type DBTX interface {
Exec(context.Context, string, ...interface{}) (pgconn.CommandTag, error)
Query(context.Context, string, ...interface{}) (pgx.Rows, error)
QueryRow(context.Context, string, ...interface{}) pgx.Row
}
func New(db DBTX) *Queries {
return &Queries{db: db}
}
type Queries struct {
db DBTX
}
func (q *Queries) WithTx(tx pgx.Tx) *Queries {
return &Queries{
db: tx,
}
}
@@ -0,0 +1,99 @@
// Code generated by sqlc. DO NOT EDIT.
// versions:
// sqlc v1.31.1
// source: message_idempotence.sql
package sqlc
import (
"context"
"github.com/jackc/pgx/v5/pgtype"
)
const deleteExpiredMessageIdempotency = `-- name: DeleteExpiredMessageIdempotency :exec
DELETE FROM message_idempotency
WHERE user_id = $1 AND chat_id = $2 AND idempotency_key = $3 AND expires_at <= now()
`
type DeleteExpiredMessageIdempotencyParams struct {
UserID pgtype.UUID
ChatID pgtype.UUID
IdempotencyKey string
}
func (q *Queries) DeleteExpiredMessageIdempotency(ctx context.Context, arg DeleteExpiredMessageIdempotencyParams) error {
_, err := q.db.Exec(ctx, deleteExpiredMessageIdempotency, arg.UserID, arg.ChatID, arg.IdempotencyKey)
return err
}
const getIdempotentMessage = `-- name: GetIdempotentMessage :one
SELECT m.id, m.chat_id, m.sender_id, m.text, m.created_at
FROM message_idempotency AS i
JOIN messages AS m ON m.id = i.message_id
WHERE i.user_id = $1
AND i.chat_id = $2
AND i.idempotency_key = $3
`
type GetIdempotentMessageParams struct {
UserID pgtype.UUID
ChatID pgtype.UUID
IdempotencyKey string
}
func (q *Queries) GetIdempotentMessage(ctx context.Context, arg GetIdempotentMessageParams) (Message, error) {
row := q.db.QueryRow(ctx, getIdempotentMessage, arg.UserID, arg.ChatID, arg.IdempotencyKey)
var i Message
err := row.Scan(
&i.ID,
&i.ChatID,
&i.SenderID,
&i.Text,
&i.CreatedAt,
)
return i, err
}
const insertMessageIdempotency = `-- name: InsertMessageIdempotency :execrows
INSERT INTO message_idempotency(user_id, chat_id, idempotency_key, message_id)
VALUES ($1, $2, $3, $4)
ON CONFLICT (user_id, chat_id, idempotency_key) DO NOTHING
`
type InsertMessageIdempotencyParams struct {
UserID pgtype.UUID
ChatID pgtype.UUID
IdempotencyKey string
MessageID pgtype.UUID
}
func (q *Queries) InsertMessageIdempotency(ctx context.Context, arg InsertMessageIdempotencyParams) (int64, error) {
result, err := q.db.Exec(ctx, insertMessageIdempotency,
arg.UserID,
arg.ChatID,
arg.IdempotencyKey,
arg.MessageID,
)
if err != nil {
return 0, err
}
return result.RowsAffected(), nil
}
const lockMessageRequest = `-- name: LockMessageRequest :exec
SELECT pg_advisory_xact_lock(hashtextextended(
'message/' || ($1::uuid)::text || '/' ||
($2::uuid)::text || '/' || $3::text, 0))
`
type LockMessageRequestParams struct {
UserID pgtype.UUID
ChatID pgtype.UUID
IdempotencyKey string
}
func (q *Queries) LockMessageRequest(ctx context.Context, arg LockMessageRequestParams) error {
_, err := q.db.Exec(ctx, lockMessageRequest, arg.UserID, arg.ChatID, arg.IdempotencyKey)
return err
}
+91
View File
@@ -0,0 +1,91 @@
// Code generated by sqlc. DO NOT EDIT.
// versions:
// sqlc v1.31.1
// source: messages.sql
package sqlc
import (
"context"
"github.com/jackc/pgx/v5/pgtype"
)
const createMessage = `-- name: CreateMessage :one
INSERT INTO messages (chat_id, sender_id, text)
VALUES ($1, $2, $3)
RETURNING id, chat_id, sender_id, text, created_at
`
type CreateMessageParams struct {
ChatID pgtype.UUID
SenderID pgtype.UUID
Text string
}
func (q *Queries) CreateMessage(ctx context.Context, arg CreateMessageParams) (Message, error) {
row := q.db.QueryRow(ctx, createMessage, arg.ChatID, arg.SenderID, arg.Text)
var i Message
err := row.Scan(
&i.ID,
&i.ChatID,
&i.SenderID,
&i.Text,
&i.CreatedAt,
)
return i, err
}
const getMessagesByChatID = `-- name: GetMessagesByChatID :many
SELECT id, chat_id, sender_id, text, created_at
FROM messages
WHERE chat_id = $1
AND (
$2::timestamptz IS NULL
OR (created_at, id) >
($2::timestamptz,
$3::uuid
)
)
ORDER BY created_at ASC, id ASC
LIMIT $4
`
type GetMessagesByChatIDParams struct {
ChatID pgtype.UUID
AfterCreatedAt pgtype.Timestamptz
AfterID pgtype.UUID
LimitCount int32
}
func (q *Queries) GetMessagesByChatID(ctx context.Context, arg GetMessagesByChatIDParams) ([]Message, error) {
rows, err := q.db.Query(ctx, getMessagesByChatID,
arg.ChatID,
arg.AfterCreatedAt,
arg.AfterID,
arg.LimitCount,
)
if err != nil {
return nil, err
}
defer rows.Close()
var items []Message
for rows.Next() {
var i Message
if err := rows.Scan(
&i.ID,
&i.ChatID,
&i.SenderID,
&i.Text,
&i.CreatedAt,
); err != nil {
return nil, err
}
items = append(items, i)
}
if err := rows.Err(); err != nil {
return nil, err
}
return items, nil
}
@@ -0,0 +1,107 @@
// Code generated by sqlc. DO NOT EDIT.
// versions:
// sqlc v1.31.1
// source: messages_attachments.sql
package sqlc
import (
"context"
"github.com/jackc/pgx/v5/pgtype"
)
const createMessagesAttachments = `-- name: CreateMessagesAttachments :execrows
INSERT INTO messages_attachments (message_id, attachment_id, chat_id, position)
SELECT $1, $2, $3, $4
FROM attachments
WHERE id = $2 AND chat_id = $3
`
type CreateMessagesAttachmentsParams struct {
MessageID pgtype.UUID
AttachmentID pgtype.UUID
ChatID pgtype.UUID
Position int16
}
func (q *Queries) CreateMessagesAttachments(ctx context.Context, arg CreateMessagesAttachmentsParams) (int64, error) {
result, err := q.db.Exec(ctx, createMessagesAttachments,
arg.MessageID,
arg.AttachmentID,
arg.ChatID,
arg.Position,
)
if err != nil {
return 0, err
}
return result.RowsAffected(), nil
}
const getMessagesAttachments = `-- name: GetMessagesAttachments :many
SELECT attachment_id, position
FROM messages_attachments
WHERE message_id = $1
ORDER BY position ASC
`
type GetMessagesAttachmentsRow struct {
AttachmentID pgtype.UUID
Position int16
}
func (q *Queries) GetMessagesAttachments(ctx context.Context, messageID pgtype.UUID) ([]GetMessagesAttachmentsRow, error) {
rows, err := q.db.Query(ctx, getMessagesAttachments, messageID)
if err != nil {
return nil, err
}
defer rows.Close()
var items []GetMessagesAttachmentsRow
for rows.Next() {
var i GetMessagesAttachmentsRow
if err := rows.Scan(&i.AttachmentID, &i.Position); err != nil {
return nil, err
}
items = append(items, i)
}
if err := rows.Err(); err != nil {
return nil, err
}
return items, nil
}
const lockAttachmentsForMessage = `-- name: LockAttachmentsForMessage :many
SELECT id, status FROM attachments
WHERE chat_id = $1 AND id = ANY($2::uuid[])
ORDER BY id FOR SHARE
`
type LockAttachmentsForMessageParams struct {
ChatID pgtype.UUID
AttachmentIds []pgtype.UUID
}
type LockAttachmentsForMessageRow struct {
ID pgtype.UUID
Status string
}
func (q *Queries) LockAttachmentsForMessage(ctx context.Context, arg LockAttachmentsForMessageParams) ([]LockAttachmentsForMessageRow, error) {
rows, err := q.db.Query(ctx, lockAttachmentsForMessage, arg.ChatID, arg.AttachmentIds)
if err != nil {
return nil, err
}
defer rows.Close()
var items []LockAttachmentsForMessageRow
for rows.Next() {
var i LockAttachmentsForMessageRow
if err := rows.Scan(&i.ID, &i.Status); err != nil {
return nil, err
}
items = append(items, i)
}
if err := rows.Err(); err != nil {
return nil, err
}
return items, nil
}
+124
View File
@@ -0,0 +1,124 @@
// Code generated by sqlc. DO NOT EDIT.
// versions:
// sqlc v1.31.1
package sqlc
import (
"github.com/jackc/pgx/v5/pgtype"
)
type Attachment struct {
ID pgtype.UUID
ChatID pgtype.UUID
OwnerID pgtype.UUID
Filename string
ObjectKey pgtype.Text
ContentType string
SizeBytes int64
Sha256 string
CreatedAt pgtype.Timestamptz
UpdatedAt pgtype.Timestamptz
Status string
ErrorCode pgtype.Text
StagingKey pgtype.Text
PipelineVersion int32
}
type AttachmentAttempt struct {
JobID pgtype.UUID
LeaseGeneration int32
Attempt int32
StartedAt pgtype.Timestamptz
FinishedAt pgtype.Timestamptz
Outcome pgtype.Text
ErrorCode pgtype.Text
}
type AttachmentJob struct {
ID pgtype.UUID
AttachmentID pgtype.UUID
PipelineVersion int32
AttemptsStarted int32
MaxAttempts int32
NextAttemptAt pgtype.Timestamptz
LeaseGeneration int32
LeaseUntil pgtype.Timestamptz
CreatedAt pgtype.Timestamptz
UpdatedAt pgtype.Timestamptz
PublishedOriginalKey pgtype.Text
}
type AttachmentReplay struct {
RequestID pgtype.UUID
JobID pgtype.UUID
OperatorID string
Reason string
ExtraAttempts int32
CreatedAt pgtype.Timestamptz
}
type AttachmentVariant struct {
AttachmentID pgtype.UUID
PipelineVersion int32
Name string
ObjectKey string
ContentType string
SizeBytes int64
Width int32
Height int32
Sha256 string
}
type Chat struct {
ID pgtype.UUID
Title string
CreatedAt pgtype.Timestamptz
}
type ChatMember struct {
ChatID pgtype.UUID
UserID pgtype.UUID
}
type Message struct {
ID pgtype.UUID
ChatID pgtype.UUID
SenderID pgtype.UUID
Text string
CreatedAt pgtype.Timestamptz
}
type MessageIdempotency struct {
UserID pgtype.UUID
ChatID pgtype.UUID
IdempotencyKey string
MessageID pgtype.UUID
CreatedAt pgtype.Timestamptz
ExpiresAt pgtype.Timestamptz
}
type MessagesAttachment struct {
MessageID pgtype.UUID
AttachmentID pgtype.UUID
ChatID pgtype.UUID
Position int16
}
type Outbox struct {
ID pgtype.UUID
JobID pgtype.UUID
Payload []byte
AvailableAt pgtype.Timestamptz
SentAt pgtype.Timestamptz
CreatedAt pgtype.Timestamptz
DedupKey string
}
type User struct {
ID pgtype.UUID
Username string
DisplayName string
PasswordHash string
CreatedAt pgtype.Timestamptz
}
+128
View File
@@ -0,0 +1,128 @@
// Code generated by sqlc. DO NOT EDIT.
// versions:
// sqlc v1.31.1
// source: outbox.sql
package sqlc
import (
"context"
"github.com/jackc/pgx/v5/pgtype"
)
const createInitialOutboxEvent = `-- name: CreateInitialOutboxEvent :exec
INSERT INTO outbox (job_id, payload, available_at, dedup_key)
SELECT id, attachment_job_payload(id, attachment_id, pipeline_version, 1, created_at)
|| jsonb_strip_nulls(jsonb_build_object(
'traceparent', NULLIF($1::text, ''),
'tracestate', NULLIF($2::text, '')
)),
next_attempt_at, 'initial/' || id
FROM attachment_jobs WHERE id = $3::uuid
`
type CreateInitialOutboxEventParams struct {
Traceparent string
Tracestate string
JobID pgtype.UUID
}
func (q *Queries) CreateInitialOutboxEvent(ctx context.Context, arg CreateInitialOutboxEventParams) error {
_, err := q.db.Exec(ctx, createInitialOutboxEvent, arg.Traceparent, arg.Tracestate, arg.JobID)
return err
}
const enqueueExpiredLeases = `-- name: EnqueueExpiredLeases :many
WITH expired AS (
SELECT j.id, j.attachment_id, j.pipeline_version, j.attempts_started, j.max_attempts, j.next_attempt_at, j.lease_generation, j.lease_until, j.created_at, j.updated_at, j.published_original_key, 'recovery/' || j.id || '/' || j.lease_generation AS recovery_key
FROM attachment_jobs AS j
JOIN attachments AS a ON a.id = j.attachment_id AND a.pipeline_version = j.pipeline_version
WHERE a.status = 'processing' AND j.lease_until <= clock_timestamp()
AND NOT EXISTS (
SELECT 1 FROM outbox AS o WHERE o.dedup_key = 'recovery/' || j.id || '/' || j.lease_generation
)
ORDER BY j.lease_until, j.id
LIMIT $1
FOR UPDATE OF j SKIP LOCKED
)
INSERT INTO outbox (job_id, payload, dedup_key)
SELECT e.id, attachment_job_payload(e.id, e.attachment_id, e.pipeline_version,
LEAST(e.attempts_started + 1, e.max_attempts), clock_timestamp())
|| COALESCE((
SELECT jsonb_strip_nulls(jsonb_build_object(
'traceparent', o.payload->>'traceparent', 'tracestate', o.payload->>'tracestate'
))
FROM outbox AS o WHERE o.job_id = e.id
ORDER BY o.created_at DESC, o.id DESC LIMIT 1
), '{}'::jsonb),
e.recovery_key
FROM expired AS e
ON CONFLICT (dedup_key) DO NOTHING
RETURNING id, job_id, payload, available_at, sent_at, created_at, dedup_key
`
func (q *Queries) EnqueueExpiredLeases(ctx context.Context, batchSize int32) ([]Outbox, error) {
rows, err := q.db.Query(ctx, enqueueExpiredLeases, batchSize)
if err != nil {
return nil, err
}
defer rows.Close()
var items []Outbox
for rows.Next() {
var i Outbox
if err := rows.Scan(
&i.ID,
&i.JobID,
&i.Payload,
&i.AvailableAt,
&i.SentAt,
&i.CreatedAt,
&i.DedupKey,
); err != nil {
return nil, err
}
items = append(items, i)
}
if err := rows.Err(); err != nil {
return nil, err
}
return items, nil
}
const markOutboxEventSent = `-- name: MarkOutboxEventSent :execrows
UPDATE outbox SET sent_at = clock_timestamp()
WHERE id = $1 AND sent_at IS NULL
`
func (q *Queries) MarkOutboxEventSent(ctx context.Context, id pgtype.UUID) (int64, error) {
result, err := q.db.Exec(ctx, markOutboxEventSent, id)
if err != nil {
return 0, err
}
return result.RowsAffected(), nil
}
const nextPendingOutboxEvent = `-- name: NextPendingOutboxEvent :one
SELECT id, job_id, payload, available_at, sent_at, created_at, dedup_key
FROM outbox
WHERE sent_at IS NULL AND available_at <= statement_timestamp()
ORDER BY available_at, id
LIMIT 1
FOR UPDATE SKIP LOCKED
`
func (q *Queries) NextPendingOutboxEvent(ctx context.Context) (Outbox, error) {
row := q.db.QueryRow(ctx, nextPendingOutboxEvent)
var i Outbox
err := row.Scan(
&i.ID,
&i.JobID,
&i.Payload,
&i.AvailableAt,
&i.SentAt,
&i.CreatedAt,
&i.DedupKey,
)
return i, err
}
+77
View File
@@ -0,0 +1,77 @@
// Code generated by sqlc. DO NOT EDIT.
// versions:
// sqlc v1.31.1
// source: users.sql
package sqlc
import (
"context"
"github.com/jackc/pgx/v5/pgtype"
)
const createUser = `-- name: CreateUser :one
INSERT INTO users (username, display_name, password_hash)
VALUES ($1, $2, $3)
RETURNING id, username, display_name, password_hash, created_at
`
type CreateUserParams struct {
Username string
DisplayName string
PasswordHash string
}
func (q *Queries) CreateUser(ctx context.Context, arg CreateUserParams) (User, error) {
row := q.db.QueryRow(ctx, createUser, arg.Username, arg.DisplayName, arg.PasswordHash)
var i User
err := row.Scan(
&i.ID,
&i.Username,
&i.DisplayName,
&i.PasswordHash,
&i.CreatedAt,
)
return i, err
}
const getUserByID = `-- name: GetUserByID :one
SELECT id, username, display_name, password_hash, created_at
FROM users
WHERE id = $1
`
func (q *Queries) GetUserByID(ctx context.Context, id pgtype.UUID) (User, error) {
row := q.db.QueryRow(ctx, getUserByID, id)
var i User
err := row.Scan(
&i.ID,
&i.Username,
&i.DisplayName,
&i.PasswordHash,
&i.CreatedAt,
)
return i, err
}
const getUserByUsername = `-- name: GetUserByUsername :one
SELECT id, username, display_name, password_hash, created_at
FROM users
WHERE username = $1
`
func (q *Queries) GetUserByUsername(ctx context.Context, username string) (User, error) {
row := q.db.QueryRow(ctx, getUserByUsername, username)
var i User
err := row.Scan(
&i.ID,
&i.Username,
&i.DisplayName,
&i.PasswordHash,
&i.CreatedAt,
)
return i, err
}
@@ -0,0 +1,111 @@
// Code generated by sqlc. DO NOT EDIT.
// versions:
// sqlc v1.31.1
// source: versioned_pipeline.sql
package sqlc
import (
"context"
"github.com/jackc/pgx/v5/pgtype"
)
const getPipelineJobID = `-- name: GetPipelineJobID :one
SELECT id FROM attachment_jobs
WHERE attachment_id = $1::uuid AND pipeline_version = $2::int
`
type GetPipelineJobIDParams struct {
AttachmentID pgtype.UUID
PipelineVersion int32
}
func (q *Queries) GetPipelineJobID(ctx context.Context, arg GetPipelineJobIDParams) (pgtype.UUID, error) {
row := q.db.QueryRow(ctx, getPipelineJobID, arg.AttachmentID, arg.PipelineVersion)
var id pgtype.UUID
err := row.Scan(&id)
return id, err
}
const getPublishedOriginalKey = `-- name: GetPublishedOriginalKey :one
SELECT published_original_key FROM attachment_jobs
WHERE attachment_id = $1::uuid AND pipeline_version = $2::int
AND published_original_key IS NOT NULL
`
type GetPublishedOriginalKeyParams struct {
AttachmentID pgtype.UUID
PipelineVersion int32
}
func (q *Queries) GetPublishedOriginalKey(ctx context.Context, arg GetPublishedOriginalKeyParams) (pgtype.Text, error) {
row := q.db.QueryRow(ctx, getPublishedOriginalKey, arg.AttachmentID, arg.PipelineVersion)
var published_original_key pgtype.Text
err := row.Scan(&published_original_key)
return published_original_key, err
}
const isPublishedOriginalKey = `-- name: IsPublishedOriginalKey :one
SELECT EXISTS (SELECT 1 FROM attachment_jobs WHERE published_original_key = $1::text)
`
func (q *Queries) IsPublishedOriginalKey(ctx context.Context, objectKey string) (bool, error) {
row := q.db.QueryRow(ctx, isPublishedOriginalKey, objectKey)
var exists bool
err := row.Scan(&exists)
return exists, err
}
const queueAttachmentUpgrade = `-- name: QueueAttachmentUpgrade :one
UPDATE attachments
SET pipeline_version = 2, status = 'queued', staging_key = $1::text,
object_key = NULL, error_code = NULL, updated_at = clock_timestamp()
WHERE id = $2::uuid AND pipeline_version = 1
RETURNING id, chat_id, owner_id, filename, object_key, content_type, size_bytes, sha256, created_at, updated_at, status, error_code, staging_key, pipeline_version
`
type QueueAttachmentUpgradeParams struct {
SourceKey string
ID pgtype.UUID
}
func (q *Queries) QueueAttachmentUpgrade(ctx context.Context, arg QueueAttachmentUpgradeParams) (Attachment, error) {
row := q.db.QueryRow(ctx, queueAttachmentUpgrade, arg.SourceKey, arg.ID)
var i Attachment
err := row.Scan(
&i.ID,
&i.ChatID,
&i.OwnerID,
&i.Filename,
&i.ObjectKey,
&i.ContentType,
&i.SizeBytes,
&i.Sha256,
&i.CreatedAt,
&i.UpdatedAt,
&i.Status,
&i.ErrorCode,
&i.StagingKey,
&i.PipelineVersion,
)
return i, err
}
const supersedeAttachmentJob = `-- name: SupersedeAttachmentJob :exec
WITH superseded AS (
UPDATE attachment_jobs
SET lease_until = NULL, next_attempt_at = NULL, updated_at = clock_timestamp()
WHERE id = $1::uuid
RETURNING id, updated_at
)
UPDATE attachment_attempts AS attempt
SET outcome = 'superseded', error_code = 'superseded', finished_at = superseded.updated_at
FROM superseded
WHERE attempt.job_id = superseded.id AND attempt.finished_at IS NULL
`
func (q *Queries) SupersedeAttachmentJob(ctx context.Context, jobID pgtype.UUID) error {
_, err := q.db.Exec(ctx, supersedeAttachmentJob, jobID)
return err
}
+43
View File
@@ -0,0 +1,43 @@
// Package fault provides one-shot pause points for an isolated lab Compose project.
package fault
import (
"context"
"errors"
"os"
"path/filepath"
"time"
)
func Pause(ctx context.Context, point, id string) error {
dir := os.Getenv("LAB5_FAULT_DIR")
if dir == "" {
return nil
}
base := filepath.Join(dir, point)
if err := os.Rename(base+".arm", base+".used"); err != nil {
if errors.Is(err, os.ErrNotExist) {
return nil
}
return err
}
if err := os.WriteFile(base+".hit", []byte(id+"\n"), 0644); err != nil {
return err
}
wait, cancel := context.WithTimeout(ctx, 90*time.Second)
defer cancel()
poll := time.NewTicker(50 * time.Millisecond)
defer poll.Stop()
for {
if _, err := os.Stat(base + ".release"); err == nil {
return nil
} else if !errors.Is(err, os.ErrNotExist) {
return err
}
select {
case <-wait.Done():
return wait.Err()
case <-poll.C:
}
}
}
+164
View File
@@ -0,0 +1,164 @@
package imageproc
import (
"context"
"crypto/sha256"
"encoding/hex"
"errors"
"fmt"
"image"
_ "image/jpeg"
_ "image/png"
"io"
"os"
"os/exec"
"path/filepath"
"strconv"
"strings"
"highload/internal/model"
)
const (
MaxBytes = 10 * 1024 * 1024
MaxPixels = 40_000_000
)
var (
ErrInvalidImage = errors.New("invalid image")
ErrResourceLimit = errors.New("image processing resource limit")
)
type File struct {
Path string
Variant model.AttachmentVariant
}
type variantSpec struct {
name, extension, mime string
width, height int
args []string
}
func Generate(ctx context.Context, dir, input, mime string, version int32) ([]File, error) {
var jpegQuality, webpQuality string
switch version {
case 1:
jpegQuality, webpQuality = "85", "80"
case 2:
jpegQuality, webpQuality = "90", "72"
default:
return nil, fmt.Errorf("unsupported image pipeline version %d", version)
}
f, err := os.Open(input)
if err != nil {
return nil, err
}
cfg, format, decodeErr := image.DecodeConfig(io.LimitReader(f, MaxBytes))
_ = f.Close()
if decodeErr != nil || (format != "png" && format != "jpeg") || mime != "image/"+format ||
cfg.Width <= 0 || cfg.Height <= 0 || cfg.Width > MaxPixels/cfg.Height {
return nil, ErrInvalidImage
}
// Paths are generated on the server; input format is independently verified.
source := format + ":" + input
orient, err := magick(ctx, "identify", "-ping", "-format", "%[orientation]", source)
if err != nil {
return nil, err
}
w, h := cfg.Width, cfg.Height
switch strings.TrimSpace(orient) {
case "LeftTop", "RightTop", "RightBottom", "LeftBottom":
w, h = h, w
}
tw, th := fit(w, h, 256)
ow, oh := fit(w, h, 1280)
side := min(w, h)
specs := []variantSpec{
{name: "thumbnail", extension: "jpg", mime: "image/jpeg", width: tw, height: th,
args: []string{"-resize", geometry(tw, th), "-quality", jpegQuality}},
{name: "crop", extension: "jpg", mime: "image/jpeg", width: 128, height: 128,
args: []string{"-gravity", "center", "-crop", fmt.Sprintf("%dx%d+0+0", side, side), "+repage", "-resize", "128x128!", "-quality", jpegQuality}},
{name: "optimized", extension: "webp", mime: "image/webp", width: ow, height: oh,
args: []string{"-resize", geometry(ow, oh), "-quality", webpQuality, "-define", "webp:method=4"}},
}
files := make([]File, 0, len(specs))
for _, spec := range specs {
out := filepath.Join(dir, spec.name+"."+spec.extension)
args := []string{"-regard-warnings", source, "-auto-orient"}
if spec.mime == "image/jpeg" {
args = append(args, "-background", "white", "-alpha", "remove", "-alpha", "off")
}
args = append(args, spec.args...)
args = append(args, "-strip", out)
if _, err := magick(ctx, args...); err != nil {
return nil, err
}
meta, err := inspectFile(ctx, out, spec)
if err != nil {
return nil, err
}
files = append(files, File{Path: out, Variant: meta})
}
return files, nil
}
func fit(w, h, bound int) (int, int) {
if max(w, h) <= bound {
return w, h
}
if w >= h {
return bound, max(1, h*bound/w)
}
return max(1, w*bound/h), bound
}
func geometry(w, h int) string {
return fmt.Sprintf("%dx%d!", w, h)
}
func inspectFile(ctx context.Context, path string, spec variantSpec) (model.AttachmentVariant, error) {
var meta model.AttachmentVariant
desc, err := magick(ctx, "identify", "-ping", "-format", "%m %w %h", path)
if err != nil {
return meta, err
}
format := strings.ToUpper(strings.TrimPrefix(spec.mime, "image/"))
if strings.TrimSpace(desc) != format+" "+strconv.Itoa(spec.width)+" "+strconv.Itoa(spec.height) {
return meta, fmt.Errorf("generated variant has unexpected format or size")
}
f, err := os.Open(path)
if err != nil {
return meta, err
}
defer f.Close()
sum := sha256.New()
size, err := io.Copy(sum, f)
if err != nil {
return meta, err
}
return model.AttachmentVariant{Name: spec.name, ContentType: spec.mime, Width: spec.width, Height: spec.height,
SizeBytes: size, SHA256: hex.EncodeToString(sum.Sum(nil))}, nil
}
func magick(ctx context.Context, args ...string) (string, error) {
cmd := exec.CommandContext(ctx, "magick", args...)
out, err := cmd.CombinedOutput()
if ctx.Err() != nil {
return "", ctx.Err()
}
if err == nil {
return string(out), nil
}
message := strings.ToLower(string(out))
if strings.Contains(message, "cache resources exhausted") || strings.Contains(message, "time limit") ||
strings.Contains(message, "memory allocation") || strings.Contains(message, "list length exceeds") {
return "", ErrResourceLimit
}
if strings.Contains(message, "corrupt") || strings.Contains(message, "unexpected end") ||
strings.Contains(message, "insufficient image data") || strings.Contains(message, "decode") {
return "", ErrInvalidImage
}
return "", fmt.Errorf("ImageMagick command failed: %w", err)
}
+31
View File
@@ -0,0 +1,31 @@
package model
import "time"
type Attachment struct {
ID string
ChatID string
OwnerID string
Filename string
ObjectKey string
StagingKey string
ErrorCode *string
Variants []AttachmentVariant
ContentType string
SizeBytes int64
SHA256 string
Status string
PipelineVersion int32
CreatedAt time.Time
UpdatedAt time.Time
}
type AttachmentVariant struct {
Name string
ObjectKey string
ContentType string
SizeBytes int64
Width int
Height int
SHA256 string
}
+27
View File
@@ -0,0 +1,27 @@
package model
import "time"
type AttachmentJob struct {
ID string
AttachmentID string
PipelineVersion int32
AttemptsStarted int32
MaxAttempts int32
LeaseGeneration int32
LeaseUntil time.Time
AcceptedAt time.Time
}
type AttachmentJobState struct {
AttachmentID string
PipelineVersion int32
ID string
Status string
LeaseGeneration int32
AttemptsStarted int32
MaxAttempts int32
LeaseUntil *time.Time
LeaseExpired bool
NextAttemptAt *time.Time
}
+10
View File
@@ -0,0 +1,10 @@
package model
import "time"
type Chat struct {
ID string
Title string
MemberIDs []string
CreatedAt time.Time
}
+12
View File
@@ -0,0 +1,12 @@
package model
import "time"
type Message struct {
ID string
ChatID string
SenderID string
Text string
AttachmentIDs []string
CreatedAt time.Time
}
+11
View File
@@ -0,0 +1,11 @@
package model
import "time"
type Session struct {
TokenHash []byte
UserID string
CreatedAt time.Time
ExpiresAt time.Time
RevokedAt *time.Time
}
+11
View File
@@ -0,0 +1,11 @@
package model
import "time"
type User struct {
ID string
Username string
DisplayName string
PasswordHash string
CreatedAt time.Time
}
+65
View File
@@ -0,0 +1,65 @@
package observability
import (
"context"
"sync/atomic"
"time"
"github.com/jackc/pgx/v5/pgxpool"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/metric"
)
func WatchPool(pool *pgxpool.Pool, waiting *atomic.Int64) {
connections := instrument(meter.Int64ObservableGauge("messenger_db_pool_connections"))
acquiring := instrument(meter.Int64ObservableGauge("messenger_db_pool_waiting"))
waitTime := instrument(meter.Float64ObservableCounter("messenger_db_pool_wait", metric.WithUnit("s")))
_, _ = meter.RegisterCallback(func(_ context.Context, o metric.Observer) error {
stats := pool.Stat()
o.ObserveInt64(connections, int64(stats.AcquiredConns()), metric.WithAttributes(attribute.String("state", "used")))
o.ObserveInt64(connections, int64(stats.MaxConns()), metric.WithAttributes(attribute.String("state", "capacity")))
o.ObserveInt64(acquiring, waiting.Load())
o.ObserveFloat64(waitTime, stats.EmptyAcquireWaitTime().Seconds())
return nil
}, connections, acquiring, waitTime)
}
type QueueSample struct {
Depth int64
OldestSeconds float64
At time.Time
}
func WatchQueue(queue string) func(QueueSample) {
depth := instrument(meter.Int64ObservableGauge("messenger_queue_depth"))
age := instrument(meter.Float64ObservableGauge("messenger_queue_oldest_age", metric.WithUnit("s")))
freshness := instrument(meter.Float64ObservableGauge("messenger_queue_sample_age", metric.WithUnit("s")))
var current atomic.Pointer[QueueSample]
attrs := metric.WithAttributes(attribute.String("queue", queue))
_, _ = meter.RegisterCallback(func(_ context.Context, o metric.Observer) error {
if sample := current.Load(); sample != nil {
o.ObserveInt64(depth, sample.Depth, attrs)
o.ObserveFloat64(age, sample.OldestSeconds, attrs)
o.ObserveFloat64(freshness, time.Since(sample.At).Seconds(), attrs)
}
return nil
}, depth, age, freshness)
return func(sample QueueSample) { current.Store(&sample) }
}
var workerBusy atomic.Int64
func WatchWorker(lanes int) {
busy := instrument(meter.Int64ObservableGauge("messenger_worker_busy"))
capacity := instrument(meter.Int64ObservableGauge("messenger_worker_capacity"))
_, _ = meter.RegisterCallback(func(_ context.Context, o metric.Observer) error {
o.ObserveInt64(busy, workerBusy.Load())
o.ObserveInt64(capacity, int64(lanes))
return nil
}, busy, capacity)
}
func WorkerAttempt() func() {
workerBusy.Add(1)
return func() { workerBusy.Add(-1) }
}
+19
View File
@@ -0,0 +1,19 @@
package observability
import (
"context"
"go.opentelemetry.io/otel/propagation"
)
func Carrier(ctx context.Context) propagation.MapCarrier {
carrier := propagation.MapCarrier{}
(propagation.TraceContext{}).Inject(ctx, carrier)
return carrier
}
func JobContext(ctx context.Context, traceparent, tracestate string) context.Context {
return (propagation.TraceContext{}).Extract(ctx, propagation.MapCarrier{
"traceparent": traceparent, "tracestate": tracestate,
})
}
+109
View File
@@ -0,0 +1,109 @@
package observability
import (
"context"
"errors"
"log/slog"
"os"
"time"
"go.opentelemetry.io/otel"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp"
"go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp"
"go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp"
"go.opentelemetry.io/otel/log/global"
"go.opentelemetry.io/otel/propagation"
sdklog "go.opentelemetry.io/otel/sdk/log"
sdkmetric "go.opentelemetry.io/otel/sdk/metric"
"go.opentelemetry.io/otel/sdk/resource"
sdktrace "go.opentelemetry.io/otel/sdk/trace"
)
const scope = "highload/messenger"
const exportTimeout = 1500 * time.Millisecond
var W3C = propagation.TraceContext{}
// Start initializes all three signals. Export failures are asynchronous and
// telemetry configuration is independent of business readiness.
func Start(ctx context.Context, fallbackName string) func() {
if os.Getenv("OTEL_EXPORTER_OTLP_ENDPOINT") == "" {
return func() {}
}
// Exporter errors can contain endpoints. Suppress the raw error and limit
// stderr noise to one message per second for a broken telemetry sink.
otel.SetErrorHandler(otel.ErrorHandlerFunc(func(error) {
reportExportError()
}))
shutdown, err := start(ctx, fallbackName)
if err != nil {
slog.Warn("OpenTelemetry initialization failed")
return func() {}
}
return func() {
ctx, cancel := context.WithTimeout(context.Background(), 4*time.Second)
defer cancel()
if err := shutdown(ctx); err != nil {
slog.Warn("OpenTelemetry shutdown could not flush every signal")
}
}
}
func start(ctx context.Context, fallbackName string) (func(context.Context) error, error) {
serviceName := os.Getenv("OTEL_SERVICE_NAME")
if serviceName == "" {
serviceName = fallbackName
}
version := os.Getenv("OTEL_SERVICE_VERSION")
if version == "" {
version = "lab6"
}
instance, err := os.Hostname()
if err != nil {
return nil, err
}
res := resource.NewSchemaless(
attribute.String("service.name", serviceName),
attribute.String("service.version", version),
attribute.String("service.instance.id", instance),
attribute.String("deployment.environment.name", "lab"),
)
traces, err := otlptracehttp.New(ctx, otlptracehttp.WithTimeout(exportTimeout),
otlptracehttp.WithRetry(otlptracehttp.RetryConfig{Enabled: false}))
if err != nil {
return nil, err
}
metrics, err := otlpmetrichttp.New(ctx, otlpmetrichttp.WithTimeout(exportTimeout),
otlpmetrichttp.WithRetry(otlpmetrichttp.RetryConfig{Enabled: false}))
if err != nil {
_ = traces.Shutdown(ctx)
return nil, err
}
logs, err := otlploghttp.New(ctx, otlploghttp.WithTimeout(exportTimeout),
otlploghttp.WithRetry(otlploghttp.RetryConfig{Enabled: false}))
if err != nil {
_ = metrics.Shutdown(ctx)
_ = traces.Shutdown(ctx)
return nil, err
}
tp := sdktrace.NewTracerProvider(sdktrace.WithResource(res),
sdktrace.WithSampler(sdktrace.ParentBased(sdktrace.AlwaysSample())),
sdktrace.WithBatcher(traces, sdktrace.WithMaxQueueSize(512), sdktrace.WithMaxExportBatchSize(128),
sdktrace.WithBatchTimeout(500*time.Millisecond), sdktrace.WithExportTimeout(exportTimeout)))
mp := sdkmetric.NewMeterProvider(sdkmetric.WithResource(res),
sdkmetric.WithReader(sdkmetric.NewPeriodicReader(metrics,
sdkmetric.WithInterval(2*time.Second), sdkmetric.WithTimeout(exportTimeout))))
lp := sdklog.NewLoggerProvider(sdklog.WithResource(res),
sdklog.WithProcessor(sdklog.NewBatchProcessor(logs, sdklog.WithMaxQueueSize(512),
sdklog.WithExportMaxBatchSize(128), sdklog.WithExportInterval(500*time.Millisecond),
sdklog.WithExportTimeout(exportTimeout))))
otel.SetTracerProvider(tp)
otel.SetMeterProvider(mp)
global.SetLoggerProvider(lp)
otel.SetTextMapPropagator(W3C)
observeProcess(mp.Meter(scope))
return func(ctx context.Context) error {
return errors.Join(tp.Shutdown(ctx), mp.Shutdown(ctx), lp.Shutdown(ctx))
}, nil
}
+143
View File
@@ -0,0 +1,143 @@
package observability
import (
"context"
"encoding/json"
"errors"
"log/slog"
"os"
"strconv"
"strings"
"sync/atomic"
"syscall"
"time"
"go.opentelemetry.io/otel"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/codes"
"go.opentelemetry.io/otel/log"
"go.opentelemetry.io/otel/log/global"
"go.opentelemetry.io/otel/metric"
"go.opentelemetry.io/otel/trace"
)
var Tracer = otel.Tracer(scope)
var meter = otel.Meter(scope)
var (
httpRequests = instrument(meter.Int64Counter("messenger_http_requests"))
httpLatency = instrument(meter.Float64Histogram("messenger_http_request_duration", metric.WithUnit("s"),
metric.WithExplicitBucketBoundaries(.001, .003, .005, .01, .025, .05, .1, .25, .5, 1, 2, 5, 10)))
jobs = instrument(meter.Int64Counter("messenger_jobs"))
jobLatency = instrument(meter.Float64Histogram("messenger_job_duration", metric.WithUnit("s"),
metric.WithExplicitBucketBoundaries(.01, .025, .05, .1, .25, .5, 1, 2, 5, 10, 20, 40, 60, 120)))
readyLatency = instrument(meter.Float64Histogram("messenger_attachment_time_to_ready", metric.WithUnit("s"),
metric.WithExplicitBucketBoundaries(.05, .1, .25, .5, 1, 2, 5, 10, 20, 40, 60, 120, 300)))
lastExportError atomic.Int64
)
func instrument[T any](value T, err error) T {
if err != nil {
panic(err) // All instrument names and descriptors are code constants.
}
return value
}
func reportExportError() {
now := time.Now().Unix()
last := lastExportError.Load()
if last != now && lastExportError.CompareAndSwap(last, now) {
slog.Warn("OpenTelemetry export failed")
}
}
func ObserveHTTP(ctx context.Context, method, route string, code int, duration time.Duration) {
attrs := metric.WithAttributes(attribute.String("method", method), attribute.String("route", route),
attribute.String("status_class", strconv.Itoa(code/100)+"xx"))
httpRequests.Add(ctx, 1, attrs)
httpLatency.Record(ctx, duration.Seconds(), attrs)
}
func ObserveJob(ctx context.Context, outcome string, duration time.Duration) {
attrs := metric.WithAttributes(attribute.String("job_type", "attachment.process"), attribute.String("outcome", outcome))
jobs.Add(ctx, 1, attrs)
jobLatency.Record(ctx, duration.Seconds(), attrs)
}
func ObserveReady(ctx context.Context, acceptedAt time.Time) {
readyLatency.Record(ctx, max(time.Since(acceptedAt).Seconds(), 0),
metric.WithAttributes(attribute.String("job_type", "attachment.process")))
}
// Log exports only deliberate operational fields, never arbitrary slog values.
// The JSON body allows LogQL queries and Grafana's trace link to use the same IDs.
func Log(ctx context.Context, event string, level log.Severity, fields ...attribute.KeyValue) {
values := make(map[string]any, len(fields)+3)
values["event"] = event
for _, field := range fields {
values[string(field.Key)] = field.Value.AsInterface()
}
if span := trace.SpanContextFromContext(ctx); span.IsValid() {
values["trace_id"], values["span_id"] = span.TraceID().String(), span.SpanID().String()
}
if requestID := RequestID(ctx); requestID != "" {
values["request_id"] = requestID
}
body, err := json.Marshal(values)
if err != nil {
return
}
var record log.Record
record.SetTimestamp(time.Now())
record.SetSeverity(level)
record.SetSeverityText(level.String())
record.SetBody(log.StringValue(string(body)))
global.GetLoggerProvider().Logger(scope).Emit(ctx, record)
}
type requestIDKey struct{}
func WithRequestID(ctx context.Context, id string) context.Context {
return context.WithValue(ctx, requestIDKey{}, id)
}
func RequestID(ctx context.Context) string {
id, _ := ctx.Value(requestIDKey{}).(string)
return id
}
func End(span trace.Span, err error) {
if err != nil {
kind := "operation_failed"
switch {
case errors.Is(err, context.DeadlineExceeded):
kind = "deadline_exceeded"
case errors.Is(err, context.Canceled):
kind = "canceled"
}
span.SetAttributes(attribute.String("error.type", kind))
span.SetStatus(codes.Error, kind)
}
span.End()
}
func observeProcess(m metric.Meter) {
cpu := instrument(m.Float64ObservableCounter("messenger_process_cpu_time", metric.WithUnit("s")))
rss := instrument(m.Int64ObservableGauge("messenger_process_memory", metric.WithUnit("By")))
_, _ = m.RegisterCallback(func(_ context.Context, observer metric.Observer) error {
var usage syscall.Rusage
if err := syscall.Getrusage(syscall.RUSAGE_SELF, &usage); err == nil {
seconds := float64(usage.Utime.Sec+usage.Stime.Sec) + float64(usage.Utime.Usec+usage.Stime.Usec)/1e6
observer.ObserveFloat64(cpu, seconds)
}
// The production binaries run on Linux. statm reports resident pages.
if raw, err := os.ReadFile("/proc/self/statm"); err == nil {
if fields := strings.Fields(string(raw)); len(fields) > 1 {
if pages, err := strconv.ParseInt(fields[1], 10, 64); err == nil {
observer.ObserveInt64(rss, pages*int64(os.Getpagesize()))
}
}
}
return nil
}, cpu, rss)
}
+64
View File
@@ -0,0 +1,64 @@
package pipeline
import (
"context"
"log/slog"
"time"
"highload/internal/repository/postgres"
"highload/internal/storage"
)
type Cleanup struct {
Repo *postgres.AttachmentRepository
Objects storage.ObjectCleanup
Grace time.Duration
Retention time.Duration
DryRun bool
}
func (c Cleanup) RunOnce(ctx context.Context) (int, error) {
policy := postgres.CleanupPolicy{Now: time.Now(), Grace: c.Grace, Retention: c.Retention, DryRun: c.DryRun}
removed := 0
for _, prefix := range []string{"staging/", "results/", "attachments/"} {
var token *string
for {
pageCtx, cancel := context.WithTimeout(ctx, 5*time.Second)
objects, next, err := c.Objects.ListPage(pageCtx, token, prefix)
cancel()
if err != nil {
return removed, err
}
for _, obj := range objects {
eligible, err := c.Repo.CleanupObject(ctx, obj, policy, c.Objects.Delete)
if err != nil {
return removed, err
}
if eligible {
removed++
}
}
if next == nil {
break
}
token = next
}
}
return removed, nil
}
func (c Cleanup) Serve(ctx context.Context, period time.Duration) {
for ctx.Err() == nil {
count, err := c.RunOnce(ctx)
if err != nil && ctx.Err() == nil {
slog.Warn("cleanup pass failed", "error", err)
}
if count > 0 {
slog.Info("cleanup pass", "objects", count, "dry_run", c.DryRun)
}
select {
case <-ctx.Done():
case <-time.After(period):
}
}
}
+410
View File
@@ -0,0 +1,410 @@
package pipeline
import (
"bytes"
"context"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"io"
"log/slog"
"math/rand/v2"
"os"
"path/filepath"
"time"
"highload/internal/broker"
"highload/internal/config"
"highload/internal/fault"
"highload/internal/imageproc"
"highload/internal/model"
"highload/internal/observability"
"highload/internal/repository"
"highload/internal/storage"
"github.com/google/uuid"
amqp "github.com/rabbitmq/amqp091-go"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/codes"
"go.opentelemetry.io/otel/log"
"go.opentelemetry.io/otel/trace"
)
type WorkerRepository interface {
ClaimJob(context.Context, string, time.Time) (*model.AttachmentJob, error)
GetAttachmentJobState(context.Context, string) (*model.AttachmentJobState, error)
GetAttachmentByID(context.Context, string) (*model.Attachment, error)
CompleteJob(context.Context, string, int32, string, []model.AttachmentVariant) (bool, error)
PublishedObjectKeys(context.Context, string, int32) ([]string, error)
ScheduleRetry(context.Context, string, int32, string, time.Time) (bool, error)
FailJob(context.Context, string, int32, string) (bool, error)
}
type WorkerObjects interface {
storage.ObjectWriter
Head(context.Context, string) error
}
type Worker struct {
Repo WorkerRepository
Objects WorkerObjects
Config config.Pipeline
}
type message struct {
SchemaVersion int `json:"schema_version"`
Type string `json:"type"`
JobID string `json:"job_id"`
AttachmentID string `json:"attachment_id"`
PipelineVersion int32 `json:"pipeline_version"`
Attempt int `json:"attempt"`
CreatedAt time.Time `json:"created_at"`
Traceparent string `json:"traceparent,omitempty"`
Tracestate string `json:"tracestate,omitempty"`
}
func decodeMessage(payload []byte) (message, error) {
var msg message
if len(payload) > 4096 {
return msg, errors.New("oversize job")
}
d := json.NewDecoder(bytes.NewReader(payload))
d.DisallowUnknownFields()
if err := d.Decode(&msg); err != nil {
return msg, err
}
if err := d.Decode(new(any)); err != io.EOF {
return msg, errors.New("trailing job data")
}
jobID, err1 := uuid.Parse(msg.JobID)
attachmentID, err2 := uuid.Parse(msg.AttachmentID)
if err1 != nil || err2 != nil || msg.SchemaVersion != 1 || msg.Type != "attachment.process.v1" ||
msg.PipelineVersion <= 0 || msg.Attempt <= 0 || msg.CreatedAt.IsZero() {
return msg, errors.New("invalid job envelope")
}
msg.JobID, msg.AttachmentID = jobID.String(), attachmentID.String()
return msg, nil
}
func (w Worker) Serve(ctx context.Context) {
for ctx.Err() == nil {
if err := w.consume(ctx); err != nil && ctx.Err() == nil {
slog.Warn("worker consumer reconnecting", "error", err)
}
select {
case <-ctx.Done():
case <-time.After(500 * time.Millisecond):
}
}
}
func (w Worker) consume(ctx context.Context) error {
conn, ch, err := broker.Connect(w.Config.BrokerURL)
if err != nil {
return err
}
defer broker.Close(conn)
if err := ch.Qos(w.Config.Prefetch, 0, false); err != nil {
return err
}
deliveries, err := ch.ConsumeWithContext(ctx, broker.Queue, "", false, false, false, false, nil)
if err != nil {
return err
}
for {
select {
case <-ctx.Done():
return ctx.Err()
case delivery, ok := <-deliveries:
if !ok {
return errors.New("broker delivery channel closed")
}
msg, err := decodeMessage(delivery.Body)
if err != nil {
slog.Warn("rejecting invalid job envelope")
if err := delivery.Reject(false); err != nil {
return err
}
continue
}
if err := w.consumeDelivery(ctx, delivery, msg); err != nil {
// Closing the channel requeues an unacknowledged delivery.
return err
}
}
}
}
func (w Worker) consumeDelivery(ctx context.Context, delivery amqp.Delivery, msg message) (err error) {
ctx = observability.JobContext(ctx, msg.Traceparent, msg.Tracestate)
ctx, span := observability.Tracer.Start(ctx, "attachment.process",
trace.WithSpanKind(trace.SpanKindConsumer), trace.WithAttributes(
attribute.String("messaging.system", "rabbitmq"),
attribute.String("messaging.destination.name", broker.Queue),
attribute.String("messaging.operation.name", "process"),
attribute.String("messaging.message.id", msg.JobID),
))
defer func() { observability.End(span, err) }()
observability.Log(ctx, "job.received", log.SeverityInfo,
attribute.String("job_id", msg.JobID), attribute.Int("envelope_attempt", msg.Attempt))
reject, err := w.handle(ctx, msg)
if err != nil {
return err
}
if reject {
span.SetAttributes(attribute.String("messaging.delivery.disposition", "rejected"))
return delivery.Reject(false)
}
span.SetAttributes(attribute.String("messaging.delivery.disposition", "acked"))
return delivery.Ack(false)
}
func (w Worker) handle(ctx context.Context, msg message) (bool, error) {
state, err := w.Repo.GetAttachmentJobState(ctx, msg.JobID)
if errors.Is(err, repository.ErrNotFound) {
return true, nil
}
if err != nil {
return false, err
}
if state.AttachmentID != msg.AttachmentID || state.PipelineVersion != msg.PipelineVersion {
return true, nil
}
if state.Status == "superseded" {
return false, nil
}
job, err := w.Repo.ClaimJob(ctx, msg.JobID, time.Now().Add(w.Config.Lease))
if err != nil {
return false, err
}
if job == nil {
return false, w.handleUnclaimed(ctx, msg)
}
slog.Info("job claimed", "job_id", job.ID, "attempt", job.AttemptsStarted, "lease_generation", job.LeaseGeneration)
outcome, err := w.runAttempt(ctx, job, msg)
if err != nil || outcome != "ready" {
return false, err
}
pauseCtx, pauseCancel := context.WithTimeout(ctx, 30*time.Second)
defer pauseCancel()
return false, fault.Pause(pauseCtx, "before_ack", job.ID)
}
func (w Worker) runAttempt(ctx context.Context, job *model.AttachmentJob, msg message) (outcome string, err error) {
started := time.Now()
span := trace.SpanFromContext(ctx)
span.SetAttributes(attribute.Int("job.attempt", int(job.AttemptsStarted)))
done := observability.WorkerAttempt()
defer func() {
done()
observability.ObserveJob(ctx, outcome, time.Since(started))
span.SetAttributes(attribute.String("job.outcome", outcome))
if outcome != "ready" {
span.SetStatus(codes.Error, outcome)
}
observability.Log(ctx, "job.finished", log.SeverityInfo,
attribute.String("job_id", job.ID), attribute.Int("attempt", int(job.AttemptsStarted)),
attribute.String("outcome", outcome))
}()
attemptCtx, cancel := context.WithTimeout(ctx, w.Config.Timeout)
failure := w.process(attemptCtx, job)
cancel()
if ctx.Err() != nil {
return "interrupted", ctx.Err()
}
if failure == nil {
return "ready", nil
}
if errors.Is(failure, errLostLease) {
return "lease_lost", w.handleUnclaimed(ctx, msg)
}
return w.saveFailure(ctx, job, failure)
}
func (w Worker) handleUnclaimed(ctx context.Context, msg message) error {
state, err := w.Repo.GetAttachmentJobState(ctx, msg.JobID)
if err != nil {
return err
}
switch state.Status {
case "ready":
checkCtx, cancel := context.WithTimeout(ctx, 5*time.Second)
defer cancel()
keys, err := w.Repo.PublishedObjectKeys(checkCtx, state.AttachmentID, state.PipelineVersion)
if err != nil {
return err
}
for _, key := range keys {
if err := w.Objects.Head(checkCtx, key); err != nil {
return err
}
}
return nil
case "failed", "queued", "superseded":
return nil // A queued retry is already in the transactionally durable outbox.
case "processing":
if state.AttemptsStarted >= state.MaxAttempts && state.LeaseExpired {
_, err = w.Repo.FailJob(ctx, msg.JobID, state.LeaseGeneration, "retry_exhausted")
return err
}
return nil // The recovery loop publishes one event for this expired lease.
default:
return errors.New("unrecognized job state")
}
}
var (
errLostLease = errors.New("job lease lost")
errOriginalMismatch = errors.New("staging hash or size differs")
)
func (w Worker) process(ctx context.Context, job *model.AttachmentJob) error {
att, err := w.Repo.GetAttachmentByID(ctx, job.AttachmentID)
if err != nil {
return err
}
if att.PipelineVersion != job.PipelineVersion {
return errLostLease
}
if att.StagingKey == "" {
return errOriginalMismatch
}
dir, err := os.MkdirTemp("", "attachment-job-")
if err != nil {
return err
}
defer os.RemoveAll(dir)
source := filepath.Join(dir, "input")
if err := w.download(ctx, att, source); err != nil {
return err
}
files, err := imageproc.Generate(ctx, dir, source, att.ContentType, job.PipelineVersion)
if err != nil {
return err
}
if err := fault.Pause(ctx, "before_objects", job.ID); err != nil {
return err
}
prefix := fmt.Sprintf("results/%s/v%d/%d/", job.AttachmentID, job.PipelineVersion, job.LeaseGeneration)
originalKey := prefix + "original"
if err := w.putFile(ctx, originalKey, att.ContentType, source, att.SizeBytes); err != nil {
return err
}
variants := make([]model.AttachmentVariant, 0, len(files))
for _, f := range files {
v := f.Variant
v.ObjectKey = prefix + filepath.Base(f.Path)
if err := w.putFile(ctx, v.ObjectKey, v.ContentType, f.Path, v.SizeBytes); err != nil {
return err
}
variants = append(variants, v)
}
if err := fault.Pause(ctx, "after_objects", job.ID); err != nil {
return err
}
done, err := w.Repo.CompleteJob(ctx, job.ID, job.LeaseGeneration, originalKey, variants)
if err != nil {
return err
}
if !done {
slog.Info("job finalization rejected", "job_id", job.ID, "lease_generation", job.LeaseGeneration)
return errLostLease
}
observability.ObserveReady(ctx, job.AcceptedAt)
observability.Log(ctx, "job.ready", log.SeverityInfo, attribute.String("job_id", job.ID))
slog.Info("job ready", "job_id", job.ID, "lease_generation", job.LeaseGeneration)
return nil
}
func (w Worker) download(ctx context.Context, att *model.Attachment, path string) error {
if att.SizeBytes <= 0 || att.SizeBytes > imageproc.MaxBytes {
return errOriginalMismatch
}
reader, err := w.Objects.Get(ctx, att.StagingKey)
if err != nil {
return err
}
defer reader.Close()
f, err := os.Create(path)
if err != nil {
return err
}
sum := sha256.New()
n, copyErr := io.Copy(io.MultiWriter(f, sum), io.LimitReader(reader, att.SizeBytes+1))
closeErr := f.Close()
if copyErr != nil {
return copyErr
}
if closeErr != nil {
return closeErr
}
if n != att.SizeBytes || hex.EncodeToString(sum.Sum(nil)) != att.SHA256 {
return errOriginalMismatch
}
return nil
}
func (w Worker) putFile(ctx context.Context, key, mime, path string, size int64) error {
f, err := os.Open(path)
if err != nil {
return err
}
defer f.Close()
return w.Objects.Put(ctx, key, mime, f, size)
}
func (w Worker) saveFailure(ctx context.Context, job *model.AttachmentJob, failure error) (string, error) {
code := "processing_failed"
permanent := false
switch {
case errors.Is(failure, errOriginalMismatch), errors.Is(failure, imageproc.ErrInvalidImage):
code, permanent = "invalid_image", true
case errors.Is(failure, imageproc.ErrResourceLimit):
code, permanent = "resource_limit", true
case errors.Is(failure, storage.ErrObjectNotFound):
code, permanent = "original_missing", true
case errors.Is(failure, storage.ErrServiceUnavailable):
code = "s3_unavailable"
case errors.Is(failure, context.DeadlineExceeded):
code = "processing_timeout"
}
dbCtx, cancel := context.WithTimeout(ctx, 5*time.Second)
defer cancel()
if permanent || job.AttemptsStarted >= job.MaxAttempts {
if !permanent {
code = "retry_exhausted"
}
changed, err := w.Repo.FailJob(dbCtx, job.ID, job.LeaseGeneration, code)
if err != nil {
return "error", err
}
if !changed {
return "lease_lost", nil
}
observability.Log(ctx, "job.failed", log.SeverityWarn, attribute.String("job_id", job.ID),
attribute.String("error_code", code))
slog.Warn("job failed", "job_id", job.ID, "error_code", code)
return "failed", nil
}
// The floor avoids immediate requeue loops. The upper bound grows with
// each attempt and is capped to keep the lab recoverable.
upper := min(30*time.Second, w.Config.RetryBase<<min(job.AttemptsStarted-1, 4))
delay := time.Second + time.Duration(rand.Int64N(int64(upper-time.Second+1)))
changed, err := w.Repo.ScheduleRetry(dbCtx, job.ID, job.LeaseGeneration, code, time.Now().Add(delay))
if err != nil {
return "error", err
}
if !changed {
return "lease_lost", nil
}
observability.Log(ctx, "job.retry_scheduled", log.SeverityInfo, attribute.String("job_id", job.ID),
attribute.String("error_code", code), attribute.Int64("delay_ms", delay.Milliseconds()))
slog.Info("job retry scheduled", "job_id", job.ID, "error_code", code, "delay_ms", delay.Milliseconds())
return "retry", nil
}
+36
View File
@@ -0,0 +1,36 @@
package repository
import (
"context"
"highload/internal/model"
"time"
)
type AttachmentRepository interface {
CreateAttachment(ctx context.Context, params CreateAttachmentParams) (*model.Attachment, error)
GetAttachmentForUser(ctx context.Context, userID, attachmentID string) (*model.Attachment, error)
ExistsByObjectKey(ctx context.Context, key string) (bool, error)
WithObjectKeyLock(ctx context.Context, key string, fn func(context.Context, AttachmentTx) error) (bool, error)
}
type AttachmentKey struct {
ID string
ObjectKey string
CreatedAt time.Time
}
type AttachmentReconcileRepository interface {
ListAttachmentKeys(ctx context.Context, afterKey string, pageSize int32) ([]AttachmentKey, error)
GetAttachmentByID(ctx context.Context, id string) (*model.Attachment, error)
WithObjectKeyLock(ctx context.Context, key string, fn func(context.Context, AttachmentTx) error) (bool, error)
}
type CreateAttachmentParams struct {
ChatID string
OwnerID string
Filename string
ObjectKey string
ContentType string
Sha256 string
SizeBytes int64
}
+13
View File
@@ -0,0 +1,13 @@
package repository
import (
"context"
"highload/internal/model"
)
type AttachmentTx interface {
CreateAttachment(ctx context.Context, params CreateAttachmentParams) (*model.Attachment, error)
CreateQueued(ctx context.Context, id, chatID, ownerID, mime, sha256, key, filename string, size int64) (*model.Attachment, error)
ExistsByObjectKey(ctx context.Context, key string) (bool, error)
GetAttachmentByID(ctx context.Context, id string) (*model.Attachment, error)
}

Some files were not shown because too many files have changed in this diff Show More