ЛР 6 — БВТ2303 — Гришко — на 4 #24
@@ -4,11 +4,28 @@ CA_FILE ?=
|
||||
ACTION_SCRIPT ?=
|
||||
TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",)
|
||||
|
||||
CLEANUP_ARGS ?= --dry-run
|
||||
SERVICES ?=
|
||||
JOB_ID ?=
|
||||
ATTACHMENT_ID ?=
|
||||
REASON ?=
|
||||
export JOB_ID ATTACHMENT_ID REASON
|
||||
|
||||
COMPOSE = docker compose --project-name highload-lab6 --env-file .env -f compose.yml
|
||||
|
||||
.PHONY: help check validate test load test-persistence test-failover
|
||||
.PHONY: app-init app-build app-up app-start app-stop app-down app-restart app-logs app-status
|
||||
.PHONY: app-migrate app-s3-bootstrap worker-start worker-stop queue-status attachments-cleanup
|
||||
.PHONY: jobs-failed jobs-replay jobs-upgrade jobs-cleanup-staging
|
||||
.PHONY: test-backend test-pipeline frontend-build
|
||||
|
||||
|
||||
help:
|
||||
@echo "check: offline structure; validate: full OpenAPI (requirements-dev.txt); test: running API; load: synthetic writes"
|
||||
|
||||
@echo "app-init / app-build / app-up: configure, build, start the stack"
|
||||
@echo "app-status / app-logs / app-down: inspect or stop the stack"
|
||||
|
||||
check:
|
||||
$(PYTHON) scripts/check_contract.py
|
||||
|
||||
@@ -28,3 +45,77 @@ test-persistence:
|
||||
test-failover:
|
||||
@test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable stop-one hook"; exit 2)
|
||||
$(PYTHON) tests/resilience.py --mode failover --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS)
|
||||
|
||||
app-init:
|
||||
$(PYTHON) scripts/init_env.py
|
||||
|
||||
app-build: app-init
|
||||
$(COMPOSE) build
|
||||
|
||||
app-up: app-init
|
||||
$(COMPOSE) up -d --build --wait
|
||||
|
||||
app-start: app-init
|
||||
$(COMPOSE) up -d --no-build --wait
|
||||
|
||||
app-stop:
|
||||
$(COMPOSE) stop
|
||||
|
||||
app-down:
|
||||
$(COMPOSE) down
|
||||
|
||||
app-restart: app-init
|
||||
./scripts/restart.sh
|
||||
|
||||
app-logs:
|
||||
$(COMPOSE) logs -f --tail=100 $(SERVICES)
|
||||
|
||||
app-status:
|
||||
$(COMPOSE) ps --all
|
||||
|
||||
app-migrate: app-init
|
||||
$(COMPOSE) run --rm migrate
|
||||
|
||||
app-s3-bootstrap: app-init
|
||||
$(COMPOSE) run --rm s3-bootstrap
|
||||
|
||||
worker-start:
|
||||
$(COMPOSE) start worker
|
||||
|
||||
worker-stop:
|
||||
$(COMPOSE) stop worker
|
||||
|
||||
queue-status:
|
||||
$(COMPOSE) exec -T broker rabbitmqctl -q list_queues name messages_ready messages_unacknowledged --formatter json
|
||||
|
||||
attachments-cleanup:
|
||||
$(COMPOSE) run --rm --no-deps upload-cleanup ./messenger attachments-cleanup $(CLEANUP_ARGS)
|
||||
|
||||
jobs-failed:
|
||||
$(COMPOSE) exec -T dispatcher ./messenger jobs list-failed
|
||||
|
||||
jobs-replay:
|
||||
@test -n "$$JOB_ID" && test -n "$$REASON" || { echo "Set JOB_ID=... REASON='...'"; exit 2; }
|
||||
$(COMPOSE) exec -T dispatcher ./messenger jobs replay "$$JOB_ID" "$$REASON"
|
||||
|
||||
jobs-upgrade:
|
||||
@test -n "$$ATTACHMENT_ID" && test -n "$$REASON" || { echo "Set ATTACHMENT_ID=... REASON='...'"; exit 2; }
|
||||
$(COMPOSE) exec -T dispatcher ./messenger jobs upgrade "$$ATTACHMENT_ID" 2 "$$REASON"
|
||||
|
||||
jobs-cleanup-staging:
|
||||
@test -n "$$JOB_ID" || { echo "Set JOB_ID=..."; exit 2; }
|
||||
$(COMPOSE) exec -T dispatcher ./messenger jobs cleanup-staging "$$JOB_ID"
|
||||
|
||||
test-backend: app-init
|
||||
$(COMPOSE) --profile test run --rm --build backend-tests
|
||||
|
||||
test-pipeline: app-init
|
||||
$(COMPOSE) --profile test run --rm --build backend-tests go test -race -count=1 -v ./backend/tests ./backend/workers -run 'TestPipeline|TestVersionedRendering|TestThumbnail|TestOptimized'
|
||||
|
||||
frontend-build:
|
||||
$(COMPOSE) exec -T frontend npm run build
|
||||
|
||||
|
||||
.PHONY: evidence
|
||||
evidence:
|
||||
$(PYTHON) scripts/generate_evidence.py --base-url "$(BASE_URL)"
|
||||
|
||||
@@ -1,37 +1,113 @@
|
||||
# Отчёт — лабораторная 6
|
||||
|
||||
- Автор, группа: TODO
|
||||
- Целевая оценка; выбранный трек 5, если нужен: TODO
|
||||
- Commit/PR: TODO после публикации
|
||||
- Использованные источники, библиотеки и помощь ИИ: TODO
|
||||
- Автор — Гришко Дмитрий Михайлович, БВТ2303.
|
||||
- Целевая оценка — 4.
|
||||
- PR: https://gitea.privcloud.tech/mtusi2026/mtusi2026lab6/pulls/24.
|
||||
- Источники — [задание](README.md), [контракт наблюдаемости](contracts/observability.md), [OpenAPI](contracts/openapi.json), [OpenTelemetry Go](https://opentelemetry.io/docs/languages/go/instrumentation/), [OTLP в Loki](https://grafana.com/docs/loki/latest/send-data/otel/). Зависимости закреплены в [go.mod](src/go.mod). Помощь ИИ — OpenAI Codex, реализация, проверки и оформление.
|
||||
|
||||
## Запуск из чистого clone
|
||||
## Запуск из чистого клона
|
||||
|
||||
TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных.
|
||||
Нужны Linux, Git, Make, Python 3.9+, Docker Engine с Compose и доступ к реестрам образов и зависимостей при первой сборке. Склонируйте ветку решения и перейдите в каталог проекта.
|
||||
|
||||
```sh
|
||||
git clone --branch solution/Grishko-BVT2303 --single-branch \
|
||||
https://gitea.privcloud.tech/TAskMAster339/mtusi2026lab6-fork.git
|
||||
cd mtusi2026lab6-fork
|
||||
```
|
||||
|
||||
Создайте локальную конфигурацию, проверьте Compose и запустите стенд.
|
||||
|
||||
```sh
|
||||
make app-init
|
||||
docker compose config --quiet
|
||||
make app-up
|
||||
curl --fail http://127.0.0.1:8080/health/ready
|
||||
```
|
||||
|
||||
После ответа `{"status":"ok"}` запустите проверки.
|
||||
|
||||
```sh
|
||||
make check
|
||||
make test BASE_URL=http://127.0.0.1:8080
|
||||
make evidence BASE_URL=http://127.0.0.1:8080
|
||||
```
|
||||
|
||||
`app-init` создаёт локальные `.env` и `secrets/`. Compose применяет миграции и создаёт приватный S3 bucket до запуска API и worker. По умолчанию API доступен на 8080, Grafana на 3000, UI на 5173, S3 на 9000 и статистика HAProxy на 8404. Dashboard открывается в гостевом режиме. Для Explore при первом входе используются стандартные локальные данные Grafana с последующей сменой пароля.
|
||||
|
||||
`make app-down` останавливает стенд с сохранением томов. Для намеренного удаления учебных данных используется `docker compose down --volumes`.
|
||||
|
||||
Приложение собирается Go 1.22.2 в Debian 12.15. Frontend использует Node 22. В Compose закреплены PostgreSQL 16.10, Redis 7.4.11, RabbitMQ 4.2.0, HAProxy 3.2.25, Collector Contrib 0.120.0, Prometheus 3.2.1, Loki 3.4.2, Tempo 2.7.1 и Grafana 11.5.2. MinIO собирается из закреплённого коммита [Dockerfile](infra/minio.Dockerfile). OpenTelemetry Go использует 1.34.0 для трасс и метрик и 0.10.0 для логов.
|
||||
|
||||
## Архитектура и решения
|
||||
|
||||
TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab5, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации.
|
||||
```text
|
||||
Клиент → HAProxy → API ×2 → PostgreSQL / Redis / S3 staging
|
||||
PostgreSQL outbox → dispatcher → RabbitMQ → worker → S3 / PostgreSQL
|
||||
API / dispatcher / worker → OTLP HTTP → Collector
|
||||
├→ Prometheus (scrape)
|
||||
├→ Loki (OTLP)
|
||||
└→ Tempo (OTLP)
|
||||
Grafana → Prometheus / Loki / Tempo
|
||||
```
|
||||
|
||||
Из №5 перенесены сессии Redis, балансировка, идемпотентность сообщений, приватный S3 и фоновая обработка. API отвечает 202 после записи оригинала в устойчивый staging и фиксации метаданных и задания в PostgreSQL. Dispatcher подтверждает публикацию через RabbitMQ. Worker завершает обработку и фиксирует результат или намерение повторной попытки перед ACK. PostgreSQL хранит бизнес-данные и состояние задания. Готовность API зависит от PostgreSQL, Redis и S3.
|
||||
|
||||
Миграция 10 добавляет W3C `traceparent` и `tracestate` в outbox. Dispatcher восстанавливает контекст, создаёт producer span и переносит его в заголовки AMQP. Worker создаёт дочерний consumer span с номером попытки. Повторная публикация восстанавливает тот же контекст upload. В Tempo доступны спаны S3 и PostgreSQL, а Loki хранит trace ID, span ID, исход операции и технические поля корреляции. SQL, параметры запросов, заголовки авторизации, тексты и содержимое файлов не включаются в события. В метриках маршруты представлены шаблонами.
|
||||
|
||||
Prometheus хранит временные ряды, Loki — журналы, Tempo — трассы. У хранилищ отдельные тома. Grafana загружает источники и dashboard из [provisioning](infra/otel/grafana/provisioning). Collector использует batch, memory limiter 128 МиБ, лимит контейнера 192 МиБ и очереди экспорта Loki/Tempo по 64 пакета с retry до 30 секунд. Экспорт выполняется асинхронно.
|
||||
|
||||
Compose использует `APP_*`, `API_REPLICAS`, `SESSION_TTL_SECONDS`, `IDEMPOTENCY_TTL_SECONDS`, параметры S3, файлы подключения и `JOB_MAX_ATTEMPTS` / `JOB_TIMEOUT_SECONDS`. Сессии используют Redis. Конкурентность задаётся количеством процессов worker, по одному слоту на экземпляр. Пределы API — 10 МиБ и 40 млн пикселей, worker — 8 млн по умолчанию через `WORKER_MAX_PIXELS`. Failed staging хранится 24 часа. Код задаёт ресурсные атрибуты и имя сервиса по роли процесса, использует OTLP HTTP/protobuf. Endpoint задаётся через `OTEL_EXPORTER_OTLP_ENDPOINT`.
|
||||
|
||||
## Доказательства критериев
|
||||
|
||||
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
|
||||
| Критерий | Команда или сценарий | Наблюдение | Доказательство |
|
||||
| --- | --- | --- | --- |
|
||||
| Публичный smoke | make test | TODO | TODO |
|
||||
| Остальные пункты уровня 3 | TODO | TODO | TODO |
|
||||
| Все пункты уровня 4, если заявлен | TODO | TODO | TODO |
|
||||
| Выбранный трек 5, если заявлен | TODO | TODO | TODO |
|
||||
| 3. Минимум №5 | `make test` | Сессии, ACL, сообщения, идемпотентность, 202 → ready, оригинал и thumbnail | [Smoke](evidence/criteria/grade3-smoke.md) |
|
||||
| 3. Единый стенд, хранилища, provisioning | Compose, Collector validate, Grafana API | Тома и три источника данных, импортированный dashboard | [Стенд](evidence/criteria/grade3-stand.md) |
|
||||
| 3. OTLP и корреляция сигналов | `check_observability.py signals` | Метрики API и worker, совпадение trace/span ID в Loki и Tempo, проверка контрольных чувствительных значений | [Сигналы](evidence/criteria/grade3-signals.md) |
|
||||
| 4. Контекст через брокер и retry | `pipeline`, затем временная остановка S3 | Родительская цепочка до worker и операции S3/DB, обе попытки в одной трассе | [Трассировка](evidence/criteria/grade4-pipeline.md) |
|
||||
| 4. Диагностические метрики и SLO | `check_observability.py dashboard`, `promtool check rules` | PromQL для панелей, SLI за 30 минут, два валидных правила | [Dashboard и SLO](evidence/criteria/grade4-dashboard.md) |
|
||||
| 4. Алерт возраста очереди | Остановка worker | `MessengerQueueLag` firing → resolved, принятое вложение стало ready | [Очередь](evidence/criteria/grade4-queue.md) |
|
||||
| 4. Алерт доли 5xx | Временная блокировка таблицы чатов | `MessengerHighErrorRate` firing → resolved, задержка видна в дочернем спане PostgreSQL | [PostgreSQL](evidence/criteria/grade4-database.md) |
|
||||
| 4. Недоступность хранилища сигналов | Остановка Tempo | API принимает сообщения, worker обрабатывает вложение, Collector ограничивает буфер и регистрирует потерю спанов | [Tempo](evidence/criteria/grade4-telemetry.md) |
|
||||
|
||||
TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены.
|
||||
SLI — доля бизнес-запросов без 5xx, SLO ≥99% за 30 минут при наличии трафика. Health исключён. Перед итоговыми инцидентами SLI составил 99,93%. p50/p95/p99 вычисляются из агрегированных histogram buckets в секундах. Глубина очереди включает все незавершённые задания, в том числе ожидающие retry. Подробные запросы находятся в [dashboard JSON](infra/otel/grafana/dashboards/messenger.json), порядок диагностики — в [runbook](infra/otel/runbook.md).
|
||||
|
||||
## Эксперименты и отказы
|
||||
|
||||
TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения.
|
||||
Сценарии запускаются последовательно на локальном стенде. Проверяется, можно ли по сохранённым сигналам различить остановку обработчика и задержку в БД, а также сохраняется ли бизнес-путь при недоступности хранилища трасс.
|
||||
|
||||
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
|
||||
| --- | --- | --- | --- |
|
||||
| TODO | TODO | TODO | TODO |
|
||||
| Воздействие | Наблюдение | Восстановление |
|
||||
| --- | --- | --- |
|
||||
| Остановка worker | Возраст заданий растёт, alert firing после превышения 20 секунд в течение 15 секунд | Запуск worker, то же вложение ready, alert resolved |
|
||||
| Блокировка таблицы чатов | 5xx и долгий дочерний спан PostgreSQL. Правило проверяет порог 5% за минуту с выдержкой 15 секунд | Снятие блокировки, успешное чтение чата, alert resolved после выхода ошибок из окна |
|
||||
| Остановка S3 | Первая попытка обработки откладывается. Следующая видна в той же трассе | Запуск S3, исходное вложение ready |
|
||||
| Остановка Tempo | Сообщения приняты, вложение достигло ready. Collector отбросил 54 спана. Память по снимку — 56,43 МиБ из 192 МиБ | Tempo запущен после проверки |
|
||||
|
||||
Один короткий профиль нагрузки выполнен на AMD Ryzen 9 PRO 7940HS с 16 логическими CPU и 14 ГиБ RAM. Две API-реплики ограничены 0,5 CPU и 512 МиБ каждая. Worker ограничен 1 CPU и 512 МиБ. Профиль — 30 секунд, четыре параллельных клиента, closed-loop POST сообщений, один прогон без отдельного прогрева. В итоговом прогоне подтверждено 37 708 запросов, ошибок 0, пропускная способность 1256,75 RPS. Задержки p50/p95/p99 составили 2,828 / 4,327 / 7,211 мс. Полный вывод и снимок ресурсов после прогона сохранены в [результатах нагрузки](evidence/criteria/grade4-load.md).
|
||||
|
||||
### Скриншоты Grafana
|
||||
|
||||

|
||||
|
||||
*Рисунок 1. Во время нагрузочного прогона частота запросов выросла, а частота ответов `5xx` осталась на нуле. Панели p50, p95 и p99 показывают задержку API.*
|
||||
|
||||

|
||||
|
||||
*Рисунок 2. В одной трассе Tempo видны HTTP-ответ `202`, публикации в RabbitMQ и спаны worker. Первая обработка завершилась ошибкой, затем worker получил задание повторно. Trace ID — `9718ad3f6a66598ef72ed8ec1cb5101f`.*
|
||||
|
||||

|
||||
|
||||
*Рисунок 3. При остановке worker в очереди оставалось одно незавершённое задание. Возраст старейшего задания превысил 30 секунд. После возобновления обработки оба показателя вернулись к нулю.*
|
||||
|
||||

|
||||
|
||||
*Рисунок 4. В Grafana Explore запрос к метрике `ALERTS` показывает переход `MessengerQueueLag` из `pending` в `firing` во время роста возраста задания.*
|
||||
|
||||

|
||||
|
||||
*Рисунок 5. При блокировке таблицы чатов API вернул `503`. Ошибочный дочерний спан `postgresql query` занял около пяти секунд, почти всю длительность HTTP-запроса.*
|
||||
|
||||
## Ограничения и следующие шаги
|
||||
|
||||
TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза.
|
||||
Стенд локальный, PostgreSQL, S3, RabbitMQ и балансировщик развёрнуты в одном экземпляре. Отказы этих компонентов могут прерывать соответствующие бизнес-операции. Снимок памяти Collector относится к короткому отказу. При переполнении очереди или исчерпании повторов возможна потеря телеметрии. Результат closed-loop прогона зависит от клиента и не оценивает предел пропускной способности. Заявленная оценка — 4.
|
||||
|
||||
+386
@@ -0,0 +1,386 @@
|
||||
name: highload-lab6
|
||||
|
||||
services:
|
||||
redis:
|
||||
cpus: 0.25
|
||||
mem_limit: 128m
|
||||
networks: [data]
|
||||
image: redis:7.4.11-bookworm
|
||||
command:
|
||||
[
|
||||
"redis-server",
|
||||
"--appendonly",
|
||||
"yes",
|
||||
"--appendfsync",
|
||||
"everysec",
|
||||
"--dir",
|
||||
"/data",
|
||||
]
|
||||
volumes:
|
||||
- redis_data:/data
|
||||
healthcheck:
|
||||
test: ["CMD", "redis-cli", "ping"]
|
||||
interval: 3s
|
||||
timeout: 3s
|
||||
retries: 20
|
||||
db:
|
||||
cpus: 1.0
|
||||
mem_limit: 512m
|
||||
networks: [data]
|
||||
image: postgres:16.10-bookworm
|
||||
environment:
|
||||
POSTGRES_DB: ${POSTGRES_DB}
|
||||
POSTGRES_USER: ${POSTGRES_USER}
|
||||
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD}
|
||||
volumes:
|
||||
- postgres_data:/var/lib/postgresql/data
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "pg_isready -U $$POSTGRES_USER -d $$POSTGRES_DB"]
|
||||
interval: 3s
|
||||
timeout: 3s
|
||||
retries: 20
|
||||
migrate:
|
||||
cpus: 0.5
|
||||
mem_limit: 128m
|
||||
networks: [data]
|
||||
build: ./src
|
||||
environment:
|
||||
DATABASE_URL_FILE: ${DATABASE_URL_FILE:-/run/secrets/database_url}
|
||||
DATABASE_URL: postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@db:5432/${POSTGRES_DB}?sslmode=disable
|
||||
volumes:
|
||||
- ./secrets/database_url:${DATABASE_URL_FILE:-/run/secrets/database_url}:ro
|
||||
command: ["./messenger", "migrate", "up"]
|
||||
depends_on:
|
||||
db:
|
||||
condition: service_healthy
|
||||
s3:
|
||||
cpus: 1.0
|
||||
mem_limit: 512m
|
||||
networks: [edge, data]
|
||||
ports:
|
||||
- "127.0.0.1:9000:9000"
|
||||
build:
|
||||
context: ./infra
|
||||
dockerfile: minio.Dockerfile
|
||||
environment:
|
||||
MINIO_ROOT_USER_FILE: /run/secrets/s3_access_key_id
|
||||
MINIO_ROOT_PASSWORD_FILE: /run/secrets/s3_secret_access_key
|
||||
MINIO_BROWSER: "off"
|
||||
MINIO_API_CORS_ALLOW_ORIGIN: ${S3_CORS_ORIGINS:-http://127.0.0.1:5173,http://localhost:5173}
|
||||
volumes:
|
||||
- s3_data:/data
|
||||
- ./secrets/s3_access_key_id:/run/secrets/s3_access_key_id:ro
|
||||
- ./secrets/s3_secret_access_key:/run/secrets/s3_secret_access_key:ro
|
||||
healthcheck:
|
||||
test: ["CMD", "curl", "-f", "http://127.0.0.1:9000/minio/health/ready"]
|
||||
interval: 3s
|
||||
timeout: 2s
|
||||
retries: 30
|
||||
s3-bootstrap:
|
||||
cpus: 0.5
|
||||
mem_limit: 128m
|
||||
networks: [data]
|
||||
build: ./src
|
||||
environment: &s3-environment
|
||||
S3_ROOT_ACCESS_KEY_ID_FILE: /run/secrets/s3_root_access_key_id
|
||||
S3_ROOT_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_root_secret_access_key
|
||||
S3_ENDPOINT: ${S3_ENDPOINT:-http://s3:9000}
|
||||
S3_PUBLIC_ENDPOINT: ${S3_PUBLIC_ENDPOINT:-http://127.0.0.1:9000}
|
||||
S3_REGION: ${S3_REGION:-us-east-1}
|
||||
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
|
||||
S3_ACCESS_KEY_ID_FILE: ${S3_ACCESS_KEY_ID_FILE:-/run/secrets/s3_access_key_id}
|
||||
S3_SECRET_ACCESS_KEY_FILE: ${S3_SECRET_ACCESS_KEY_FILE:-/run/secrets/s3_secret_access_key}
|
||||
volumes:
|
||||
- ./secrets/s3_access_key_id:/run/secrets/s3_root_access_key_id:ro
|
||||
- ./secrets/s3_secret_access_key:/run/secrets/s3_root_secret_access_key:ro
|
||||
- ./secrets/s3_api_access_key_id:${S3_ACCESS_KEY_ID_FILE:-/run/secrets/s3_access_key_id}:ro
|
||||
- ./secrets/s3_api_secret_access_key:${S3_SECRET_ACCESS_KEY_FILE:-/run/secrets/s3_secret_access_key}:ro
|
||||
command: ["./messenger", "s3-bootstrap"]
|
||||
depends_on:
|
||||
s3:
|
||||
condition: service_healthy
|
||||
api:
|
||||
cpus: 0.5
|
||||
mem_limit: 512m
|
||||
networks: [proxy, data, telemetry]
|
||||
build: ./src
|
||||
deploy:
|
||||
replicas: ${API_REPLICAS:-2}
|
||||
stop_grace_period: 15s
|
||||
environment: &api-environment
|
||||
<<: *s3-environment
|
||||
OTEL_EXPORTER_OTLP_ENDPOINT: ${OTEL_EXPORTER_OTLP_ENDPOINT-http://otel-collector:4318}
|
||||
S3_ROOT_ACCESS_KEY_ID_FILE: ""
|
||||
S3_ROOT_SECRET_ACCESS_KEY_FILE: ""
|
||||
BROKER_URL_FILE: /run/secrets/broker_url
|
||||
JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5}
|
||||
JOB_TIMEOUT_SECONDS: ${JOB_TIMEOUT_SECONDS:-60}
|
||||
WORKER_MAX_PIXELS: ${WORKER_MAX_PIXELS:-8000000}
|
||||
UPLOAD_TMP_DIR: /tmp/uploads
|
||||
UPLOAD_CLEANUP_GRACE_SECONDS: ${UPLOAD_CLEANUP_GRACE_SECONDS:-600}
|
||||
# Go 1.22 sees host CPUs; keep parallelism suitable for this 0.5 CPU limit.
|
||||
GOMAXPROCS: ${API_GOMAXPROCS:-1}
|
||||
GOMEMLIMIT: 384MiB
|
||||
APP_HOST: ${APP_HOST:-0.0.0.0}
|
||||
APP_PORT: ${APP_PORT:-8080}
|
||||
APP_ENV: ${APP_ENV:-lab}
|
||||
IDEMPOTENCY_TTL_SECONDS: ${IDEMPOTENCY_TTL_SECONDS:-86400}
|
||||
DATABASE_URL_FILE: ${DATABASE_URL_FILE:-/run/secrets/database_url}
|
||||
CACHE_URL_FILE: ${CACHE_URL_FILE:-/run/secrets/cache_url}
|
||||
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600}
|
||||
REDIS_ADDR: redis:6379
|
||||
DATABASE_URL: postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@db:5432/${POSTGRES_DB}?sslmode=disable
|
||||
tmpfs:
|
||||
- /tmp/uploads:rw,noexec,nosuid,size=16m,mode=1777
|
||||
volumes: &api-volumes
|
||||
- ./secrets/s3_api_access_key_id:${S3_ACCESS_KEY_ID_FILE:-/run/secrets/s3_access_key_id}:ro
|
||||
- ./secrets/s3_api_secret_access_key:${S3_SECRET_ACCESS_KEY_FILE:-/run/secrets/s3_secret_access_key}:ro
|
||||
- ./secrets/broker_url:/run/secrets/broker_url:ro
|
||||
- ./secrets/database_url:${DATABASE_URL_FILE:-/run/secrets/database_url}:ro
|
||||
- ./secrets/cache_url:${CACHE_URL_FILE:-/run/secrets/cache_url}:ro
|
||||
depends_on:
|
||||
s3-bootstrap:
|
||||
condition: service_completed_successfully
|
||||
migrate:
|
||||
condition: service_completed_successfully
|
||||
db:
|
||||
condition: service_healthy
|
||||
redis:
|
||||
condition: service_healthy
|
||||
healthcheck:
|
||||
test: ["CMD", "./messenger", "healthcheck"]
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 20
|
||||
broker:
|
||||
image: rabbitmq:4.2.0-management
|
||||
hostname: broker
|
||||
environment:
|
||||
RABBITMQ_SERVER_ADDITIONAL_ERL_ARGS: "+S 2:2"
|
||||
RABBITMQ_CTL_ERL_ARGS: "+S 1:1"
|
||||
cpus: 0.5
|
||||
mem_limit: 512m
|
||||
networks: [data]
|
||||
command:
|
||||
- sh
|
||||
- -ec
|
||||
- |
|
||||
export RABBITMQ_DEFAULT_USER="$$(cat /run/secrets/broker_user)"
|
||||
export RABBITMQ_DEFAULT_PASS="$$(cat /run/secrets/broker_password)"
|
||||
exec docker-entrypoint.sh rabbitmq-server
|
||||
volumes:
|
||||
- rabbitmq_data:/var/lib/rabbitmq
|
||||
- ./secrets/broker_user:/run/secrets/broker_user:ro
|
||||
- ./secrets/broker_password:/run/secrets/broker_password:ro
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "rabbitmq-diagnostics -q check_running && rabbitmq-diagnostics -q check_port_connectivity"]
|
||||
interval: 5s
|
||||
timeout: 10s
|
||||
start_period: 20s
|
||||
retries: 30
|
||||
worker:
|
||||
tmpfs:
|
||||
- /tmp:rw,noexec,nosuid,size=64m,mode=1777
|
||||
build: ./src
|
||||
restart: unless-stopped
|
||||
cpus: 1.0
|
||||
mem_limit: 512m
|
||||
networks: [data, telemetry]
|
||||
environment: *api-environment
|
||||
volumes: *api-volumes
|
||||
command: ["./messenger", "worker"]
|
||||
healthcheck:
|
||||
disable: true
|
||||
depends_on:
|
||||
migrate:
|
||||
condition: service_completed_successfully
|
||||
s3-bootstrap:
|
||||
condition: service_completed_successfully
|
||||
broker:
|
||||
condition: service_healthy
|
||||
dispatcher:
|
||||
build: ./src
|
||||
restart: unless-stopped
|
||||
cpus: 0.25
|
||||
mem_limit: 128m
|
||||
networks: [data, telemetry]
|
||||
environment: *api-environment
|
||||
volumes: *api-volumes
|
||||
command: ["./messenger", "dispatcher"]
|
||||
healthcheck:
|
||||
disable: true
|
||||
depends_on:
|
||||
migrate:
|
||||
condition: service_completed_successfully
|
||||
broker:
|
||||
condition: service_healthy
|
||||
upload-cleanup:
|
||||
restart: unless-stopped
|
||||
cpus: 0.25
|
||||
mem_limit: 128m
|
||||
networks: [data]
|
||||
build: ./src
|
||||
command: ["./messenger", "attachments-cleanup", "--loop"]
|
||||
environment:
|
||||
<<: *s3-environment
|
||||
S3_ROOT_ACCESS_KEY_ID_FILE: ""
|
||||
S3_ROOT_SECRET_ACCESS_KEY_FILE: ""
|
||||
DATABASE_URL_FILE: ${DATABASE_URL_FILE:-/run/secrets/database_url}
|
||||
volumes:
|
||||
- ./secrets/s3_api_access_key_id:${S3_ACCESS_KEY_ID_FILE:-/run/secrets/s3_access_key_id}:ro
|
||||
- ./secrets/s3_api_secret_access_key:${S3_SECRET_ACCESS_KEY_FILE:-/run/secrets/s3_secret_access_key}:ro
|
||||
- ./secrets/database_url:${DATABASE_URL_FILE:-/run/secrets/database_url}:ro
|
||||
healthcheck:
|
||||
disable: true
|
||||
depends_on:
|
||||
migrate:
|
||||
condition: service_completed_successfully
|
||||
s3-bootstrap:
|
||||
condition: service_completed_successfully
|
||||
haproxy:
|
||||
environment:
|
||||
APP_PORT: ${APP_PORT:-8080}
|
||||
API_REPLICAS: ${API_REPLICAS:-2}
|
||||
cpus: 0.5
|
||||
mem_limit: 128m
|
||||
networks: [edge, proxy]
|
||||
image: haproxy:3.2.25-alpine
|
||||
ports:
|
||||
- "127.0.0.1:${API_HTTP_PORT:-8080}:8080"
|
||||
- "127.0.0.1:8404:8404"
|
||||
volumes:
|
||||
- ./infra/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro
|
||||
depends_on:
|
||||
api:
|
||||
condition: service_healthy
|
||||
healthcheck:
|
||||
test:
|
||||
[
|
||||
"CMD",
|
||||
"wget",
|
||||
"-q",
|
||||
"-O",
|
||||
"/dev/null",
|
||||
"http://127.0.0.1:8080/health/ready",
|
||||
]
|
||||
interval: 2s
|
||||
timeout: 3s
|
||||
retries: 20
|
||||
backend-tests:
|
||||
cpus: 2.0
|
||||
mem_limit: 1g
|
||||
networks: [data]
|
||||
profiles: ["test"]
|
||||
build:
|
||||
context: ./src
|
||||
target: builder
|
||||
environment:
|
||||
<<: *s3-environment
|
||||
S3_PUBLIC_ENDPOINT: http://s3:9000
|
||||
TEST_DATABASE_URL: postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@db:5432/${POSTGRES_DB}?sslmode=disable
|
||||
TEST_REDIS_ADDR: redis:6379
|
||||
volumes:
|
||||
- ./secrets/s3_api_access_key_id:${S3_ACCESS_KEY_ID_FILE:-/run/secrets/s3_access_key_id}:ro
|
||||
- ./secrets/s3_api_secret_access_key:${S3_SECRET_ACCESS_KEY_FILE:-/run/secrets/s3_secret_access_key}:ro
|
||||
- ./secrets/s3_access_key_id:/run/secrets/s3_root_access_key_id:ro
|
||||
- ./secrets/s3_secret_access_key:/run/secrets/s3_root_secret_access_key:ro
|
||||
depends_on:
|
||||
s3-bootstrap:
|
||||
condition: service_completed_successfully
|
||||
db:
|
||||
condition: service_healthy
|
||||
redis:
|
||||
condition: service_healthy
|
||||
command: ["go", "test", "-race", "-count=1", "-v", "./backend/..."]
|
||||
frontend:
|
||||
cpus: 0.5
|
||||
mem_limit: 512m
|
||||
networks: [edge]
|
||||
build:
|
||||
context: ./src/frontend
|
||||
environment:
|
||||
API_PROXY_TARGET: http://haproxy:8080
|
||||
ports:
|
||||
- "127.0.0.1:5173:5173"
|
||||
volumes:
|
||||
- ./src/frontend:/app
|
||||
- /app/node_modules
|
||||
depends_on:
|
||||
haproxy:
|
||||
condition: service_healthy
|
||||
healthcheck:
|
||||
test:
|
||||
[
|
||||
"CMD",
|
||||
"node",
|
||||
"-e",
|
||||
"fetch('http://127.0.0.1:5173/').then(r => process.exit(r.ok ? 0 : 1)).catch(() => process.exit(1))",
|
||||
]
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 20
|
||||
|
||||
otel-collector:
|
||||
image: otel/opentelemetry-collector-contrib:0.120.0
|
||||
networks: [telemetry]
|
||||
mem_limit: 192m
|
||||
command: ["--config=/etc/otelcol-contrib/config.yaml"]
|
||||
volumes:
|
||||
- ./infra/otel/collector.yaml:/etc/otelcol-contrib/config.yaml:ro
|
||||
|
||||
prometheus:
|
||||
image: prom/prometheus:v3.2.1
|
||||
networks: [telemetry]
|
||||
command: ["--config.file=/etc/prometheus/prometheus.yml", "--storage.tsdb.path=/prometheus", "--storage.tsdb.retention.time=7d"]
|
||||
volumes:
|
||||
- ./infra/otel/prometheus.yaml:/etc/prometheus/prometheus.yml:ro
|
||||
- ./infra/otel/alerts.yaml:/etc/prometheus/alerts.yaml:ro
|
||||
- prometheus_data:/prometheus
|
||||
|
||||
loki:
|
||||
image: grafana/loki:3.4.2
|
||||
networks: [telemetry]
|
||||
command: ["-config.file=/etc/loki/lab.yaml"]
|
||||
volumes:
|
||||
- ./infra/otel/loki.yaml:/etc/loki/lab.yaml:ro
|
||||
- loki_data:/loki
|
||||
|
||||
tempo:
|
||||
image: grafana/tempo:2.7.1
|
||||
networks: [telemetry]
|
||||
command: ["-config.file=/etc/tempo/lab.yaml"]
|
||||
volumes:
|
||||
- ./infra/otel/tempo.yaml:/etc/tempo/lab.yaml:ro
|
||||
- tempo_data:/var/tempo
|
||||
|
||||
grafana:
|
||||
image: grafana/grafana:11.5.2
|
||||
networks: [telemetry, edge]
|
||||
environment:
|
||||
GF_AUTH_ANONYMOUS_ENABLED: "true"
|
||||
GF_AUTH_ANONYMOUS_ORG_ROLE: Viewer
|
||||
ports:
|
||||
- "127.0.0.1:${GRAFANA_HTTP_PORT:-3000}:3000"
|
||||
volumes:
|
||||
- ./infra/otel/grafana/provisioning:/etc/grafana/provisioning:ro
|
||||
- ./infra/otel/grafana/dashboards:/var/lib/grafana/dashboards:ro
|
||||
- grafana_data:/var/lib/grafana
|
||||
|
||||
volumes:
|
||||
prometheus_data:
|
||||
loki_data:
|
||||
tempo_data:
|
||||
grafana_data:
|
||||
rabbitmq_data:
|
||||
postgres_data:
|
||||
redis_data:
|
||||
s3_data:
|
||||
|
||||
networks:
|
||||
edge:
|
||||
proxy:
|
||||
internal: true
|
||||
data:
|
||||
internal: true
|
||||
telemetry:
|
||||
internal: true
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 306 KiB |
@@ -0,0 +1,13 @@
|
||||
# Доказательства критериев
|
||||
|
||||
[Сводка прогона 20261113T155722.910201Z](runs/20261113T155722.910201Z/summary.json)
|
||||
|
||||
- [grade3-stand](grade3-stand.md) — PASS
|
||||
- [grade3-smoke](grade3-smoke.md) — PASS
|
||||
- [grade3-signals](grade3-signals.md) — PASS
|
||||
- [grade4-pipeline](grade4-pipeline.md) — PASS
|
||||
- [grade4-dashboard](grade4-dashboard.md) — PASS
|
||||
- [grade4-queue](grade4-queue.md) — PASS
|
||||
- [grade4-database](grade4-database.md) — PASS
|
||||
- [grade4-telemetry](grade4-telemetry.md) — PASS
|
||||
- [grade4-load](grade4-load.md) — PASS
|
||||
@@ -0,0 +1,22 @@
|
||||
# Метрики, структурные логи и трассы API и worker
|
||||
|
||||
Предпосылки — локальный синтетический стенд после make app-up и доступ к Docker.
|
||||
|
||||
Контрольный POST сопоставляется с логом по X-Request-Id. Оба сервиса сопоставляются с Tempo по trace_id и span_id. Prometheus возвращает счётчики и histogram buckets. Проверка ищет утечки синтетических значений в полученных сигналах.
|
||||
|
||||
Команда `/usr/bin/python3 scripts/check_observability.py --base-url http://127.0.0.1:8080 --grafana-url http://127.0.0.1:3000 signals`. Код выхода **0**. Время 7.2 с.
|
||||
|
||||
[Вывод](runs/20261113T155722.910201Z/grade3-signals-1.txt)
|
||||
|
||||
````text
|
||||
Worker: изображение 3000×3000 завершилось в состоянии failed.
|
||||
messenger-api: http.request.completed, trace=91735be6f8944614b886c2a34bfc95a8, span=7aca4e9abb3620f6
|
||||
messenger-worker: job.processing.completed, trace=8f05225357fb027e222f890074249eed, span=49632776d49e21fe
|
||||
messenger-worker: job.processing.completed, trace=c958e0f21140e39dddb4192b5982442e, span=f459a5c48084c984
|
||||
Prometheus: messenger_http_requests_total, рядов=40
|
||||
Prometheus: messenger_http_duration_seconds_bucket, рядов=560
|
||||
Prometheus: messenger_worker_jobs_total{job_outcome="success"}, рядов=1
|
||||
Prometheus: messenger_worker_jobs_total{job_outcome="error"}, рядов=1
|
||||
Prometheus: messenger_worker_duration_seconds_bucket, рядов=42
|
||||
Проверенные сигналы не содержат контрольных credentials, текста, имени/байтов файла и бизнес-ID.
|
||||
````
|
||||
@@ -0,0 +1,27 @@
|
||||
# Совместимость API при включённой телеметрии
|
||||
|
||||
Предпосылки — локальный синтетический стенд после make app-up и доступ к Docker.
|
||||
|
||||
Smoke проверяет сессии, сообщения, ACL, идемпотентность и готовность вложения с оригиналом и thumbnail.
|
||||
|
||||
Команда `make check`. Код выхода **0**. Время 0.05 с.
|
||||
|
||||
[Вывод](runs/20261113T155722.910201Z/grade3-smoke-1.txt)
|
||||
|
||||
````text
|
||||
make[1]: Entering directory '/home/TAskMAster339/Projects/highload-systems/mtusi2026lab6-fork'
|
||||
python3 scripts/check_contract.py
|
||||
PASS: offline skeleton structure, local refs and Python syntax; application NOT tested.
|
||||
make[1]: Leaving directory '/home/TAskMAster339/Projects/highload-systems/mtusi2026lab6-fork'
|
||||
````
|
||||
|
||||
Команда `make test BASE_URL=http://127.0.0.1:8080`. Код выхода **0**. Время 3.51 с.
|
||||
|
||||
[Вывод](runs/20261113T155722.910201Z/grade3-smoke-2.txt)
|
||||
|
||||
````text
|
||||
make[1]: Entering directory '/home/TAskMAster339/Projects/highload-systems/mtusi2026lab6-fork'
|
||||
python3 tests/smoke.py --base-url "http://127.0.0.1:8080"
|
||||
PASS: lab6 public API scenario; infrastructure and grading evidence remain separate.
|
||||
make[1]: Leaving directory '/home/TAskMAster339/Projects/highload-systems/mtusi2026lab6-fork'
|
||||
````
|
||||
@@ -0,0 +1,74 @@
|
||||
# Стенд, тома и provisioning
|
||||
|
||||
Предпосылки — локальный синтетический стенд после make app-up и доступ к Docker.
|
||||
|
||||
Prometheus хранит временные ряды, Loki — журналы, Tempo — трассы. Collector принимает и маршрутизирует OTLP. Grafana читает данные и загружает dashboard из JSON.
|
||||
|
||||
Команда `docker compose config --quiet`. Код выхода **0**. Время 0.08 с.
|
||||
|
||||
[Вывод](runs/20261113T155722.910201Z/grade3-stand-1.txt)
|
||||
|
||||
````text
|
||||
(пусто)
|
||||
````
|
||||
|
||||
Команда `docker compose ps --all`. Код выхода **0**. Время 0.11 с.
|
||||
|
||||
[Вывод](runs/20261113T155722.910201Z/grade3-stand-2.txt)
|
||||
|
||||
````text
|
||||
NAME IMAGE COMMAND SERVICE CREATED STATUS PORTS
|
||||
highload-lab6-api-1 highload-lab6-api "./messenger" api 31 minutes ago Up 31 minutes (healthy) 8080/tcp
|
||||
highload-lab6-api-2 highload-lab6-api "./messenger" api 31 minutes ago Up 31 minutes (healthy) 8080/tcp
|
||||
highload-lab6-broker-1 rabbitmq:4.2.0-management "docker-entrypoint.s…" broker About an hour ago Up About an hour (healthy) 4369/tcp, 5671-5672/tcp, 15671-15672/tcp, 15691-15692/tcp, 25672/tcp
|
||||
highload-lab6-db-1 postgres:16.10-bookworm "docker-entrypoint.s…" db About an hour ago Up About an hour (healthy) 5432/tcp
|
||||
highload-lab6-dispatcher-1 highload-lab6-dispatcher "./messenger dispatc…" dispatcher 31 minutes ago Up 31 minutes 8080/tcp
|
||||
highload-lab6-frontend-1 sha256:6f2bacde48d02aca251e04c4707e11d6707c797af56bf6623a671b2e0c20da30 "docker-entrypoint.s…" frontend About an hour ago Up About an hour (healthy) 127.0.0.1:5173->5173/tcp
|
||||
highload-lab6-grafana-1 grafana/grafana:11.5.2 "/run.sh" grafana About an hour ago Up About an hour 127.0.0.1:3000->3000/tcp
|
||||
highload-lab6-haproxy-1 haproxy:3.2.25-alpine "docker-entrypoint.s…" haproxy About an hour ago Up About an hour (healthy) 127.0.0.1:8080->8080/tcp, 127.0.0.1:8404->8404/tcp
|
||||
highload-lab6-loki-1 grafana/loki:3.4.2 "/usr/bin/loki -conf…" loki About an hour ago Up About an hour 3100/tcp
|
||||
highload-lab6-migrate-1 highload-lab6-migrate "./messenger migrate…" migrate 31 minutes ago Exited (0) 3 minutes ago
|
||||
highload-lab6-otel-collector-1 otel/opentelemetry-collector-contrib:0.120.0 "/otelcol-contrib --…" otel-collector About an hour ago Up About an hour 4317-4318/tcp, 55678-55679/tcp
|
||||
highload-lab6-prometheus-1 prom/prometheus:v3.2.1 "/bin/prometheus --c…" prometheus 31 minutes ago Up 31 minutes 9090/tcp
|
||||
highload-lab6-redis-1 redis:7.4.11-bookworm "docker-entrypoint.s…" redis About an hour ago Up About an hour (healthy) 6379/tcp
|
||||
highload-lab6-s3-1 sha256:b891ef2bb2e30a5a240ee0d267c8938ab43f2ce5e5c89347e580e7c1b1cf4e5e "minio server /data …" s3 About an hour ago Up 3 minutes (healthy) 127.0.0.1:9000->9000/tcp
|
||||
highload-lab6-s3-bootstrap-1 highload-lab6-s3-bootstrap "./messenger s3-boot…" s3-bootstrap 31 minutes ago Exited (0) 3 minutes ago
|
||||
highload-lab6-tempo-1 grafana/tempo:2.7.1 "/tempo -config.file…" tempo About an hour ago Up 34 seconds
|
||||
highload-lab6-upload-cleanup-1 highload-lab6-upload-cleanup "./messenger attachm…" upload-cleanup 31 minutes ago Up 31 minutes 8080/tcp
|
||||
highload-lab6-worker-1 highload-lab6-worker "./messenger worker" worker 31 minutes ago Up 3 minutes 8080/tcp
|
||||
````
|
||||
|
||||
Команда `docker compose config --volumes`. Код выхода **0**. Время 0.08 с.
|
||||
|
||||
[Вывод](runs/20261113T155722.910201Z/grade3-stand-3.txt)
|
||||
|
||||
````text
|
||||
s3_data
|
||||
loki_data
|
||||
postgres_data
|
||||
prometheus_data
|
||||
redis_data
|
||||
tempo_data
|
||||
grafana_data
|
||||
rabbitmq_data
|
||||
````
|
||||
|
||||
Команда `docker compose run --rm --no-deps otel-collector validate --config=/etc/otelcol-contrib/config.yaml`. Код выхода **0**. Время 1.28 с.
|
||||
|
||||
[Вывод](runs/20261113T155722.910201Z/grade3-stand-4.txt)
|
||||
|
||||
````text
|
||||
Container highload-lab6-otel-collector-run-0fb806dd4ede Creating
|
||||
Container highload-lab6-otel-collector-run-0fb806dd4ede Created
|
||||
````
|
||||
|
||||
Команда `/usr/bin/python3 scripts/check_observability.py --base-url http://127.0.0.1:8080 --grafana-url http://127.0.0.1:3000 provisioning`. Код выхода **0**. Время 0.13 с.
|
||||
|
||||
[Вывод](runs/20261113T155722.910201Z/grade3-stand-5.txt)
|
||||
|
||||
````text
|
||||
Источник Grafana: prometheus
|
||||
Источник Grafana: loki
|
||||
Источник Grafana: tempo
|
||||
Загружен dashboard messenger-lab6, панелей: 17
|
||||
````
|
||||
@@ -0,0 +1,38 @@
|
||||
# Метрики диагностики и SLO
|
||||
|
||||
Предпосылки — локальный синтетический стенд после make app-up и доступ к Docker.
|
||||
|
||||
Проверяются запросы панелей, SLI за 30 минут и синтаксис двух правил.
|
||||
|
||||
Команда `/usr/bin/python3 scripts/check_observability.py --base-url http://127.0.0.1:8080 --grafana-url http://127.0.0.1:3000 dashboard`. Код выхода **0**. Время 0.16 с.
|
||||
|
||||
[Вывод](runs/20261113T155722.910201Z/grade4-dashboard-1.txt)
|
||||
|
||||
````text
|
||||
API · запросы в секунду — sum(rate(messenger_http_requests_total[1m])) — рядов: 1
|
||||
API · 5xx в секунду — sum(rate(messenger_http_requests_total{http_status_class="5xx"}[1m])) or 0 * sum(rate(messenger_http_requests_total[1m])) — рядов: 1
|
||||
API · p95 длительности — histogram_quantile(0.95, sum by (le) (rate(messenger_http_duration_seconds_bucket[1m]))) — рядов: 1
|
||||
Worker · задания в минуту — 60 * sum by (job_outcome) (rate(messenger_worker_jobs_total[1m])) — рядов: 3
|
||||
Worker · p95 длительности — histogram_quantile(0.95, sum by (le,job_outcome) (rate(messenger_worker_duration_seconds_bucket[1m]))) — рядов: 3
|
||||
API · p50 длительности — histogram_quantile(0.5, sum by (le) (rate(messenger_http_duration_seconds_bucket[1m]))) — рядов: 1
|
||||
API · p99 длительности — histogram_quantile(0.99, sum by (le) (rate(messenger_http_duration_seconds_bucket[1m]))) — рядов: 1
|
||||
API · доля 5xx — (sum(rate(messenger_http_requests_total{http_status_class="5xx"}[1m])) or 0 * sum(rate(messenger_http_requests_total[1m]))) / sum(rate(messenger_http_requests_total[1m])) — рядов: 1
|
||||
Очередь attachments.v1 · незавершённые задания — max(messenger_queue_depth) — рядов: 1
|
||||
Очередь attachments.v1 · возраст старейшего задания — max(messenger_queue_oldest_seconds) — рядов: 1
|
||||
PostgreSQL · занятые соединения — sum by (service_name) (messenger_db_pool_used) — рядов: 3
|
||||
PostgreSQL · запросы к пулу, ожидавшие соединение, в секунду — sum by (service_name) (rate(messenger_db_pool_waited_total[1m])) — рядов: 3
|
||||
Worker · загрузка — sum(messenger_worker_busy) / sum(messenger_worker_capacity) — рядов: 1
|
||||
Вложения · p95 до ready — histogram_quantile(0.95, sum by (le) (rate(messenger_attachment_ready_seconds_bucket[1m]))) — рядов: 1
|
||||
Процессы · загрузка CPU — sum by (service_name) (rate(messenger_process_cpu_seconds_total[1m])) — рядов: 3
|
||||
Процессы · резидентная память — sum by (service_name) (messenger_process_rss_bytes) — рядов: 3
|
||||
SLI за 30 минут: 0.9992789095149922 цель: >= 0.99
|
||||
````
|
||||
|
||||
Команда `docker compose exec -T prometheus promtool check rules /etc/prometheus/alerts.yaml`. Код выхода **0**. Время 0.23 с.
|
||||
|
||||
[Вывод](runs/20261113T155722.910201Z/grade4-dashboard-2.txt)
|
||||
|
||||
````text
|
||||
Checking /etc/prometheus/alerts.yaml
|
||||
SUCCESS: 2 rules found
|
||||
````
|
||||
@@ -0,0 +1,25 @@
|
||||
# Задержка PostgreSQL и восстановление
|
||||
|
||||
Предпосылки — локальный синтетический стенд после make app-up и доступ к Docker.
|
||||
|
||||
При блокировке таблицы растёт доля 5xx и алерт срабатывает. Задержку показывает дочерний спан PostgreSQL. После снятия блокировки алерт переходит в resolved.
|
||||
|
||||
Команда `/usr/bin/python3 scripts/check_observability_incidents.py --base-url http://127.0.0.1:8080 --grafana-url http://127.0.0.1:3000 database`. Код выхода **0**. Время 104.65 с.
|
||||
|
||||
[Вывод](runs/20261113T155722.910201Z/grade4-database-1.txt)
|
||||
|
||||
````text
|
||||
MessengerHighErrorRate firing
|
||||
pg_terminate_backend
|
||||
----------------------
|
||||
t
|
||||
(1 row)
|
||||
|
||||
FATAL: terminating connection due to administrator command
|
||||
server closed the connection unexpectedly
|
||||
This probably means the server terminated abnormally
|
||||
before or while processing the request.
|
||||
connection to server was lost
|
||||
Медленный запрос PostgreSQL подтверждён в trace: 624d2df384e6bb8e27ad11ca29a41d0b
|
||||
MessengerHighErrorRate resolved
|
||||
````
|
||||
@@ -0,0 +1,125 @@
|
||||
# Короткая нагрузка на API
|
||||
|
||||
Предпосылки — локальный синтетический стенд после make app-up и доступ к Docker.
|
||||
|
||||
Один closed-loop прогон POST сообщений с четырьмя клиентами, отдельно от инцидентов. Снимок ресурсов сделан сразу после прогона.
|
||||
|
||||
Команда `lscpu`. Код выхода **0**. Время 0.01 с.
|
||||
|
||||
[Вывод](runs/20261113T155722.910201Z/grade4-load-1.txt)
|
||||
|
||||
````text
|
||||
Architecture: x86_64
|
||||
CPU op-mode(s): 32-bit, 64-bit
|
||||
Address sizes: 48 bits physical, 48 bits virtual
|
||||
Byte Order: Little Endian
|
||||
CPU(s): 16
|
||||
On-line CPU(s) list: 0-15
|
||||
Vendor ID: AuthenticAMD
|
||||
Model name: AMD Ryzen 9 PRO 7940HS w/ Radeon 780M Graphics
|
||||
CPU family: 25
|
||||
Model: 116
|
||||
Thread(s) per core: 2
|
||||
Core(s) per socket: 8
|
||||
Socket(s): 1
|
||||
Stepping: 1
|
||||
Microcode version: 0xa70410a
|
||||
Frequency boost: enabled
|
||||
CPU(s) scaling MHz: 90%
|
||||
CPU max MHz: 5263,0610
|
||||
CPU min MHz: 402,7860
|
||||
BogoMIPS: 7985,14
|
||||
Flags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good amd_lbr_v2 nopl xtopology nonstop_tsc cpuid extd_apicid aperfmperf rapl pni pclmulqdq monitor ssse3 fma cx16 sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand lahf_lm cmp_legacy svm extapic cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw ibs skinit wdt tce topoext perfctr_core perfctr_nb bpext perfctr_llc mwaitx cpuid_fault cpb cat_l3 cdp_l3 hw_pstate ssbd mba perfmon_v2 ibrs ibpb stibp ibrs_enhanced vmmcall fsgsbase bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local user_shstk avx512_bf16 clzero irperf xsaveerptr rdpru wbnoinvd cppc arat npt lbrv svm_lock nrip_save tsc_scale vmcb_clean flushbyasid decodeassists pausefilter pfthreshold vgif x2avic v_spec_ctrl vnmi avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg avx512_vpopcntdq rdpid overflow_recov succor smca fsrm flush_l1d amd_lbr_pmc_freeze
|
||||
Virtualization: AMD-V
|
||||
L1d cache: 256 KiB (8 instances)
|
||||
L1i cache: 256 KiB (8 instances)
|
||||
L2 cache: 8 MiB (8 instances)
|
||||
L3 cache: 16 MiB (1 instance)
|
||||
NUMA node(s): 1
|
||||
NUMA node0 CPU(s): 0-15
|
||||
Vulnerability Gather data sampling: Not affected
|
||||
Vulnerability Ghostwrite: Not affected
|
||||
Vulnerability Indirect target selection: Not affected
|
||||
Vulnerability Itlb multihit: Not affected
|
||||
Vulnerability L1tf: Not affected
|
||||
Vulnerability Mds: Not affected
|
||||
Vulnerability Meltdown: Not affected
|
||||
Vulnerability Mmio stale data: Not affected
|
||||
Vulnerability Old microcode: Not affected
|
||||
Vulnerability Reg file data sampling: Not affected
|
||||
Vulnerability Retbleed: Not affected
|
||||
Vulnerability Spec rstack overflow: Mitigation; Safe RET
|
||||
Vulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl
|
||||
Vulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization
|
||||
Vulnerability Spectre v2: Mitigation; Enhanced / Automatic IBRS; IBPB conditional; STIBP always-on; PBRSB-eIBRS Not affected; BHI Not affected
|
||||
Vulnerability Srbds: Not affected
|
||||
Vulnerability Tsa: Mitigation; Clear CPU buffers
|
||||
Vulnerability Tsx async abort: Not affected
|
||||
Vulnerability Vmscape: Mitigation; IBPB before exit to userspace
|
||||
````
|
||||
|
||||
Команда `free -h`. Код выхода **0**. Время 0.0 с.
|
||||
|
||||
[Вывод](runs/20261113T155722.910201Z/grade4-load-2.txt)
|
||||
|
||||
````text
|
||||
total used free shared buff/cache available
|
||||
Mem: 14Gi 11Gi 1,3Gi 139Mi 2,3Gi 3,1Gi
|
||||
Swap: 8,0Gi 8,0Gi 548Ki
|
||||
````
|
||||
|
||||
Команда `docker compose version`. Код выхода **0**. Время 0.04 с.
|
||||
|
||||
[Вывод](runs/20261113T155722.910201Z/grade4-load-3.txt)
|
||||
|
||||
````text
|
||||
Docker Compose version v5.7.0
|
||||
````
|
||||
|
||||
Команда `/usr/bin/python3 tests/load.py --base-url http://127.0.0.1:8080 --duration 30 --concurrency 4`. Код выхода **0**. Время 31.48 с.
|
||||
|
||||
[Вывод](runs/20261113T155722.910201Z/grade4-load-4.txt)
|
||||
|
||||
````text
|
||||
{
|
||||
"lab": 6,
|
||||
"workload": "closed-loop POST messages; no automatic retries",
|
||||
"requested_duration_seconds": 30.0,
|
||||
"elapsed_seconds": 30.004,
|
||||
"concurrency": 4,
|
||||
"successful_requests": 37708,
|
||||
"failed_requests": 0,
|
||||
"error_ratio": 0.0,
|
||||
"successful_rps": 1256.75,
|
||||
"successful_latency_ms": {
|
||||
"p50": 2.828,
|
||||
"p95": 4.327,
|
||||
"p99": 7.211
|
||||
},
|
||||
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
|
||||
}
|
||||
````
|
||||
|
||||
Команда `docker compose stats --no-stream`. Код выхода **0**. Время 2.17 с.
|
||||
|
||||
[Вывод](runs/20261113T155722.910201Z/grade4-load-5.txt)
|
||||
|
||||
````text
|
||||
CONTAINER ID NAME CPU % MEM USAGE / LIMIT MEM % NET I/O BLOCK I/O PIDS
|
||||
2ee09eab650d highload-lab6-worker-1 0.01% 10.86MiB / 512MiB 2.12% 274kB / 393kB 4.58MB / 0B 7
|
||||
7fdc55311297 highload-lab6-dispatcher-1 0.03% 10.59MiB / 128MiB 8.27% 1.05MB / 1.41MB 1.33MB / 422kB 7
|
||||
188cf5f1391d highload-lab6-api-2 2.56% 22.67MiB / 512MiB 4.43% 85.5MB / 161MB 8.96MB / 406kB 7
|
||||
93535c5e7b67 highload-lab6-api-1 0.06% 21.67MiB / 512MiB 4.23% 86.4MB / 163MB 8.18MB / 389kB 7
|
||||
ae279d3a717c highload-lab6-upload-cleanup-1 0.00% 4.543MiB / 128MiB 3.55% 35.6kB / 31.2kB 32.8kB / 532kB 12
|
||||
db0e2da28fb1 highload-lab6-prometheus-1 0.23% 35.93MiB / 14.89GiB 0.24% 3.86MB / 1.91MB 426MB / 6.98MB 22
|
||||
66b201cac5eb highload-lab6-grafana-1 0.68% 89.27MiB / 14.89GiB 0.59% 59.5MB / 116MB 1.32GB / 242MB 25
|
||||
0d0412e6fe25 highload-lab6-tempo-1 0.02% 307.1MiB / 14.89GiB 2.01% 9.75MB / 202kB 58.6MB / 55.3MB 21
|
||||
a8fc40b42323 highload-lab6-loki-1 0.92% 161.8MiB / 14.89GiB 1.06% 5.45MB / 32.4MB 534MB / 24.9MB 21
|
||||
a2048ea9933c highload-lab6-otel-collector-1 0.19% 60.13MiB / 192MiB 31.32% 123MB / 28.3MB 489MB / 20.9MB 21
|
||||
efeed09a36ec highload-lab6-frontend-1 0.27% 24.01MiB / 512MiB 4.69% 52.3kB / 7.76MB 562MB / 161MB 36
|
||||
2ac44f5d15bb highload-lab6-haproxy-1 2.36% 10.5MiB / 128MiB 8.20% 104MB / 106MB 30.2MB / 14MB 17
|
||||
3999d26f1e22 highload-lab6-s3-1 9.80% 354.4MiB / 512MiB 69.22% 1.07MB / 760kB 125MB / 10.2MB 21
|
||||
8837cc38b561 highload-lab6-db-1 0.66% 80.51MiB / 512MiB 15.72% 160MB / 109MB 459MB / 5.37GB 18
|
||||
3e7d2e507d3d highload-lab6-broker-1 23.55% 89.5MiB / 512MiB 17.48% 374kB / 333kB 267MB / 91.5MB 47
|
||||
65ad13aee585 highload-lab6-redis-1 2.71% 4.18MiB / 128MiB 3.27% 13.8MB / 8.47MB 163MB / 7.52MB 6
|
||||
````
|
||||
@@ -0,0 +1,45 @@
|
||||
# Сквозная трасса и повторная попытка
|
||||
|
||||
Предпосылки — локальный синтетический стенд после make app-up и доступ к Docker.
|
||||
|
||||
Tempo подтверждает родительскую связь HTTP, producer и consumer, операции S3 и PostgreSQL и две попытки одной загрузки.
|
||||
|
||||
Команда `/usr/bin/python3 scripts/check_observability.py --base-url http://127.0.0.1:8080 --grafana-url http://127.0.0.1:3000 pipeline`. Код выхода **0**. Время 4.92 с.
|
||||
|
||||
[Вывод](runs/20261113T155722.910201Z/grade4-pipeline-1.txt)
|
||||
|
||||
````text
|
||||
Upload → producer → consumer → S3/PostgreSQL, trace: a1b887f518133aa40e303527d73f514e
|
||||
````
|
||||
|
||||
Команда `/usr/bin/python3 scripts/check_observability_incidents.py --base-url http://127.0.0.1:8080 --grafana-url http://127.0.0.1:3000 retry`. Код выхода **0**. Время 15.62 с.
|
||||
|
||||
[Вывод](runs/20261113T155722.910201Z/grade4-pipeline-2.txt)
|
||||
|
||||
````text
|
||||
Container highload-lab6-worker-1 Stopping
|
||||
Container highload-lab6-worker-1 Stopped
|
||||
Container highload-lab6-s3-1 Stopping
|
||||
Container highload-lab6-s3-1 Stopped
|
||||
Container highload-lab6-worker-1 Starting
|
||||
Container highload-lab6-worker-1 Started
|
||||
Container highload-lab6-s3-1 Starting
|
||||
Container highload-lab6-s3-1 Started
|
||||
Container highload-lab6-s3-1 Waiting
|
||||
Container highload-lab6-s3-1 Healthy
|
||||
Исходное вложение перешло в ready, обе попытки в trace: 762071e58a275355660808c10b605df8
|
||||
Container highload-lab6-broker-1 Waiting
|
||||
Container highload-lab6-db-1 Waiting
|
||||
Container highload-lab6-s3-1 Waiting
|
||||
Container highload-lab6-db-1 Healthy
|
||||
Container highload-lab6-migrate-1 Starting
|
||||
Container highload-lab6-broker-1 Healthy
|
||||
Container highload-lab6-s3-1 Healthy
|
||||
Container highload-lab6-s3-bootstrap-1 Starting
|
||||
Container highload-lab6-migrate-1 Started
|
||||
Container highload-lab6-migrate-1 Waiting
|
||||
Container highload-lab6-s3-bootstrap-1 Started
|
||||
Container highload-lab6-s3-bootstrap-1 Waiting
|
||||
Container highload-lab6-migrate-1 Exited
|
||||
Container highload-lab6-s3-bootstrap-1 Exited
|
||||
````
|
||||
@@ -0,0 +1,32 @@
|
||||
# Остановка и восстановление worker
|
||||
|
||||
Предпосылки — локальный синтетический стенд после make app-up и доступ к Docker.
|
||||
|
||||
Возраст незавершённой работы вызывает firing. Вложение достигает ready после восстановления worker, алерт переходит в resolved.
|
||||
|
||||
Команда `/usr/bin/python3 scripts/check_observability_incidents.py --base-url http://127.0.0.1:8080 --grafana-url http://127.0.0.1:3000 worker`. Код выхода **0**. Время 55.56 с.
|
||||
|
||||
[Вывод](runs/20261113T155722.910201Z/grade4-queue-1.txt)
|
||||
|
||||
````text
|
||||
Container highload-lab6-worker-1 Stopping
|
||||
Container highload-lab6-worker-1 Stopped
|
||||
MessengerQueueLag firing
|
||||
Container highload-lab6-broker-1 Waiting
|
||||
Container highload-lab6-db-1 Waiting
|
||||
Container highload-lab6-s3-1 Waiting
|
||||
Container highload-lab6-db-1 Healthy
|
||||
Container highload-lab6-migrate-1 Starting
|
||||
Container highload-lab6-s3-1 Healthy
|
||||
Container highload-lab6-s3-bootstrap-1 Starting
|
||||
Container highload-lab6-broker-1 Healthy
|
||||
Container highload-lab6-migrate-1 Started
|
||||
Container highload-lab6-migrate-1 Waiting
|
||||
Container highload-lab6-s3-bootstrap-1 Started
|
||||
Container highload-lab6-s3-bootstrap-1 Waiting
|
||||
Container highload-lab6-migrate-1 Exited
|
||||
Container highload-lab6-s3-bootstrap-1 Exited
|
||||
Container highload-lab6-worker-1 Starting
|
||||
Container highload-lab6-worker-1 Started
|
||||
MessengerQueueLag resolved
|
||||
````
|
||||
@@ -0,0 +1,20 @@
|
||||
# Недоступность Tempo
|
||||
|
||||
Предпосылки — локальный синтетический стенд после make app-up и доступ к Docker.
|
||||
|
||||
Проверяются сообщения и готовность вложения при остановленном Tempo, снимок памяти Collector и число отброшенных спанов по его журналу.
|
||||
|
||||
Команда `/usr/bin/python3 scripts/check_observability_incidents.py --base-url http://127.0.0.1:8080 --grafana-url http://127.0.0.1:3000 telemetry`. Код выхода **0**. Время 46.39 с.
|
||||
|
||||
[Вывод](runs/20261113T155722.910201Z/grade4-telemetry-1.txt)
|
||||
|
||||
````text
|
||||
Container highload-lab6-tempo-1 Stopping
|
||||
Container highload-lab6-tempo-1 Stopped
|
||||
CONTAINER ID NAME CPU % MEM USAGE / LIMIT MEM % NET I/O BLOCK I/O PIDS
|
||||
a2048ea9933c highload-lab6-otel-collector-1 0.42% 56.43MiB / 192MiB 29.39% 69.6MB / 15.8MB 466MB / 20.8MB 21
|
||||
API и worker работали при остановленном Tempo.
|
||||
Container highload-lab6-tempo-1 Starting
|
||||
Container highload-lab6-tempo-1 Started
|
||||
Спанов отброшено Collector за период проверки: 54
|
||||
````
|
||||
@@ -0,0 +1,10 @@
|
||||
Worker: изображение 3000×3000 завершилось в состоянии failed.
|
||||
messenger-api: http.request.completed, trace=91735be6f8944614b886c2a34bfc95a8, span=7aca4e9abb3620f6
|
||||
messenger-worker: job.processing.completed, trace=8f05225357fb027e222f890074249eed, span=49632776d49e21fe
|
||||
messenger-worker: job.processing.completed, trace=c958e0f21140e39dddb4192b5982442e, span=f459a5c48084c984
|
||||
Prometheus: messenger_http_requests_total, рядов=40
|
||||
Prometheus: messenger_http_duration_seconds_bucket, рядов=560
|
||||
Prometheus: messenger_worker_jobs_total{job_outcome="success"}, рядов=1
|
||||
Prometheus: messenger_worker_jobs_total{job_outcome="error"}, рядов=1
|
||||
Prometheus: messenger_worker_duration_seconds_bucket, рядов=42
|
||||
Проверенные сигналы не содержат контрольных credentials, текста, имени/байтов файла и бизнес-ID.
|
||||
@@ -0,0 +1,4 @@
|
||||
make[1]: Entering directory '/home/TAskMAster339/Projects/highload-systems/mtusi2026lab6-fork'
|
||||
python3 scripts/check_contract.py
|
||||
PASS: offline skeleton structure, local refs and Python syntax; application NOT tested.
|
||||
make[1]: Leaving directory '/home/TAskMAster339/Projects/highload-systems/mtusi2026lab6-fork'
|
||||
@@ -0,0 +1,4 @@
|
||||
make[1]: Entering directory '/home/TAskMAster339/Projects/highload-systems/mtusi2026lab6-fork'
|
||||
python3 tests/smoke.py --base-url "http://127.0.0.1:8080"
|
||||
PASS: lab6 public API scenario; infrastructure and grading evidence remain separate.
|
||||
make[1]: Leaving directory '/home/TAskMAster339/Projects/highload-systems/mtusi2026lab6-fork'
|
||||
@@ -0,0 +1,19 @@
|
||||
NAME IMAGE COMMAND SERVICE CREATED STATUS PORTS
|
||||
highload-lab6-api-1 highload-lab6-api "./messenger" api 31 minutes ago Up 31 minutes (healthy) 8080/tcp
|
||||
highload-lab6-api-2 highload-lab6-api "./messenger" api 31 minutes ago Up 31 minutes (healthy) 8080/tcp
|
||||
highload-lab6-broker-1 rabbitmq:4.2.0-management "docker-entrypoint.s…" broker About an hour ago Up About an hour (healthy) 4369/tcp, 5671-5672/tcp, 15671-15672/tcp, 15691-15692/tcp, 25672/tcp
|
||||
highload-lab6-db-1 postgres:16.10-bookworm "docker-entrypoint.s…" db About an hour ago Up About an hour (healthy) 5432/tcp
|
||||
highload-lab6-dispatcher-1 highload-lab6-dispatcher "./messenger dispatc…" dispatcher 31 minutes ago Up 31 minutes 8080/tcp
|
||||
highload-lab6-frontend-1 sha256:6f2bacde48d02aca251e04c4707e11d6707c797af56bf6623a671b2e0c20da30 "docker-entrypoint.s…" frontend About an hour ago Up About an hour (healthy) 127.0.0.1:5173->5173/tcp
|
||||
highload-lab6-grafana-1 grafana/grafana:11.5.2 "/run.sh" grafana About an hour ago Up About an hour 127.0.0.1:3000->3000/tcp
|
||||
highload-lab6-haproxy-1 haproxy:3.2.25-alpine "docker-entrypoint.s…" haproxy About an hour ago Up About an hour (healthy) 127.0.0.1:8080->8080/tcp, 127.0.0.1:8404->8404/tcp
|
||||
highload-lab6-loki-1 grafana/loki:3.4.2 "/usr/bin/loki -conf…" loki About an hour ago Up About an hour 3100/tcp
|
||||
highload-lab6-migrate-1 highload-lab6-migrate "./messenger migrate…" migrate 31 minutes ago Exited (0) 3 minutes ago
|
||||
highload-lab6-otel-collector-1 otel/opentelemetry-collector-contrib:0.120.0 "/otelcol-contrib --…" otel-collector About an hour ago Up About an hour 4317-4318/tcp, 55678-55679/tcp
|
||||
highload-lab6-prometheus-1 prom/prometheus:v3.2.1 "/bin/prometheus --c…" prometheus 31 minutes ago Up 31 minutes 9090/tcp
|
||||
highload-lab6-redis-1 redis:7.4.11-bookworm "docker-entrypoint.s…" redis About an hour ago Up About an hour (healthy) 6379/tcp
|
||||
highload-lab6-s3-1 sha256:b891ef2bb2e30a5a240ee0d267c8938ab43f2ce5e5c89347e580e7c1b1cf4e5e "minio server /data …" s3 About an hour ago Up 3 minutes (healthy) 127.0.0.1:9000->9000/tcp
|
||||
highload-lab6-s3-bootstrap-1 highload-lab6-s3-bootstrap "./messenger s3-boot…" s3-bootstrap 31 minutes ago Exited (0) 3 minutes ago
|
||||
highload-lab6-tempo-1 grafana/tempo:2.7.1 "/tempo -config.file…" tempo About an hour ago Up 34 seconds
|
||||
highload-lab6-upload-cleanup-1 highload-lab6-upload-cleanup "./messenger attachm…" upload-cleanup 31 minutes ago Up 31 minutes 8080/tcp
|
||||
highload-lab6-worker-1 highload-lab6-worker "./messenger worker" worker 31 minutes ago Up 3 minutes 8080/tcp
|
||||
@@ -0,0 +1,8 @@
|
||||
s3_data
|
||||
loki_data
|
||||
postgres_data
|
||||
prometheus_data
|
||||
redis_data
|
||||
tempo_data
|
||||
grafana_data
|
||||
rabbitmq_data
|
||||
@@ -0,0 +1,2 @@
|
||||
Container highload-lab6-otel-collector-run-0fb806dd4ede Creating
|
||||
Container highload-lab6-otel-collector-run-0fb806dd4ede Created
|
||||
@@ -0,0 +1,4 @@
|
||||
Источник Grafana: prometheus
|
||||
Источник Grafana: loki
|
||||
Источник Grafana: tempo
|
||||
Загружен dashboard messenger-lab6, панелей: 17
|
||||
@@ -0,0 +1,17 @@
|
||||
API · запросы в секунду — sum(rate(messenger_http_requests_total[1m])) — рядов: 1
|
||||
API · 5xx в секунду — sum(rate(messenger_http_requests_total{http_status_class="5xx"}[1m])) or 0 * sum(rate(messenger_http_requests_total[1m])) — рядов: 1
|
||||
API · p95 длительности — histogram_quantile(0.95, sum by (le) (rate(messenger_http_duration_seconds_bucket[1m]))) — рядов: 1
|
||||
Worker · задания в минуту — 60 * sum by (job_outcome) (rate(messenger_worker_jobs_total[1m])) — рядов: 3
|
||||
Worker · p95 длительности — histogram_quantile(0.95, sum by (le,job_outcome) (rate(messenger_worker_duration_seconds_bucket[1m]))) — рядов: 3
|
||||
API · p50 длительности — histogram_quantile(0.5, sum by (le) (rate(messenger_http_duration_seconds_bucket[1m]))) — рядов: 1
|
||||
API · p99 длительности — histogram_quantile(0.99, sum by (le) (rate(messenger_http_duration_seconds_bucket[1m]))) — рядов: 1
|
||||
API · доля 5xx — (sum(rate(messenger_http_requests_total{http_status_class="5xx"}[1m])) or 0 * sum(rate(messenger_http_requests_total[1m]))) / sum(rate(messenger_http_requests_total[1m])) — рядов: 1
|
||||
Очередь attachments.v1 · незавершённые задания — max(messenger_queue_depth) — рядов: 1
|
||||
Очередь attachments.v1 · возраст старейшего задания — max(messenger_queue_oldest_seconds) — рядов: 1
|
||||
PostgreSQL · занятые соединения — sum by (service_name) (messenger_db_pool_used) — рядов: 3
|
||||
PostgreSQL · запросы к пулу, ожидавшие соединение, в секунду — sum by (service_name) (rate(messenger_db_pool_waited_total[1m])) — рядов: 3
|
||||
Worker · загрузка — sum(messenger_worker_busy) / sum(messenger_worker_capacity) — рядов: 1
|
||||
Вложения · p95 до ready — histogram_quantile(0.95, sum by (le) (rate(messenger_attachment_ready_seconds_bucket[1m]))) — рядов: 1
|
||||
Процессы · загрузка CPU — sum by (service_name) (rate(messenger_process_cpu_seconds_total[1m])) — рядов: 3
|
||||
Процессы · резидентная память — sum by (service_name) (messenger_process_rss_bytes) — рядов: 3
|
||||
SLI за 30 минут: 0.9992789095149922 цель: >= 0.99
|
||||
@@ -0,0 +1,2 @@
|
||||
Checking /etc/prometheus/alerts.yaml
|
||||
SUCCESS: 2 rules found
|
||||
@@ -0,0 +1,13 @@
|
||||
MessengerHighErrorRate firing
|
||||
pg_terminate_backend
|
||||
----------------------
|
||||
t
|
||||
(1 row)
|
||||
|
||||
FATAL: terminating connection due to administrator command
|
||||
server closed the connection unexpectedly
|
||||
This probably means the server terminated abnormally
|
||||
before or while processing the request.
|
||||
connection to server was lost
|
||||
Медленный запрос PostgreSQL подтверждён в trace: 624d2df384e6bb8e27ad11ca29a41d0b
|
||||
MessengerHighErrorRate resolved
|
||||
@@ -0,0 +1,47 @@
|
||||
Architecture: x86_64
|
||||
CPU op-mode(s): 32-bit, 64-bit
|
||||
Address sizes: 48 bits physical, 48 bits virtual
|
||||
Byte Order: Little Endian
|
||||
CPU(s): 16
|
||||
On-line CPU(s) list: 0-15
|
||||
Vendor ID: AuthenticAMD
|
||||
Model name: AMD Ryzen 9 PRO 7940HS w/ Radeon 780M Graphics
|
||||
CPU family: 25
|
||||
Model: 116
|
||||
Thread(s) per core: 2
|
||||
Core(s) per socket: 8
|
||||
Socket(s): 1
|
||||
Stepping: 1
|
||||
Microcode version: 0xa70410a
|
||||
Frequency boost: enabled
|
||||
CPU(s) scaling MHz: 90%
|
||||
CPU max MHz: 5263,0610
|
||||
CPU min MHz: 402,7860
|
||||
BogoMIPS: 7985,14
|
||||
Flags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good amd_lbr_v2 nopl xtopology nonstop_tsc cpuid extd_apicid aperfmperf rapl pni pclmulqdq monitor ssse3 fma cx16 sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand lahf_lm cmp_legacy svm extapic cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw ibs skinit wdt tce topoext perfctr_core perfctr_nb bpext perfctr_llc mwaitx cpuid_fault cpb cat_l3 cdp_l3 hw_pstate ssbd mba perfmon_v2 ibrs ibpb stibp ibrs_enhanced vmmcall fsgsbase bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local user_shstk avx512_bf16 clzero irperf xsaveerptr rdpru wbnoinvd cppc arat npt lbrv svm_lock nrip_save tsc_scale vmcb_clean flushbyasid decodeassists pausefilter pfthreshold vgif x2avic v_spec_ctrl vnmi avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg avx512_vpopcntdq rdpid overflow_recov succor smca fsrm flush_l1d amd_lbr_pmc_freeze
|
||||
Virtualization: AMD-V
|
||||
L1d cache: 256 KiB (8 instances)
|
||||
L1i cache: 256 KiB (8 instances)
|
||||
L2 cache: 8 MiB (8 instances)
|
||||
L3 cache: 16 MiB (1 instance)
|
||||
NUMA node(s): 1
|
||||
NUMA node0 CPU(s): 0-15
|
||||
Vulnerability Gather data sampling: Not affected
|
||||
Vulnerability Ghostwrite: Not affected
|
||||
Vulnerability Indirect target selection: Not affected
|
||||
Vulnerability Itlb multihit: Not affected
|
||||
Vulnerability L1tf: Not affected
|
||||
Vulnerability Mds: Not affected
|
||||
Vulnerability Meltdown: Not affected
|
||||
Vulnerability Mmio stale data: Not affected
|
||||
Vulnerability Old microcode: Not affected
|
||||
Vulnerability Reg file data sampling: Not affected
|
||||
Vulnerability Retbleed: Not affected
|
||||
Vulnerability Spec rstack overflow: Mitigation; Safe RET
|
||||
Vulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl
|
||||
Vulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization
|
||||
Vulnerability Spectre v2: Mitigation; Enhanced / Automatic IBRS; IBPB conditional; STIBP always-on; PBRSB-eIBRS Not affected; BHI Not affected
|
||||
Vulnerability Srbds: Not affected
|
||||
Vulnerability Tsa: Mitigation; Clear CPU buffers
|
||||
Vulnerability Tsx async abort: Not affected
|
||||
Vulnerability Vmscape: Mitigation; IBPB before exit to userspace
|
||||
@@ -0,0 +1,3 @@
|
||||
total used free shared buff/cache available
|
||||
Mem: 14Gi 11Gi 1,3Gi 139Mi 2,3Gi 3,1Gi
|
||||
Swap: 8,0Gi 8,0Gi 548Ki
|
||||
@@ -0,0 +1 @@
|
||||
Docker Compose version v5.7.0
|
||||
@@ -0,0 +1,17 @@
|
||||
{
|
||||
"lab": 6,
|
||||
"workload": "closed-loop POST messages; no automatic retries",
|
||||
"requested_duration_seconds": 30.0,
|
||||
"elapsed_seconds": 30.004,
|
||||
"concurrency": 4,
|
||||
"successful_requests": 37708,
|
||||
"failed_requests": 0,
|
||||
"error_ratio": 0.0,
|
||||
"successful_rps": 1256.75,
|
||||
"successful_latency_ms": {
|
||||
"p50": 2.828,
|
||||
"p95": 4.327,
|
||||
"p99": 7.211
|
||||
},
|
||||
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
CONTAINER ID NAME CPU % MEM USAGE / LIMIT MEM % NET I/O BLOCK I/O PIDS
|
||||
2ee09eab650d highload-lab6-worker-1 0.01% 10.86MiB / 512MiB 2.12% 274kB / 393kB 4.58MB / 0B 7
|
||||
7fdc55311297 highload-lab6-dispatcher-1 0.03% 10.59MiB / 128MiB 8.27% 1.05MB / 1.41MB 1.33MB / 422kB 7
|
||||
188cf5f1391d highload-lab6-api-2 2.56% 22.67MiB / 512MiB 4.43% 85.5MB / 161MB 8.96MB / 406kB 7
|
||||
93535c5e7b67 highload-lab6-api-1 0.06% 21.67MiB / 512MiB 4.23% 86.4MB / 163MB 8.18MB / 389kB 7
|
||||
ae279d3a717c highload-lab6-upload-cleanup-1 0.00% 4.543MiB / 128MiB 3.55% 35.6kB / 31.2kB 32.8kB / 532kB 12
|
||||
db0e2da28fb1 highload-lab6-prometheus-1 0.23% 35.93MiB / 14.89GiB 0.24% 3.86MB / 1.91MB 426MB / 6.98MB 22
|
||||
66b201cac5eb highload-lab6-grafana-1 0.68% 89.27MiB / 14.89GiB 0.59% 59.5MB / 116MB 1.32GB / 242MB 25
|
||||
0d0412e6fe25 highload-lab6-tempo-1 0.02% 307.1MiB / 14.89GiB 2.01% 9.75MB / 202kB 58.6MB / 55.3MB 21
|
||||
a8fc40b42323 highload-lab6-loki-1 0.92% 161.8MiB / 14.89GiB 1.06% 5.45MB / 32.4MB 534MB / 24.9MB 21
|
||||
a2048ea9933c highload-lab6-otel-collector-1 0.19% 60.13MiB / 192MiB 31.32% 123MB / 28.3MB 489MB / 20.9MB 21
|
||||
efeed09a36ec highload-lab6-frontend-1 0.27% 24.01MiB / 512MiB 4.69% 52.3kB / 7.76MB 562MB / 161MB 36
|
||||
2ac44f5d15bb highload-lab6-haproxy-1 2.36% 10.5MiB / 128MiB 8.20% 104MB / 106MB 30.2MB / 14MB 17
|
||||
3999d26f1e22 highload-lab6-s3-1 9.80% 354.4MiB / 512MiB 69.22% 1.07MB / 760kB 125MB / 10.2MB 21
|
||||
8837cc38b561 highload-lab6-db-1 0.66% 80.51MiB / 512MiB 15.72% 160MB / 109MB 459MB / 5.37GB 18
|
||||
3e7d2e507d3d highload-lab6-broker-1 23.55% 89.5MiB / 512MiB 17.48% 374kB / 333kB 267MB / 91.5MB 47
|
||||
65ad13aee585 highload-lab6-redis-1 2.71% 4.18MiB / 128MiB 3.27% 13.8MB / 8.47MB 163MB / 7.52MB 6
|
||||
@@ -0,0 +1 @@
|
||||
Upload → producer → consumer → S3/PostgreSQL, trace: a1b887f518133aa40e303527d73f514e
|
||||
@@ -0,0 +1,25 @@
|
||||
Container highload-lab6-worker-1 Stopping
|
||||
Container highload-lab6-worker-1 Stopped
|
||||
Container highload-lab6-s3-1 Stopping
|
||||
Container highload-lab6-s3-1 Stopped
|
||||
Container highload-lab6-worker-1 Starting
|
||||
Container highload-lab6-worker-1 Started
|
||||
Container highload-lab6-s3-1 Starting
|
||||
Container highload-lab6-s3-1 Started
|
||||
Container highload-lab6-s3-1 Waiting
|
||||
Container highload-lab6-s3-1 Healthy
|
||||
Исходное вложение перешло в ready, обе попытки в trace: 762071e58a275355660808c10b605df8
|
||||
Container highload-lab6-broker-1 Waiting
|
||||
Container highload-lab6-db-1 Waiting
|
||||
Container highload-lab6-s3-1 Waiting
|
||||
Container highload-lab6-db-1 Healthy
|
||||
Container highload-lab6-migrate-1 Starting
|
||||
Container highload-lab6-broker-1 Healthy
|
||||
Container highload-lab6-s3-1 Healthy
|
||||
Container highload-lab6-s3-bootstrap-1 Starting
|
||||
Container highload-lab6-migrate-1 Started
|
||||
Container highload-lab6-migrate-1 Waiting
|
||||
Container highload-lab6-s3-bootstrap-1 Started
|
||||
Container highload-lab6-s3-bootstrap-1 Waiting
|
||||
Container highload-lab6-migrate-1 Exited
|
||||
Container highload-lab6-s3-bootstrap-1 Exited
|
||||
@@ -0,0 +1,20 @@
|
||||
Container highload-lab6-worker-1 Stopping
|
||||
Container highload-lab6-worker-1 Stopped
|
||||
MessengerQueueLag firing
|
||||
Container highload-lab6-broker-1 Waiting
|
||||
Container highload-lab6-db-1 Waiting
|
||||
Container highload-lab6-s3-1 Waiting
|
||||
Container highload-lab6-db-1 Healthy
|
||||
Container highload-lab6-migrate-1 Starting
|
||||
Container highload-lab6-s3-1 Healthy
|
||||
Container highload-lab6-s3-bootstrap-1 Starting
|
||||
Container highload-lab6-broker-1 Healthy
|
||||
Container highload-lab6-migrate-1 Started
|
||||
Container highload-lab6-migrate-1 Waiting
|
||||
Container highload-lab6-s3-bootstrap-1 Started
|
||||
Container highload-lab6-s3-bootstrap-1 Waiting
|
||||
Container highload-lab6-migrate-1 Exited
|
||||
Container highload-lab6-s3-bootstrap-1 Exited
|
||||
Container highload-lab6-worker-1 Starting
|
||||
Container highload-lab6-worker-1 Started
|
||||
MessengerQueueLag resolved
|
||||
@@ -0,0 +1,8 @@
|
||||
Container highload-lab6-tempo-1 Stopping
|
||||
Container highload-lab6-tempo-1 Stopped
|
||||
CONTAINER ID NAME CPU % MEM USAGE / LIMIT MEM % NET I/O BLOCK I/O PIDS
|
||||
a2048ea9933c highload-lab6-otel-collector-1 0.42% 56.43MiB / 192MiB 29.39% 69.6MB / 15.8MB 466MB / 20.8MB 21
|
||||
API и worker работали при остановленном Tempo.
|
||||
Container highload-lab6-tempo-1 Starting
|
||||
Container highload-lab6-tempo-1 Started
|
||||
Спанов отброшено Collector за период проверки: 54
|
||||
@@ -0,0 +1,140 @@
|
||||
{
|
||||
"grade3-stand": [
|
||||
{
|
||||
"command": "docker compose config --quiet",
|
||||
"exit_code": 0,
|
||||
"seconds": 0.08,
|
||||
"output": "grade3-stand-1.txt"
|
||||
},
|
||||
{
|
||||
"command": "docker compose ps --all",
|
||||
"exit_code": 0,
|
||||
"seconds": 0.11,
|
||||
"output": "grade3-stand-2.txt"
|
||||
},
|
||||
{
|
||||
"command": "docker compose config --volumes",
|
||||
"exit_code": 0,
|
||||
"seconds": 0.08,
|
||||
"output": "grade3-stand-3.txt"
|
||||
},
|
||||
{
|
||||
"command": "docker compose run --rm --no-deps otel-collector validate --config=/etc/otelcol-contrib/config.yaml",
|
||||
"exit_code": 0,
|
||||
"seconds": 1.28,
|
||||
"output": "grade3-stand-4.txt"
|
||||
},
|
||||
{
|
||||
"command": "/usr/bin/python3 scripts/check_observability.py --base-url http://127.0.0.1:8080 --grafana-url http://127.0.0.1:3000 provisioning",
|
||||
"exit_code": 0,
|
||||
"seconds": 0.13,
|
||||
"output": "grade3-stand-5.txt"
|
||||
}
|
||||
],
|
||||
"grade3-smoke": [
|
||||
{
|
||||
"command": "make check",
|
||||
"exit_code": 0,
|
||||
"seconds": 0.05,
|
||||
"output": "grade3-smoke-1.txt"
|
||||
},
|
||||
{
|
||||
"command": "make test BASE_URL=http://127.0.0.1:8080",
|
||||
"exit_code": 0,
|
||||
"seconds": 3.51,
|
||||
"output": "grade3-smoke-2.txt"
|
||||
}
|
||||
],
|
||||
"grade3-signals": [
|
||||
{
|
||||
"command": "/usr/bin/python3 scripts/check_observability.py --base-url http://127.0.0.1:8080 --grafana-url http://127.0.0.1:3000 signals",
|
||||
"exit_code": 0,
|
||||
"seconds": 7.2,
|
||||
"output": "grade3-signals-1.txt"
|
||||
}
|
||||
],
|
||||
"grade4-pipeline": [
|
||||
{
|
||||
"command": "/usr/bin/python3 scripts/check_observability.py --base-url http://127.0.0.1:8080 --grafana-url http://127.0.0.1:3000 pipeline",
|
||||
"exit_code": 0,
|
||||
"seconds": 4.92,
|
||||
"output": "grade4-pipeline-1.txt"
|
||||
},
|
||||
{
|
||||
"command": "/usr/bin/python3 scripts/check_observability_incidents.py --base-url http://127.0.0.1:8080 --grafana-url http://127.0.0.1:3000 retry",
|
||||
"exit_code": 0,
|
||||
"seconds": 15.62,
|
||||
"output": "grade4-pipeline-2.txt"
|
||||
}
|
||||
],
|
||||
"grade4-dashboard": [
|
||||
{
|
||||
"command": "/usr/bin/python3 scripts/check_observability.py --base-url http://127.0.0.1:8080 --grafana-url http://127.0.0.1:3000 dashboard",
|
||||
"exit_code": 0,
|
||||
"seconds": 0.16,
|
||||
"output": "grade4-dashboard-1.txt"
|
||||
},
|
||||
{
|
||||
"command": "docker compose exec -T prometheus promtool check rules /etc/prometheus/alerts.yaml",
|
||||
"exit_code": 0,
|
||||
"seconds": 0.23,
|
||||
"output": "grade4-dashboard-2.txt"
|
||||
}
|
||||
],
|
||||
"grade4-queue": [
|
||||
{
|
||||
"command": "/usr/bin/python3 scripts/check_observability_incidents.py --base-url http://127.0.0.1:8080 --grafana-url http://127.0.0.1:3000 worker",
|
||||
"exit_code": 0,
|
||||
"seconds": 55.56,
|
||||
"output": "grade4-queue-1.txt"
|
||||
}
|
||||
],
|
||||
"grade4-database": [
|
||||
{
|
||||
"command": "/usr/bin/python3 scripts/check_observability_incidents.py --base-url http://127.0.0.1:8080 --grafana-url http://127.0.0.1:3000 database",
|
||||
"exit_code": 0,
|
||||
"seconds": 104.65,
|
||||
"output": "grade4-database-1.txt"
|
||||
}
|
||||
],
|
||||
"grade4-telemetry": [
|
||||
{
|
||||
"command": "/usr/bin/python3 scripts/check_observability_incidents.py --base-url http://127.0.0.1:8080 --grafana-url http://127.0.0.1:3000 telemetry",
|
||||
"exit_code": 0,
|
||||
"seconds": 46.39,
|
||||
"output": "grade4-telemetry-1.txt"
|
||||
}
|
||||
],
|
||||
"grade4-load": [
|
||||
{
|
||||
"command": "lscpu",
|
||||
"exit_code": 0,
|
||||
"seconds": 0.01,
|
||||
"output": "grade4-load-1.txt"
|
||||
},
|
||||
{
|
||||
"command": "free -h",
|
||||
"exit_code": 0,
|
||||
"seconds": 0.0,
|
||||
"output": "grade4-load-2.txt"
|
||||
},
|
||||
{
|
||||
"command": "docker compose version",
|
||||
"exit_code": 0,
|
||||
"seconds": 0.04,
|
||||
"output": "grade4-load-3.txt"
|
||||
},
|
||||
{
|
||||
"command": "/usr/bin/python3 tests/load.py --base-url http://127.0.0.1:8080 --duration 30 --concurrency 4",
|
||||
"exit_code": 0,
|
||||
"seconds": 31.48,
|
||||
"output": "grade4-load-4.txt"
|
||||
},
|
||||
{
|
||||
"command": "docker compose stats --no-stream",
|
||||
"exit_code": 0,
|
||||
"seconds": 2.17,
|
||||
"output": "grade4-load-5.txt"
|
||||
}
|
||||
]
|
||||
}
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 403 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 423 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 418 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 427 KiB |
@@ -0,0 +1,49 @@
|
||||
global
|
||||
# Operator access only inside this container; no host port is published.
|
||||
stats socket ipv4@127.0.0.1:9999 level admin
|
||||
log stdout format raw local0
|
||||
maxconn 1024
|
||||
|
||||
defaults
|
||||
log global
|
||||
mode http
|
||||
option httplog
|
||||
timeout connect 1s
|
||||
timeout client 35s
|
||||
timeout server 35s
|
||||
timeout check 1s
|
||||
# Let the client decide which failed POST is safe to repeat.
|
||||
retries 0
|
||||
|
||||
# Docker returns the current addresses of all containers of the api service.
|
||||
resolvers docker
|
||||
nameserver docker_dns 127.0.0.11:53
|
||||
accepted_payload_size 8192
|
||||
resolve_retries 3
|
||||
timeout resolve 1s
|
||||
timeout retry 1s
|
||||
hold valid 1s
|
||||
hold obsolete 1s
|
||||
hold nx 1s
|
||||
hold timeout 1s
|
||||
hold refused 1s
|
||||
hold other 1s
|
||||
|
||||
frontend api_entry
|
||||
bind :8080
|
||||
default_backend api_replicas
|
||||
|
||||
backend api_replicas
|
||||
balance roundrobin
|
||||
option httpchk GET /health/ready
|
||||
http-check expect status 200
|
||||
# Separate slots receive distinct Docker DNS addresses. No sticky sessions.
|
||||
server-template api "${API_REPLICAS}" "api:${APP_PORT}" check inter 1s fall 2 rise 2 resolvers docker resolve-prefer ipv4 init-addr libc,none
|
||||
|
||||
# Local teaching aid: backend addresses, UP/DOWN state and request counters.
|
||||
listen statistics
|
||||
bind :8404
|
||||
stats enable
|
||||
stats show-legends
|
||||
stats uri /stats
|
||||
stats refresh 2s
|
||||
@@ -0,0 +1,19 @@
|
||||
FROM golang:1.24.2-bookworm AS builder
|
||||
WORKDIR /source
|
||||
RUN git init . \
|
||||
&& git remote add origin https://github.com/minio/minio.git \
|
||||
&& git fetch --depth 1 origin 0d7408fc9969caf07de6a8c3a84f9fbb10a6739e \
|
||||
&& git checkout --detach FETCH_HEAD
|
||||
RUN CGO_ENABLED=0 go build -trimpath \
|
||||
-ldflags="-s -w -X github.com/minio/minio/cmd.Version=2025-04-22T22:12:26Z -X github.com/minio/minio/cmd.ReleaseTag=RELEASE.2025-04-22T22-12-26Z -X github.com/minio/minio/cmd.CommitID=0d7408fc9969caf07de6a8c3a84f9fbb10a6739e" \
|
||||
-o /out/minio .
|
||||
|
||||
FROM debian:12.15-slim
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends ca-certificates curl \
|
||||
&& rm -rf /var/lib/apt/lists/* \
|
||||
&& groupadd --system storage && useradd --system --gid storage storage \
|
||||
&& mkdir /data && chown storage:storage /data
|
||||
COPY --from=builder /out/minio /usr/local/bin/minio
|
||||
USER storage:storage
|
||||
ENTRYPOINT ["minio"]
|
||||
CMD ["server", "/data", "--console-address", ":9001"]
|
||||
@@ -0,0 +1,20 @@
|
||||
groups:
|
||||
- name: messenger
|
||||
interval: 5s
|
||||
rules:
|
||||
- alert: MessengerHighErrorRate
|
||||
expr: (sum(rate(messenger_http_requests_total{http_status_class="5xx"}[1m])) / sum(rate(messenger_http_requests_total[1m]))) > 0.05
|
||||
for: 15s
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: More than 5% of business requests return 5xx over one minute
|
||||
runbook: Check API error spans and PostgreSQL operations in Tempo.
|
||||
- alert: MessengerQueueLag
|
||||
expr: max(messenger_queue_oldest_seconds) > 20
|
||||
for: 15s
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: Oldest outstanding attachment job is over 20 seconds old
|
||||
runbook: Check outstanding jobs, worker saturation and consumer traces.
|
||||
@@ -0,0 +1,45 @@
|
||||
receivers:
|
||||
otlp:
|
||||
protocols:
|
||||
http:
|
||||
endpoint: 0.0.0.0:4318
|
||||
processors:
|
||||
memory_limiter:
|
||||
check_interval: 1s
|
||||
limit_mib: 128
|
||||
spike_limit_mib: 32
|
||||
batch:
|
||||
timeout: 1s
|
||||
send_batch_size: 256
|
||||
send_batch_max_size: 512
|
||||
exporters:
|
||||
prometheus:
|
||||
endpoint: 0.0.0.0:8889
|
||||
resource_to_telemetry_conversion:
|
||||
enabled: true
|
||||
otlphttp/loki:
|
||||
endpoint: http://loki:3100/otlp
|
||||
sending_queue:
|
||||
queue_size: 64
|
||||
retry_on_failure:
|
||||
max_elapsed_time: 30s
|
||||
otlphttp/tempo:
|
||||
endpoint: http://tempo:4318
|
||||
sending_queue:
|
||||
queue_size: 64
|
||||
retry_on_failure:
|
||||
max_elapsed_time: 30s
|
||||
service:
|
||||
pipelines:
|
||||
metrics:
|
||||
receivers: [otlp]
|
||||
processors: [memory_limiter, batch]
|
||||
exporters: [prometheus]
|
||||
logs:
|
||||
receivers: [otlp]
|
||||
processors: [memory_limiter, batch]
|
||||
exporters: [otlphttp/loki]
|
||||
traces:
|
||||
receivers: [otlp]
|
||||
processors: [memory_limiter, batch]
|
||||
exporters: [otlphttp/tempo]
|
||||
@@ -0,0 +1,513 @@
|
||||
{
|
||||
"uid": "messenger-lab6",
|
||||
"title": "Мессенджер · API и worker",
|
||||
"schemaVersion": 39,
|
||||
"refresh": "5s",
|
||||
"time": {
|
||||
"from": "now-15m",
|
||||
"to": "now"
|
||||
},
|
||||
"panels": [
|
||||
{
|
||||
"id": 1,
|
||||
"title": "API · запросы в секунду",
|
||||
"type": "timeseries",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "lab6-prometheus"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "lab6-prometheus"
|
||||
},
|
||||
"expr": "sum(rate(messenger_http_requests_total[1m]))",
|
||||
"legendFormat": ""
|
||||
}
|
||||
],
|
||||
"gridPos": {
|
||||
"x": 0,
|
||||
"y": 0,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "reqps"
|
||||
},
|
||||
"overrides": []
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 2,
|
||||
"title": "API · 5xx в секунду",
|
||||
"type": "timeseries",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "lab6-prometheus"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "lab6-prometheus"
|
||||
},
|
||||
"expr": "sum(rate(messenger_http_requests_total{http_status_class=\"5xx\"}[1m])) or 0 * sum(rate(messenger_http_requests_total[1m]))",
|
||||
"legendFormat": ""
|
||||
}
|
||||
],
|
||||
"gridPos": {
|
||||
"x": 12,
|
||||
"y": 0,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "reqps"
|
||||
},
|
||||
"overrides": []
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 3,
|
||||
"title": "API · p95 длительности",
|
||||
"type": "timeseries",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "lab6-prometheus"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "lab6-prometheus"
|
||||
},
|
||||
"expr": "histogram_quantile(0.95, sum by (le) (rate(messenger_http_duration_seconds_bucket[1m])))",
|
||||
"legendFormat": ""
|
||||
}
|
||||
],
|
||||
"gridPos": {
|
||||
"x": 0,
|
||||
"y": 8,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "s"
|
||||
},
|
||||
"overrides": []
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 4,
|
||||
"title": "Worker · задания в минуту",
|
||||
"type": "timeseries",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "lab6-prometheus"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "lab6-prometheus"
|
||||
},
|
||||
"expr": "60 * sum by (job_outcome) (rate(messenger_worker_jobs_total[1m]))",
|
||||
"legendFormat": "{{job_outcome}}"
|
||||
}
|
||||
],
|
||||
"gridPos": {
|
||||
"x": 12,
|
||||
"y": 8,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "short"
|
||||
},
|
||||
"overrides": []
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 5,
|
||||
"title": "Worker · p95 длительности",
|
||||
"type": "timeseries",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "lab6-prometheus"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "lab6-prometheus"
|
||||
},
|
||||
"expr": "histogram_quantile(0.95, sum by (le,job_outcome) (rate(messenger_worker_duration_seconds_bucket[1m])))",
|
||||
"legendFormat": "{{job_outcome}}"
|
||||
}
|
||||
],
|
||||
"gridPos": {
|
||||
"x": 0,
|
||||
"y": 16,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "s"
|
||||
},
|
||||
"overrides": []
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 6,
|
||||
"title": "API и worker · события",
|
||||
"type": "logs",
|
||||
"datasource": {
|
||||
"type": "loki",
|
||||
"uid": "lab6-loki"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": {
|
||||
"type": "loki",
|
||||
"uid": "lab6-loki"
|
||||
},
|
||||
"expr": "{service_name=~\"messenger-api|messenger-worker\"}",
|
||||
"legendFormat": "",
|
||||
"queryType": "range"
|
||||
}
|
||||
],
|
||||
"gridPos": {
|
||||
"x": 12,
|
||||
"y": 16,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": ""
|
||||
},
|
||||
"overrides": []
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 7,
|
||||
"title": "API · p50 длительности",
|
||||
"type": "timeseries",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "lab6-prometheus"
|
||||
},
|
||||
"gridPos": {
|
||||
"x": 0,
|
||||
"y": 24,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "histogram_quantile(0.5, sum by (le) (rate(messenger_http_duration_seconds_bucket[1m])))",
|
||||
"legendFormat": ""
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "s"
|
||||
},
|
||||
"overrides": []
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 8,
|
||||
"title": "API · p99 длительности",
|
||||
"type": "timeseries",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "lab6-prometheus"
|
||||
},
|
||||
"gridPos": {
|
||||
"x": 12,
|
||||
"y": 24,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "histogram_quantile(0.99, sum by (le) (rate(messenger_http_duration_seconds_bucket[1m])))",
|
||||
"legendFormat": ""
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "s"
|
||||
},
|
||||
"overrides": []
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 9,
|
||||
"title": "API · доля 5xx",
|
||||
"type": "timeseries",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "lab6-prometheus"
|
||||
},
|
||||
"gridPos": {
|
||||
"x": 0,
|
||||
"y": 32,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "(sum(rate(messenger_http_requests_total{http_status_class=\"5xx\"}[1m])) or 0 * sum(rate(messenger_http_requests_total[1m]))) / sum(rate(messenger_http_requests_total[1m]))",
|
||||
"legendFormat": ""
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "percentunit"
|
||||
},
|
||||
"overrides": []
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 10,
|
||||
"title": "Очередь attachments.v1 · незавершённые задания",
|
||||
"type": "timeseries",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "lab6-prometheus"
|
||||
},
|
||||
"gridPos": {
|
||||
"x": 12,
|
||||
"y": 32,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "max(messenger_queue_depth)",
|
||||
"legendFormat": ""
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "short"
|
||||
},
|
||||
"overrides": []
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 11,
|
||||
"title": "Очередь attachments.v1 · возраст старейшего задания",
|
||||
"type": "timeseries",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "lab6-prometheus"
|
||||
},
|
||||
"gridPos": {
|
||||
"x": 0,
|
||||
"y": 40,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "max(messenger_queue_oldest_seconds)",
|
||||
"legendFormat": ""
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "s"
|
||||
},
|
||||
"overrides": []
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 12,
|
||||
"title": "PostgreSQL · занятые соединения",
|
||||
"type": "timeseries",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "lab6-prometheus"
|
||||
},
|
||||
"gridPos": {
|
||||
"x": 12,
|
||||
"y": 40,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "sum by (service_name) (messenger_db_pool_used)",
|
||||
"legendFormat": "{{service_name}}"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "short"
|
||||
},
|
||||
"overrides": []
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 13,
|
||||
"title": "PostgreSQL · запросы к пулу, ожидавшие соединение, в секунду",
|
||||
"type": "timeseries",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "lab6-prometheus"
|
||||
},
|
||||
"gridPos": {
|
||||
"x": 0,
|
||||
"y": 48,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "sum by (service_name) (rate(messenger_db_pool_waited_total[1m]))",
|
||||
"legendFormat": "{{service_name}}"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "ops"
|
||||
},
|
||||
"overrides": []
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 14,
|
||||
"title": "Worker · загрузка",
|
||||
"type": "timeseries",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "lab6-prometheus"
|
||||
},
|
||||
"gridPos": {
|
||||
"x": 12,
|
||||
"y": 48,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "sum(messenger_worker_busy) / sum(messenger_worker_capacity)",
|
||||
"legendFormat": ""
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "percentunit"
|
||||
},
|
||||
"overrides": []
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 15,
|
||||
"title": "Вложения · p95 до ready",
|
||||
"type": "timeseries",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "lab6-prometheus"
|
||||
},
|
||||
"gridPos": {
|
||||
"x": 0,
|
||||
"y": 56,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "histogram_quantile(0.95, sum by (le) (rate(messenger_attachment_ready_seconds_bucket[1m])))",
|
||||
"legendFormat": ""
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "s"
|
||||
},
|
||||
"overrides": []
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 16,
|
||||
"title": "Процессы · загрузка CPU",
|
||||
"type": "timeseries",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "lab6-prometheus"
|
||||
},
|
||||
"gridPos": {
|
||||
"x": 12,
|
||||
"y": 56,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "sum by (service_name) (rate(messenger_process_cpu_seconds_total[1m]))",
|
||||
"legendFormat": "{{service_name}}"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "cores"
|
||||
},
|
||||
"overrides": []
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 17,
|
||||
"title": "Процессы · резидентная память",
|
||||
"type": "timeseries",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "lab6-prometheus"
|
||||
},
|
||||
"gridPos": {
|
||||
"x": 0,
|
||||
"y": 64,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "sum by (service_name) (messenger_process_rss_bytes)",
|
||||
"legendFormat": "{{service_name}}"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "bytes"
|
||||
},
|
||||
"overrides": []
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,6 @@
|
||||
apiVersion: 1
|
||||
providers:
|
||||
- name: Messenger
|
||||
type: file
|
||||
options:
|
||||
path: /var/lib/grafana/dashboards
|
||||
@@ -0,0 +1,32 @@
|
||||
apiVersion: 1
|
||||
datasources:
|
||||
- name: Prometheus
|
||||
uid: lab6-prometheus
|
||||
type: prometheus
|
||||
access: proxy
|
||||
url: http://prometheus:9090
|
||||
isDefault: true
|
||||
- name: Loki
|
||||
uid: lab6-loki
|
||||
type: loki
|
||||
access: proxy
|
||||
url: http://loki:3100
|
||||
jsonData:
|
||||
derivedFields:
|
||||
- name: Trace
|
||||
matcherType: label
|
||||
matcherRegex: trace_id
|
||||
datasourceUid: lab6-tempo
|
||||
url: '$${__value.raw}'
|
||||
- name: Tempo
|
||||
uid: lab6-tempo
|
||||
type: tempo
|
||||
access: proxy
|
||||
url: http://tempo:3200
|
||||
jsonData:
|
||||
tracesToLogsV2:
|
||||
datasourceUid: lab6-loki
|
||||
spanStartTimeShift: '-5s'
|
||||
spanEndTimeShift: '5s'
|
||||
customQuery: true
|
||||
query: '{service_name=~"messenger-api|messenger-worker"} | trace_id="$${__span.traceId}"'
|
||||
@@ -0,0 +1,31 @@
|
||||
auth_enabled: false
|
||||
server:
|
||||
http_listen_port: 3100
|
||||
common:
|
||||
path_prefix: /loki
|
||||
replication_factor: 1
|
||||
ring:
|
||||
instance_addr: 127.0.0.1
|
||||
kvstore:
|
||||
store: inmemory
|
||||
storage:
|
||||
filesystem:
|
||||
chunks_directory: /loki/chunks
|
||||
rules_directory: /loki/rules
|
||||
schema_config:
|
||||
configs:
|
||||
- from: 2024-01-01
|
||||
store: tsdb
|
||||
object_store: filesystem
|
||||
schema: v13
|
||||
index:
|
||||
prefix: index_
|
||||
period: 24h
|
||||
limits_config:
|
||||
allow_structured_metadata: true
|
||||
otlp_config:
|
||||
resource_attributes:
|
||||
ignore_defaults: true
|
||||
attributes_config:
|
||||
- action: index_label
|
||||
attributes: [service.name, deployment.environment.name]
|
||||
@@ -0,0 +1,8 @@
|
||||
global:
|
||||
scrape_interval: 5s
|
||||
rule_files:
|
||||
- /etc/prometheus/alerts.yaml
|
||||
scrape_configs:
|
||||
- job_name: otel-collector
|
||||
static_configs:
|
||||
- targets: [otel-collector:8889]
|
||||
@@ -0,0 +1,27 @@
|
||||
# Диагностика API и обработки вложений
|
||||
|
||||
SLI доступности API — доля бизнес-запросов без 5xx. Цель — ≥99% за 30 минут при наличии трафика. Проверка:
|
||||
|
||||
```promql
|
||||
1 - (sum(increase(messenger_http_requests_total{http_status_class="5xx"}[30m])) or 0 * sum(increase(messenger_http_requests_total[30m]))) / sum(increase(messenger_http_requests_total[30m]))
|
||||
```
|
||||
|
||||
Health исключён из обоих счётчиков. `MessengerHighErrorRate` срабатывает при доле 5xx выше 5% за минуту в течение 15 секунд. В Explore → Tempo найти ошибочный серверный спан и дочерний `postgresql query`. Сравнить длительность дочернего спана с длительностью HTTP, затем сверить занятые соединения и rate запросов, ожидавших пул.
|
||||
|
||||
`MessengerQueueLag` срабатывает, когда старейшее незавершённое задание старше 20 секунд в течение 15 секунд. В dashboard проверить число незавершённых заданий и наличие worker. Если возраст растёт при отсутствии новых consumer spans, проверить процесс worker. После восстановления дождаться перехода заданий в ready и исчезновения активного alert.
|
||||
|
||||
Глубина очереди измеряется по устойчивым заданиям в PostgreSQL. В неё входят работа в RabbitMQ, leases и ожидание retry. Возраст измеряется от создания задания. Время до ready включает ожидание и повторные попытки. Гистограммы времени используют секунды. Загрузка worker — доля занятых слотов, по одному слоту на процесс. Ожидание пула отображается как частота успешных получений соединения после ожидания. Метрики CPU и RSS относятся к основным процессам Go.
|
||||
|
||||
Правила доступны через API Prometheus `/api/v1/alerts` или запрос `ALERTS{alertstate="firing"}` в Grafana. При недоступности Loki или Tempo приложение продолжает работу. Collector ограничивает память и очередь экспорта, повторяет отправку до 30 секунд. Сигналы сверх лимитов или после истечения retry могут быть потеряны.
|
||||
|
||||
Команды локальной проверки после `make app-up`:
|
||||
|
||||
```sh
|
||||
python3 scripts/check_observability.py pipeline
|
||||
python3 scripts/check_observability_incidents.py worker
|
||||
python3 scripts/check_observability_incidents.py database
|
||||
python3 scripts/check_observability_incidents.py retry
|
||||
python3 scripts/check_observability_incidents.py telemetry
|
||||
```
|
||||
|
||||
Сценарии выполняются последовательно на учебном стенде. DB-инцидент держит временную блокировку таблицы чатов. Проверка retry кратко останавливает S3 и проверяет номера попыток в одной трассе исходного upload. Проверка telemetry останавливает Tempo и одновременно отправляет сообщения и обрабатывает вложение.
|
||||
@@ -0,0 +1,18 @@
|
||||
server:
|
||||
http_listen_port: 3200
|
||||
distributor:
|
||||
receivers:
|
||||
otlp:
|
||||
protocols:
|
||||
http:
|
||||
endpoint: 0.0.0.0:4318
|
||||
storage:
|
||||
trace:
|
||||
backend: local
|
||||
wal:
|
||||
path: /var/tempo/wal
|
||||
local:
|
||||
path: /var/tempo/blocks
|
||||
compactor:
|
||||
compaction:
|
||||
block_retention: 48h
|
||||
@@ -0,0 +1,204 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Проверка сохранённых сигналов через источники Grafana."""
|
||||
import argparse
|
||||
import base64
|
||||
import json
|
||||
from pathlib import Path
|
||||
import re
|
||||
import struct
|
||||
import sys
|
||||
import time
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
import uuid
|
||||
import zlib
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / "tests"))
|
||||
from client import Client, Failure, check
|
||||
|
||||
|
||||
def oversized_image():
|
||||
# 9 million pixels fit the API limit but exceed the default worker limit.
|
||||
def chunk(kind, data):
|
||||
return struct.pack(">I", len(data)) + kind + data + struct.pack(">I", zlib.crc32(kind + data))
|
||||
header = struct.pack(">IIBBBBB", 3000, 3000, 8, 0, 0, 0, 0)
|
||||
return (bytes.fromhex("89504e470d0a1a0a") + chunk(b"IHDR", header)
|
||||
+ chunk(b"IDAT", zlib.compress(bytes(3001 * 3000))) + chunk(b"IEND", b""))
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--base-url", default="http://127.0.0.1:8080")
|
||||
parser.add_argument("--grafana-url", default="http://127.0.0.1:3000")
|
||||
parser.add_argument("criterion", choices=["provisioning", "signals", "pipeline", "dashboard"])
|
||||
args = parser.parse_args()
|
||||
|
||||
def query(path, **params):
|
||||
suffix = "?" + urllib.parse.urlencode(params) if params else ""
|
||||
with urllib.request.urlopen(args.grafana_url.rstrip("/") + path + suffix, timeout=5) as response:
|
||||
return json.load(response)
|
||||
|
||||
def source(uid, path, **params):
|
||||
return query("/api/datasources/proxy/uid/lab6-" + uid + path, **params)
|
||||
|
||||
if args.criterion == "provisioning":
|
||||
for uid in ("prometheus", "loki", "tempo"):
|
||||
check(query("/api/datasources/uid/lab6-" + uid)["type"] == uid, "source type mismatch")
|
||||
print("Источник Grafana:", uid)
|
||||
dashboard = query("/api/dashboards/uid/messenger-lab6")["dashboard"]
|
||||
check(len(dashboard["panels"]) >= 6, "dashboard panels missing")
|
||||
print("Загружен dashboard messenger-lab6, панелей:", len(dashboard["panels"]))
|
||||
return
|
||||
|
||||
if args.criterion == "dashboard":
|
||||
dashboard = query("/api/dashboards/uid/messenger-lab6")["dashboard"]
|
||||
check(len(dashboard["panels"]) >= 17, "grade 4 dashboard panels missing")
|
||||
for panel in dashboard["panels"]:
|
||||
if panel["datasource"]["type"] != "prometheus":
|
||||
continue
|
||||
for target in panel["targets"]:
|
||||
result = source("prometheus", "/api/v1/query", query=target["expr"])
|
||||
check(result["status"] == "success", "PromQL query failed")
|
||||
print(panel["title"], "—", target["expr"], "— рядов:", len(result["data"]["result"]))
|
||||
expression = ('1 - (sum(increase(messenger_http_requests_total{http_status_class="5xx"}[30m])) '
|
||||
'or 0 * sum(increase(messenger_http_requests_total[30m]))) '
|
||||
'/ sum(increase(messenger_http_requests_total[30m]))')
|
||||
result = source("prometheus", "/api/v1/query", query=expression)["data"]["result"]
|
||||
check(bool(result), "SLI series missing")
|
||||
print("SLI за 30 минут:", result[0]["value"][1], "цель: >= 0.99")
|
||||
return
|
||||
|
||||
client = Client(args.base_url)
|
||||
start = time.time_ns()
|
||||
alice, bob, _, chat = client.bootstrap()
|
||||
if args.criterion == "pipeline":
|
||||
upload = client.call("POST", "/api/v1/chats/" + chat["id"] + "/attachments?filename=sample.png",
|
||||
202, actor=alice, raw=(ROOT / "tests/fixtures/sample.png").read_bytes(),
|
||||
headers={"Content-Type": "image/png"}, schema="Attachment")
|
||||
client.wait_ready(bob, upload["json"])
|
||||
request_id = upload["headers"]["X-Request-Id"]
|
||||
deadline = time.monotonic() + 45
|
||||
while time.monotonic() < deadline:
|
||||
logs = source("loki", "/loki/api/v1/query_range",
|
||||
query='{service_name="messenger-api"} | request_id="' + request_id + '"',
|
||||
start=start, end=time.time_ns())["data"]["result"]
|
||||
if logs:
|
||||
trace_id = logs[0]["stream"]["trace_id"]
|
||||
try:
|
||||
trace = source("tempo", "/api/traces/" + trace_id)
|
||||
except urllib.error.HTTPError as error:
|
||||
if error.code != 404:
|
||||
raise
|
||||
time.sleep(2)
|
||||
continue
|
||||
spans = [span for batch in trace["batches"] for scope in batch["scopeSpans"] for span in scope["spans"]]
|
||||
by_id = {span["spanId"]: span for span in spans}
|
||||
for consumer in (span for span in spans if span["kind"] == "SPAN_KIND_CONSUMER"):
|
||||
producer = by_id.get(consumer.get("parentSpanId"), {})
|
||||
api = by_id.get(producer.get("parentSpanId"), {})
|
||||
children = {span["name"] for span in spans if span.get("parentSpanId") == consumer["spanId"]}
|
||||
if (producer.get("kind") == "SPAN_KIND_PRODUCER" and api.get("kind") == "SPAN_KIND_SERVER"
|
||||
and {"postgresql finalize attachment", "s3 read staging and write variants"} <= children):
|
||||
print("Upload → producer → consumer → S3/PostgreSQL, trace:", trace_id)
|
||||
return
|
||||
time.sleep(2)
|
||||
raise Failure("upload trace did not include the worker pipeline")
|
||||
|
||||
marker = "telemetry-secret-" + uuid.uuid4().hex
|
||||
message = client.message(alice, chat, marker)
|
||||
request_id = message["headers"]["X-Request-Id"]
|
||||
attachment, original = client.attachment(alice, chat)
|
||||
client.wait_ready(bob, attachment)
|
||||
oversized = client.call("POST", "/api/v1/chats/" + chat["id"] + "/attachments?filename=oversized.png",
|
||||
202, actor=alice, raw=oversized_image(), headers={"Content-Type": "image/png"},
|
||||
schema="Attachment")["json"]
|
||||
deadline = time.monotonic() + 45
|
||||
while time.monotonic() < deadline:
|
||||
oversized = client.call("GET", "/api/v1/attachments/" + oversized["id"], actor=alice, schema="Attachment")["json"]
|
||||
if oversized["status"] in ("failed", "ready"):
|
||||
break
|
||||
time.sleep(0.5)
|
||||
check(oversized["status"] == "failed", "oversized job did not fail with the default worker limit")
|
||||
print("Worker: изображение 3000×3000 завершилось в состоянии failed.")
|
||||
bad_auth = "Basic " + base64.b64encode(("unknown:" + marker).encode()).decode()
|
||||
client.call("POST", "/api/v1/auth/sessions", 401, headers={"Authorization": bad_auth})
|
||||
forbidden = [
|
||||
marker, alice["token"], alice["registration"]["password"], chat["id"],
|
||||
message["json"]["id"], "sample.png", base64.b64encode(original).decode(),
|
||||
]
|
||||
|
||||
def metrics(expr):
|
||||
return source("prometheus", "/api/v1/query", query=expr)["data"]["result"]
|
||||
|
||||
def correlate(service, event, predicate, span_kind):
|
||||
deadline = time.monotonic() + 45
|
||||
while time.monotonic() < deadline:
|
||||
logs = source("loki", "/loki/api/v1/query_range",
|
||||
query='{service_name="' + service + '"} |= "' + event + '"',
|
||||
start=start, end=time.time_ns(), limit=100)["data"]["result"]
|
||||
for stream in logs:
|
||||
for entry in stream["values"]:
|
||||
metadata = {**stream["stream"], **(entry[2] if len(entry) > 2 else {})}
|
||||
if not predicate(metadata):
|
||||
continue
|
||||
trace_id, span_id = metadata.get("trace_id", ""), metadata.get("span_id", "")
|
||||
check(bool(re.fullmatch("[a-f0-9]{32}", trace_id)), "missing trace ID")
|
||||
check(bool(re.fullmatch("[a-f0-9]{16}", span_id)), "missing span ID")
|
||||
try:
|
||||
trace = source("tempo", "/api/traces/" + trace_id)
|
||||
except urllib.error.HTTPError as exc:
|
||||
if exc.code == 404:
|
||||
continue
|
||||
raise
|
||||
spans = [span for batch in trace["batches"] for scope in batch["scopeSpans"] for span in scope["spans"]]
|
||||
span_id_base64 = base64.b64encode(bytes.fromhex(span_id)).decode()
|
||||
matched = [span for span in spans if span["spanId"] in (span_id, span_id_base64)]
|
||||
if not matched:
|
||||
continue
|
||||
check(len(matched) == 1, "duplicate span ID in trace")
|
||||
span = matched[0]
|
||||
check(span["kind"] in (span_kind, "SPAN_KIND_SERVER" if span_kind == 2 else "SPAN_KIND_CONSUMER"), "wrong span kind")
|
||||
check(int(span["endTimeUnixNano"]) > int(span["startTimeUnixNano"]), "span duration missing")
|
||||
print(f"{service}: {event}, trace={trace_id}, span={span_id}")
|
||||
return [logs, trace]
|
||||
time.sleep(2)
|
||||
raise Failure(service + ": correlated log and span not found")
|
||||
|
||||
# Loki may include each structured-metadata value in its own result stream.
|
||||
api_data = correlate("messenger-api", "http.request.completed", lambda attrs: attrs.get("request_id") == request_id, 2)
|
||||
worker_data = correlate("messenger-worker", "job.processing.completed", lambda attrs: attrs.get("job_outcome") == "success" and attrs.get("attachment_id") == attachment["id"], 5)
|
||||
error_data = correlate("messenger-worker", "job.processing.completed",
|
||||
lambda attrs: attrs.get("job_outcome") == "error" and attrs.get("attachment_id") == oversized["id"], 5)
|
||||
expressions = [
|
||||
"messenger_http_requests_total",
|
||||
"messenger_http_duration_seconds_bucket",
|
||||
'messenger_worker_jobs_total{job_outcome="success"}',
|
||||
'messenger_worker_jobs_total{job_outcome="error"}',
|
||||
"messenger_worker_duration_seconds_bucket",
|
||||
]
|
||||
series = []
|
||||
for expression in expressions:
|
||||
deadline = time.monotonic() + 30
|
||||
samples = metrics(expression)
|
||||
while not samples and time.monotonic() < deadline:
|
||||
time.sleep(2)
|
||||
samples = metrics(expression)
|
||||
check(bool(samples), "missing metric: " + expression)
|
||||
for sample in samples:
|
||||
labels = sample["metric"]
|
||||
check(not (labels.keys() & {"user_id", "chat_id", "message_id", "attachment_id", "request_id", "url", "filename", "text"}), "unbounded labels")
|
||||
series.extend(samples)
|
||||
print(f"Prometheus: {expression}, рядов={len(samples)}")
|
||||
|
||||
captured = json.dumps([api_data, worker_data, error_data, series], ensure_ascii=False)
|
||||
check(all(value not in captured for value in forbidden), "sensitive test value in telemetry")
|
||||
print("Проверенные сигналы не содержат контрольных credentials, текста, имени/байтов файла и бизнес-ID.")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
try:
|
||||
main()
|
||||
except (Failure, OSError, ValueError, KeyError) as error:
|
||||
print("FAIL:", error)
|
||||
sys.exit(1)
|
||||
@@ -0,0 +1,171 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Локальные проверки отказов уровня 4. Результаты выводятся в терминал."""
|
||||
import argparse
|
||||
import json
|
||||
from pathlib import Path
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
import uuid
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / "tests"))
|
||||
from client import Client, Failure
|
||||
|
||||
|
||||
def compose(*args):
|
||||
subprocess.run(["docker", "compose", *args], cwd=ROOT, check=True, timeout=90)
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("incident", choices=["worker", "database", "telemetry", "retry"])
|
||||
parser.add_argument("--base-url", default="http://127.0.0.1:8080")
|
||||
parser.add_argument("--grafana-url", default="http://127.0.0.1:3000")
|
||||
args = parser.parse_args()
|
||||
client = Client(args.base_url)
|
||||
alice, _, _, chat = client.bootstrap()
|
||||
|
||||
def trace_for(request_id, matches):
|
||||
base = args.grafana_url.rstrip("/") + "/api/datasources/proxy/uid/lab6-"
|
||||
query = urllib.parse.urlencode({"query": '{service_name="messenger-api"} | request_id="' + request_id + '"'})
|
||||
deadline = time.monotonic() + 35
|
||||
while time.monotonic() < deadline:
|
||||
with urllib.request.urlopen(base + "loki/loki/api/v1/query_range?" + query, timeout=5) as response:
|
||||
logs = json.load(response)["data"]["result"]
|
||||
if logs:
|
||||
trace_id = logs[0]["stream"]["trace_id"]
|
||||
try:
|
||||
with urllib.request.urlopen(base + "tempo/api/traces/" + trace_id, timeout=5) as response:
|
||||
trace = json.load(response)
|
||||
except urllib.error.HTTPError as error:
|
||||
if error.code != 404:
|
||||
raise
|
||||
else:
|
||||
spans = [span for batch in trace["batches"] for scope in batch["scopeSpans"] for span in scope["spans"]]
|
||||
if matches(spans):
|
||||
return trace_id
|
||||
time.sleep(2)
|
||||
raise Failure("expected trace was not found")
|
||||
|
||||
def alerts(name):
|
||||
endpoint = args.grafana_url.rstrip("/") + "/api/datasources/proxy/uid/lab6-prometheus/api/v1/alerts"
|
||||
with urllib.request.urlopen(endpoint, timeout=5) as response:
|
||||
active = json.load(response)["data"]["alerts"]
|
||||
return any(item["labels"]["alertname"] == name and item["state"] == "firing" for item in active)
|
||||
|
||||
def wait_alert(name, firing, exercise=lambda: None):
|
||||
deadline = time.monotonic() + 100
|
||||
while time.monotonic() < deadline:
|
||||
exercise()
|
||||
if alerts(name) == firing:
|
||||
print(name, "firing" if firing else "resolved", flush=True)
|
||||
return
|
||||
time.sleep(2)
|
||||
raise Failure(name + ": expected alert state not observed")
|
||||
|
||||
if args.incident == "worker":
|
||||
compose("stop", "worker")
|
||||
try:
|
||||
attachment, _ = client.attachment(alice, chat)
|
||||
wait_alert("MessengerQueueLag", True)
|
||||
finally:
|
||||
compose("start", "worker")
|
||||
client.wait_ready(alice, attachment)
|
||||
wait_alert("MessengerQueueLag", False)
|
||||
elif args.incident == "database":
|
||||
# Hold a read-blocking lock on chats. The health probe can still reach PostgreSQL.
|
||||
marker = "lab6-fault-" + uuid.uuid4().hex
|
||||
command = ["docker", "compose", "exec", "-T", "db", "sh", "-c",
|
||||
'exec psql -X -v ON_ERROR_STOP=1 -U "$POSTGRES_USER" -d "$POSTGRES_DB" '
|
||||
'-c "BEGIN; LOCK TABLE chats IN ACCESS EXCLUSIVE MODE; SELECT pg_sleep(75); ROLLBACK; /* $1 */"',
|
||||
"sh", marker]
|
||||
lock = subprocess.Popen(command, cwd=ROOT, stdout=subprocess.DEVNULL)
|
||||
try:
|
||||
time.sleep(2)
|
||||
|
||||
request_ids = []
|
||||
|
||||
def exercise():
|
||||
response = client.call("GET", "/api/v1/chats/" + chat["id"], [500, 503], actor=alice)
|
||||
request_ids.append(response["headers"]["X-Request-Id"])
|
||||
|
||||
wait_alert("MessengerHighErrorRate", True, exercise)
|
||||
finally:
|
||||
# Terminate only the synthetic lock session so the transaction rolls back immediately.
|
||||
compose("exec", "-T", "db", "sh", "-c",
|
||||
'exec psql -X -U "$POSTGRES_USER" -d "$POSTGRES_DB" -c "$1"', "sh",
|
||||
"SELECT pg_terminate_backend(pid) FROM pg_stat_activity "
|
||||
"WHERE pid<>pg_backend_pid() AND query LIKE '%/* " + marker + " */'")
|
||||
lock.wait(timeout=10)
|
||||
client.call("GET", "/api/v1/chats/" + chat["id"], actor=alice, schema="Chat")
|
||||
trace_id = trace_for(request_ids[-1], lambda spans: any(
|
||||
span["name"] == "postgresql query"
|
||||
and int(span["endTimeUnixNano"]) - int(span["startTimeUnixNano"]) > 1_000_000_000 for span in spans))
|
||||
print("Медленный запрос PostgreSQL подтверждён в trace:", trace_id, flush=True)
|
||||
wait_alert("MessengerHighErrorRate", False)
|
||||
elif args.incident == "retry":
|
||||
compose("stop", "worker")
|
||||
try:
|
||||
upload = client.call("POST", "/api/v1/chats/" + chat["id"] + "/attachments?filename=sample.png",
|
||||
202, actor=alice, raw=(ROOT / "tests/fixtures/sample.png").read_bytes(),
|
||||
headers={"Content-Type": "image/png"}, schema="Attachment")
|
||||
attachment = upload["json"]
|
||||
compose("stop", "s3")
|
||||
try:
|
||||
compose("up", "-d", "--no-deps", "worker")
|
||||
deadline = time.monotonic() + 25
|
||||
while time.monotonic() < deadline:
|
||||
sql = ("SELECT count(*) FROM attachment_job_attempts a JOIN attachment_jobs j ON j.id=a.job_id "
|
||||
"WHERE a.next_attempt_at IS NOT NULL AND j.attachment_id='" + str(uuid.UUID(attachment["id"])) + "'")
|
||||
output = subprocess.check_output(
|
||||
["docker", "compose", "exec", "-T", "db", "sh", "-c",
|
||||
'exec psql -X -At -U "$POSTGRES_USER" -d "$POSTGRES_DB" -c "$1"', "sh", sql],
|
||||
cwd=ROOT, text=True, timeout=10)
|
||||
if int(output.strip()) > 0:
|
||||
break
|
||||
time.sleep(1)
|
||||
else:
|
||||
raise Failure("durable retry was not observed")
|
||||
finally:
|
||||
compose("up", "-d", "--wait", "s3")
|
||||
client.wait_ready(alice, attachment)
|
||||
def retried(spans):
|
||||
attempts = [a["value"]["intValue"] for span in spans if span["kind"] == "SPAN_KIND_CONSUMER"
|
||||
for a in span.get("attributes", []) if a["key"] == "job.attempt"]
|
||||
return len(set(attempts)) >= 2
|
||||
|
||||
trace_id = trace_for(upload["headers"]["X-Request-Id"], retried)
|
||||
print("Исходное вложение перешло в ready, обе попытки в trace:", trace_id, flush=True)
|
||||
finally:
|
||||
compose("start", "worker")
|
||||
else:
|
||||
since = str(int(time.time()))
|
||||
compose("stop", "tempo")
|
||||
try:
|
||||
for _ in range(8):
|
||||
client.message(alice, chat, "telemetry-backend-off")
|
||||
time.sleep(5)
|
||||
attachment, _ = client.attachment(alice, chat)
|
||||
client.wait_ready(alice, attachment)
|
||||
compose("stats", "--no-stream", "otel-collector")
|
||||
print("API и worker работали при остановленном Tempo.", flush=True)
|
||||
finally:
|
||||
compose("start", "tempo")
|
||||
output = subprocess.check_output(
|
||||
["docker", "compose", "logs", "--no-color", "--since", since, "otel-collector"],
|
||||
cwd=ROOT, text=True, timeout=15)
|
||||
dropped = sum(int(match) for line in output.splitlines() if '"otlphttp/tempo"' in line
|
||||
for match in re.findall(r'"dropped_items":\s*(\d+)', line))
|
||||
print("Спанов отброшено Collector за период проверки:", dropped, flush=True)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
try:
|
||||
main()
|
||||
except (Failure, OSError, subprocess.SubprocessError, ValueError) as error:
|
||||
print("FAIL:", error, file=sys.stderr)
|
||||
sys.exit(1)
|
||||
@@ -0,0 +1,6 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Совместимая точка входа для итогового генератора."""
|
||||
from generate_evidence import main
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,103 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Сбор воспроизводимых доказательств уровней 3–4."""
|
||||
import argparse
|
||||
from datetime import datetime, timezone
|
||||
import json
|
||||
from pathlib import Path
|
||||
import re
|
||||
import shlex
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--base-url", default="http://127.0.0.1:8080")
|
||||
parser.add_argument("--grafana-url", default="http://127.0.0.1:3000")
|
||||
args = parser.parse_args()
|
||||
probe = [sys.executable, "scripts/check_observability.py", "--base-url", args.base_url,
|
||||
"--grafana-url", args.grafana_url]
|
||||
incident = [sys.executable, "scripts/check_observability_incidents.py", "--base-url", args.base_url,
|
||||
"--grafana-url", args.grafana_url]
|
||||
checks = [
|
||||
("grade3-stand", "Стенд, тома и provisioning",
|
||||
[["docker", "compose", "config", "--quiet"],
|
||||
["docker", "compose", "ps", "--all"],
|
||||
["docker", "compose", "config", "--volumes"],
|
||||
["docker", "compose", "run", "--rm", "--no-deps", "otel-collector", "validate", "--config=/etc/otelcol-contrib/config.yaml"],
|
||||
probe + ["provisioning"]],
|
||||
"Prometheus хранит временные ряды, Loki — журналы, Tempo — трассы. Collector принимает и маршрутизирует OTLP. Grafana читает данные и загружает dashboard из JSON."),
|
||||
("grade3-smoke", "Совместимость API при включённой телеметрии",
|
||||
[["make", "check"], ["make", "test", "BASE_URL=" + args.base_url]],
|
||||
"Smoke проверяет сессии, сообщения, ACL, идемпотентность и готовность вложения с оригиналом и thumbnail."),
|
||||
("grade3-signals", "Метрики, структурные логи и трассы API и worker",
|
||||
[probe + ["signals"]],
|
||||
"Контрольный POST сопоставляется с логом по X-Request-Id. Оба сервиса сопоставляются с Tempo по trace_id и span_id. Prometheus возвращает счётчики и histogram buckets. Проверка ищет утечки синтетических значений в полученных сигналах."),
|
||||
("grade4-pipeline", "Сквозная трасса и повторная попытка",
|
||||
[probe + ["pipeline"], incident + ["retry"]],
|
||||
"Tempo подтверждает родительскую связь HTTP, producer и consumer, операции S3 и PostgreSQL и две попытки одной загрузки."),
|
||||
("grade4-dashboard", "Метрики диагностики и SLO",
|
||||
[probe + ["dashboard"], ["docker", "compose", "exec", "-T", "prometheus", "promtool", "check", "rules", "/etc/prometheus/alerts.yaml"]],
|
||||
"Проверяются запросы панелей, SLI за 30 минут и синтаксис двух правил."),
|
||||
("grade4-queue", "Остановка и восстановление worker",
|
||||
[incident + ["worker"]],
|
||||
"Возраст незавершённой работы вызывает firing. Вложение достигает ready после восстановления worker, алерт переходит в resolved."),
|
||||
("grade4-database", "Задержка PostgreSQL и восстановление",
|
||||
[incident + ["database"]],
|
||||
"При блокировке таблицы растёт доля 5xx и алерт срабатывает. Задержку показывает дочерний спан PostgreSQL. После снятия блокировки алерт переходит в resolved."),
|
||||
("grade4-telemetry", "Недоступность Tempo",
|
||||
[incident + ["telemetry"]],
|
||||
"Проверяются сообщения и готовность вложения при остановленном Tempo, снимок памяти Collector и число отброшенных спанов по его журналу."),
|
||||
("grade4-load", "Короткая нагрузка на API",
|
||||
[["lscpu"], ["free", "-h"], ["docker", "compose", "version"],
|
||||
[sys.executable, "tests/load.py", "--base-url", args.base_url, "--duration", "30", "--concurrency", "4"],
|
||||
["docker", "compose", "stats", "--no-stream"]],
|
||||
"Один closed-loop прогон POST сообщений с четырьмя клиентами, отдельно от инцидентов. Снимок ресурсов сделан сразу после прогона."),
|
||||
]
|
||||
output_root = ROOT / "evidence/criteria"
|
||||
stamp = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%S.%fZ")
|
||||
run = output_root / "runs" / stamp
|
||||
run.mkdir(parents=True)
|
||||
summary = {}
|
||||
failed = False
|
||||
for name, title, commands, explanation in checks:
|
||||
lines = [f"# {title}", "", "Предпосылки — локальный синтетический стенд после make app-up и доступ к Docker.",
|
||||
"", explanation, ""]
|
||||
results = []
|
||||
summary[name] = results
|
||||
for index, command in enumerate(commands, 1):
|
||||
started = time.monotonic()
|
||||
try:
|
||||
result = subprocess.run(command, cwd=ROOT, stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
|
||||
text=True, timeout=240)
|
||||
code, output = result.returncode, result.stdout
|
||||
except subprocess.TimeoutExpired:
|
||||
code, output = 124, "Истёк лимит времени."
|
||||
except OSError:
|
||||
code, output = 127, "Команда недоступна."
|
||||
failed |= code != 0
|
||||
output = re.sub(r"(https?://[^\s?]+)\?[^\s]+", r"\1?[redacted]", output).strip()
|
||||
output = "\n".join(line.rstrip() for line in output.splitlines())
|
||||
raw = f"{name}-{index}.txt"
|
||||
(run / raw).write_text(output + "\n", encoding="utf-8")
|
||||
seconds = round(time.monotonic() - started, 2)
|
||||
results.append(dict(command=shlex.join(command), exit_code=code, seconds=seconds, output=raw))
|
||||
lines.extend([f"Команда `{shlex.join(command)}`. Код выхода **{code}**. Время {seconds} с.",
|
||||
"", f"[Вывод](runs/{stamp}/{raw})", "", "````text", output or "(пусто)", "````", ""])
|
||||
path = output_root / (name + ".md")
|
||||
path.write_text("\n".join(lines), encoding="utf-8")
|
||||
print(path.relative_to(ROOT), flush=True)
|
||||
index = ["# Доказательства критериев", "", f"[Сводка прогона {stamp}](runs/{stamp}/summary.json)", ""]
|
||||
for name, results in summary.items():
|
||||
state = "PASS" if all(result["exit_code"] == 0 for result in results) else "FAIL"
|
||||
index.append(f"- [{name}]({name}.md) — {state}")
|
||||
(output_root / "README.md").write_text("\n".join(index) + "\n", encoding="utf-8")
|
||||
(run / "summary.json").write_text(json.dumps(summary, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||||
return int(failed)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,122 @@
|
||||
import os
|
||||
import secrets
|
||||
from pathlib import Path
|
||||
from urllib.parse import quote
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
|
||||
|
||||
def read_values(path):
|
||||
values = {}
|
||||
if path.exists():
|
||||
for line in path.read_text(encoding="utf-8").splitlines():
|
||||
line = line.strip()
|
||||
if line and not line.startswith("#") and "=" in line:
|
||||
key, value = line.split("=", 1)
|
||||
values[key.strip()] = value.strip().strip("\"'")
|
||||
return values
|
||||
|
||||
|
||||
def main():
|
||||
path = ROOT / ".env"
|
||||
existing = read_values(path)
|
||||
defaults = read_values(ROOT / ".env.example")
|
||||
defaults.update(
|
||||
POSTGRES_DB="messenger",
|
||||
POSTGRES_USER="messenger",
|
||||
POSTGRES_PASSWORD=secrets.token_hex(24),
|
||||
)
|
||||
defaults.update(API_HTTP_PORT="8080", UPLOAD_CLEANUP_GRACE_SECONDS="600", UPLOAD_TMP_DIR="/tmp/uploads",
|
||||
S3_PUBLIC_ENDPOINT="http://127.0.0.1:9000",
|
||||
S3_CORS_ORIGINS="http://127.0.0.1:5173,http://localhost:5173")
|
||||
values = defaults | existing
|
||||
for name, lower, upper in (
|
||||
("APP_PORT", 1, 65535),
|
||||
("API_REPLICAS", 2, 32),
|
||||
("SESSION_TTL_SECONDS", 1, 31536000),
|
||||
("IDEMPOTENCY_TTL_SECONDS", 1, 31536000),
|
||||
):
|
||||
try:
|
||||
valid = lower <= int(values[name]) <= upper
|
||||
except (ValueError, KeyError):
|
||||
valid = False
|
||||
if not valid:
|
||||
raise SystemExit(f"{name} must be between {lower} and {upper}")
|
||||
for name in (
|
||||
"POSTGRES_DB",
|
||||
"POSTGRES_USER",
|
||||
"POSTGRES_PASSWORD",
|
||||
"APP_HOST",
|
||||
"APP_ENV",
|
||||
):
|
||||
if not values[name]:
|
||||
raise SystemExit(f"{name} must not be empty")
|
||||
# Existing values, including credentials, are never replaced.
|
||||
missing = {key: value for key, value in values.items() if key not in existing}
|
||||
if missing:
|
||||
prefix = "\n" if path.exists() and path.stat().st_size else ""
|
||||
with path.open("a", encoding="utf-8") as file:
|
||||
file.write(
|
||||
prefix + "".join(f"{key}={value}\n" for key, value in missing.items())
|
||||
)
|
||||
os.chmod(path, 0o600)
|
||||
secret_dir = ROOT / "secrets"
|
||||
secret_dir.mkdir(mode=0o700, exist_ok=True)
|
||||
database_url = "postgres://{}:{}@db:5432/{}?sslmode=disable".format(
|
||||
quote(values["POSTGRES_USER"], safe=""),
|
||||
quote(values["POSTGRES_PASSWORD"], safe=""),
|
||||
quote(values["POSTGRES_DB"], safe=""),
|
||||
)
|
||||
for filename,content in (("broker_user","messenger"),("broker_password",secrets.token_hex(32))):
|
||||
target=secret_dir/filename
|
||||
if not target.exists():
|
||||
target.write_text(content+"\n",encoding="utf-8")
|
||||
os.chmod(target,0o644)
|
||||
broker_url="amqp://{}:{}@broker:5672/%2F".format(
|
||||
quote((secret_dir/"broker_user").read_text().strip(),safe=""),
|
||||
quote((secret_dir/"broker_password").read_text().strip(),safe=""))
|
||||
# Migrate only the generated legacy Redis URL; preserve custom AMQP settings.
|
||||
broker_file=secret_dir/"broker_url"
|
||||
if broker_file.exists() and broker_file.read_text().strip()=="redis://broker:6379/0":
|
||||
broker_file.write_text(broker_url+"\n",encoding="utf-8")
|
||||
for name, content in (
|
||||
("DATABASE_URL_FILE", database_url),
|
||||
("CACHE_URL_FILE", "redis://redis:6379/0"),
|
||||
("BROKER_URL_FILE", broker_url),
|
||||
("S3_ACCESS_KEY_ID_FILE", secrets.token_hex(12)),
|
||||
("S3_SECRET_ACCESS_KEY_FILE", secrets.token_hex(32)),
|
||||
):
|
||||
container_path = Path(values[name])
|
||||
if container_path.parent != Path("/run/secrets") or not container_path.name:
|
||||
raise SystemExit(f"{name} must name a file directly inside /run/secrets")
|
||||
target = (
|
||||
secret_dir
|
||||
/ {
|
||||
"DATABASE_URL_FILE": "database_url",
|
||||
"CACHE_URL_FILE": "cache_url",
|
||||
"BROKER_URL_FILE": "broker_url",
|
||||
"S3_ACCESS_KEY_ID_FILE": "s3_access_key_id",
|
||||
"S3_SECRET_ACCESS_KEY_FILE": "s3_secret_access_key",
|
||||
}[name]
|
||||
)
|
||||
if not target.exists():
|
||||
# Containers run as an unprivileged user; the private host directory
|
||||
# protects the files, while read permission lets the API consume them.
|
||||
with target.open("x", encoding="utf-8") as file:
|
||||
file.write(content + "\n")
|
||||
os.chmod(target, 0o644)
|
||||
# Existing S3 files remain the root credentials; API gets a new, separate identity.
|
||||
for filename, content in (
|
||||
("s3_api_access_key_id", secrets.token_hex(12)),
|
||||
("s3_api_secret_access_key", secrets.token_hex(32)),
|
||||
):
|
||||
target = secret_dir / filename
|
||||
if not target.exists():
|
||||
with target.open("x", encoding="utf-8") as file:
|
||||
file.write(content + "\n")
|
||||
os.chmod(target, 0o644)
|
||||
print("Local .env and connection files ready; existing values and files preserved.")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Executable
+7
@@ -0,0 +1,7 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
docker compose --project-name highload-lab6 --env-file .env -f compose.yml restart db redis s3 broker >/dev/null
|
||||
docker compose --project-name highload-lab6 --env-file .env -f compose.yml up -d --wait db redis s3 broker >/dev/null
|
||||
docker compose --project-name highload-lab6 --env-file .env -f compose.yml restart api upload-cleanup worker dispatcher >/dev/null
|
||||
docker compose --project-name highload-lab6 --env-file .env -f compose.yml up -d --wait >/dev/null
|
||||
Executable
+16
@@ -0,0 +1,16 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
|
||||
: "${TARGET_INSTANCE_ID:?Set TARGET_INSTANCE_ID to the X-Instance-Id of an API replica}"
|
||||
for container in $(docker compose --project-name highload-lab6 --env-file .env -f compose.yml ps -q api); do
|
||||
hostname=$(docker inspect --format '{{.Config.Hostname}}' "$container")
|
||||
if [ "$hostname" = "$TARGET_INSTANCE_ID" ]; then
|
||||
# Only containers from this Compose project's api service are eligible.
|
||||
docker stop --timeout 8 "$container" >/dev/null
|
||||
printf 'Stopped API instance %s\n' "$hostname"
|
||||
exit 0
|
||||
fi
|
||||
done
|
||||
printf 'No running API replica matches TARGET_INSTANCE_ID\n' >&2
|
||||
exit 1
|
||||
@@ -0,0 +1,26 @@
|
||||
.git
|
||||
.gitignore
|
||||
.github
|
||||
.env
|
||||
.env.*
|
||||
!.env.example
|
||||
secrets
|
||||
certs
|
||||
volumes
|
||||
data
|
||||
backups
|
||||
*.key
|
||||
*.pem
|
||||
*.p12
|
||||
*.pfx
|
||||
*.jks
|
||||
.venv
|
||||
__pycache__
|
||||
node_modules
|
||||
evidence
|
||||
.DS_Store
|
||||
README.md
|
||||
REPORT.md
|
||||
COURSE.md
|
||||
CONTRIBUTING.md
|
||||
MAINTAINERS.md
|
||||
@@ -0,0 +1,42 @@
|
||||
FROM golang:1.22.2-bookworm AS builder
|
||||
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends webp && rm -rf /var/lib/apt/lists/*
|
||||
|
||||
WORKDIR /src
|
||||
|
||||
COPY go.mod go.sum ./
|
||||
|
||||
RUN go mod download
|
||||
|
||||
COPY backend ./backend
|
||||
|
||||
RUN CGO_ENABLED=0 \
|
||||
GOOS=linux \
|
||||
go build \
|
||||
-trimpath \
|
||||
-ldflags="-s -w" \
|
||||
-o /out/messenger \
|
||||
./backend
|
||||
|
||||
|
||||
FROM debian:12.15-slim
|
||||
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends webp && rm -rf /var/lib/apt/lists/* \
|
||||
&& groupadd --system app \
|
||||
&& useradd --system --gid app --no-create-home app
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
COPY --from=builder /out/messenger ./messenger
|
||||
|
||||
ENV APP_HOST=0.0.0.0
|
||||
ENV APP_PORT=8080
|
||||
|
||||
EXPOSE 8080
|
||||
|
||||
HEALTHCHECK --interval=10s --timeout=3s --start-period=5s --retries=3 \
|
||||
CMD ["./messenger", "healthcheck"]
|
||||
|
||||
USER app:app
|
||||
|
||||
CMD ["./messenger"]
|
||||
@@ -0,0 +1,150 @@
|
||||
package app
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log"
|
||||
"net"
|
||||
"net/http"
|
||||
"os"
|
||||
"os/signal"
|
||||
"runtime"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"messenger/src/backend/handlers"
|
||||
"messenger/src/backend/repositories"
|
||||
"messenger/src/backend/services"
|
||||
"messenger/src/backend/telemetry"
|
||||
)
|
||||
|
||||
func serverAddress() string {
|
||||
host := os.Getenv("APP_HOST")
|
||||
if host == "" {
|
||||
host = "0.0.0.0"
|
||||
}
|
||||
|
||||
port := os.Getenv("APP_PORT")
|
||||
if port == "" {
|
||||
port = "8080"
|
||||
}
|
||||
|
||||
return net.JoinHostPort(host, port)
|
||||
}
|
||||
|
||||
func RunHTTP() error {
|
||||
if _, err := repositories.UploadCleanupGrace(); err != nil {
|
||||
return err
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
|
||||
db, err := repositories.OpenDatabase(ctx)
|
||||
cancel()
|
||||
if err != nil {
|
||||
return fmt.Errorf("database initialization failed: %w", err)
|
||||
}
|
||||
defer db.Close()
|
||||
// Opt-in experiment telemetry goes to container logs, never a public port.
|
||||
if os.Getenv("BENCH_POOL_STATS") == "1" {
|
||||
go func() {
|
||||
for range time.Tick(time.Second) {
|
||||
s := db.Stat()
|
||||
data, _ := json.Marshal(map[string]any{
|
||||
"time": time.Now().UTC(), "acquired": s.AcquiredConns(),
|
||||
"total": s.TotalConns(), "max": s.MaxConns(),
|
||||
"gomaxprocs": runtime.GOMAXPROCS(0), "goroutines": runtime.NumGoroutine(),
|
||||
"acquire_count": s.AcquireCount(), "acquire_duration_ns": s.AcquireDuration().Nanoseconds(),
|
||||
"empty_acquire_count": s.EmptyAcquireCount(), "canceled_acquire_count": s.CanceledAcquireCount(),
|
||||
})
|
||||
log.Printf("benchmark_pool %s", data)
|
||||
}
|
||||
}()
|
||||
}
|
||||
redisClient, err := repositories.RedisClientFromEnvironment()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer redisClient.Close()
|
||||
sessionStore := repositories.NewSessionStore(redisClient, "messenger:session:")
|
||||
userStore := repositories.NewUserStore(db)
|
||||
chatStore := repositories.NewChatStore(db)
|
||||
idempotencyTTL, err := repositories.IdempotencyTTL()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
messageStore := repositories.NewMessageStoreWithTTL(db, idempotencyTTL)
|
||||
ttl, err := services.SessionTTL()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
userService := services.NewUserService(userStore)
|
||||
sessionService := services.NewSessionService(userService, sessionStore, ttl)
|
||||
chatService := services.NewChatService(chatStore)
|
||||
messageService := services.NewMessageService(messageStore)
|
||||
objects, err := repositories.NewObjectStoreFromEnvironment()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
attachmentService := services.NewAttachmentService(repositories.NewAttachmentStore(db), objects)
|
||||
mux := handlers.NewRouter(services.NewHealthService(repositories.NewHealthStore(db), sessionStore, objects), userService, sessionService, chatService, messageService, attachmentService)
|
||||
|
||||
signals, stopTelemetry, err := telemetry.Start("messenger-api")
|
||||
if err != nil {
|
||||
return fmt.Errorf("telemetry initialization failed: %w", err)
|
||||
}
|
||||
defer stopTelemetry()
|
||||
if err := signals.ObservePool(db); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
server := &http.Server{
|
||||
Addr: serverAddress(),
|
||||
Handler: signals.HTTP(mux),
|
||||
ReadHeaderTimeout: 5 * time.Second,
|
||||
ReadTimeout: 30 * time.Second,
|
||||
WriteTimeout: 30 * time.Second,
|
||||
IdleTimeout: 60 * time.Second,
|
||||
}
|
||||
|
||||
shutdownContext, stop := signal.NotifyContext(
|
||||
context.Background(),
|
||||
os.Interrupt,
|
||||
syscall.SIGTERM,
|
||||
)
|
||||
defer stop()
|
||||
|
||||
serverErrors := make(chan error, 1)
|
||||
|
||||
go func() {
|
||||
log.Printf("server is listening on %s", server.Addr)
|
||||
|
||||
if err := server.ListenAndServe(); err != nil &&
|
||||
!errors.Is(err, http.ErrServerClosed) {
|
||||
serverErrors <- err
|
||||
}
|
||||
}()
|
||||
|
||||
select {
|
||||
case err := <-serverErrors:
|
||||
log.Printf("server error: %v", err)
|
||||
|
||||
case <-shutdownContext.Done():
|
||||
log.Printf("shutdown signal received")
|
||||
handlers.BeginShutdown()
|
||||
|
||||
shutdownDeadline, cancel := context.WithTimeout(
|
||||
context.Background(),
|
||||
10*time.Second,
|
||||
)
|
||||
defer cancel()
|
||||
|
||||
if err := server.Shutdown(shutdownDeadline); err != nil {
|
||||
log.Printf("graceful shutdown failed: %v", err)
|
||||
return nil
|
||||
}
|
||||
|
||||
log.Printf("server stopped gracefully")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,64 @@
|
||||
package commands
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"flag"
|
||||
"log"
|
||||
"os"
|
||||
"os/signal"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"messenger/src/backend/repositories"
|
||||
)
|
||||
|
||||
func runUploadCleanup(args []string) error {
|
||||
flags := flag.NewFlagSet("attachments-cleanup", flag.ContinueOnError)
|
||||
loop := flags.Bool("loop", false, "run every 60 seconds")
|
||||
dryRun := flags.Bool("dry-run", false, "report without deleting anything")
|
||||
if err := flags.Parse(args); err != nil {
|
||||
return err
|
||||
}
|
||||
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
|
||||
defer stop()
|
||||
init, cancel := context.WithTimeout(ctx, 20*time.Second)
|
||||
db, err := repositories.OpenDatabase(init)
|
||||
cancel()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer db.Close()
|
||||
objects, err := repositories.NewObjectStoreFromEnvironment()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
store := repositories.NewAttachmentStore(db)
|
||||
for {
|
||||
pass, cancel := context.WithTimeout(ctx, 30*time.Second)
|
||||
result, err := store.CleanupUploads(pass, *dryRun, objects.Delete)
|
||||
direct, directErr := store.CleanupDirectUploads(pass, *dryRun, objects.Delete)
|
||||
jobs, jobsErr := store.CleanupJobs(pass, 24*time.Hour, *dryRun, objects.Delete)
|
||||
cancel()
|
||||
if encodeErr := json.NewEncoder(os.Stdout).Encode(map[string]any{"originals": result, "direct": direct, "job_staging": jobs}); encodeErr != nil {
|
||||
return encodeErr
|
||||
}
|
||||
if !*loop {
|
||||
if err == nil {
|
||||
err = directErr
|
||||
if err == nil {
|
||||
err = jobsErr
|
||||
}
|
||||
}
|
||||
return err
|
||||
}
|
||||
if err != nil || directErr != nil || jobsErr != nil {
|
||||
log.Print("upload cleanup pass failed; journal retained for retry")
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return nil
|
||||
case <-time.After(60 * time.Second):
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,76 @@
|
||||
package commands
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"os"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
|
||||
"messenger/src/backend/repositories"
|
||||
)
|
||||
|
||||
func runJobAdmin(args []string) error {
|
||||
if len(args) < 1 {
|
||||
return errors.New("usage: jobs list-failed | replay JOB_ID REASON | cleanup-staging JOB_ID | upgrade ATTACHMENT_ID 2 REASON")
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
|
||||
defer cancel()
|
||||
db, err := repositories.OpenDatabase(ctx)
|
||||
if err != nil {
|
||||
return errors.New("database unavailable")
|
||||
}
|
||||
defer db.Close()
|
||||
store := repositories.NewAttachmentStore(db)
|
||||
if args[0] == "list-failed" && len(args) == 1 {
|
||||
jobs, e := store.FailedJobs(ctx)
|
||||
if e != nil {
|
||||
return errors.New("cannot read failed registry")
|
||||
}
|
||||
return json.NewEncoder(os.Stdout).Encode(jobs)
|
||||
}
|
||||
if len(args) < 2 {
|
||||
return errors.New("job id required")
|
||||
}
|
||||
if _, err = uuid.Parse(args[1]); err != nil {
|
||||
return errors.New("invalid job id")
|
||||
}
|
||||
objects, err := repositories.NewObjectStoreFromEnvironment()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var changed bool
|
||||
switch {
|
||||
case args[0] == "upgrade" && len(args) == 4:
|
||||
version, e := strconv.Atoi(args[2])
|
||||
if e != nil {
|
||||
return errors.New("invalid pipeline version")
|
||||
}
|
||||
changed, err = store.UpgradePipeline(ctx, args[1], version, args[3], func(ctx context.Context, key string) error {
|
||||
r, e := objects.Get(ctx, key)
|
||||
if e == nil {
|
||||
r.Close()
|
||||
}
|
||||
return e
|
||||
})
|
||||
case args[0] == "replay" && len(args) == 3:
|
||||
changed, err = store.ReplayJob(ctx, args[1], args[2], func(ctx context.Context, key string) error {
|
||||
r, e := objects.Get(ctx, key)
|
||||
if e == nil {
|
||||
r.Close()
|
||||
}
|
||||
return e
|
||||
})
|
||||
case args[0] == "cleanup-staging" && len(args) == 2:
|
||||
changed, err = store.CleanupJobStaging(ctx, args[1], 24*time.Hour, false, objects.Delete)
|
||||
default:
|
||||
return errors.New("usage: jobs list-failed | replay JOB_ID REASON | cleanup-staging JOB_ID | upgrade ATTACHMENT_ID 2 REASON")
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return json.NewEncoder(os.Stdout).Encode(map[string]bool{"changed": changed})
|
||||
}
|
||||
@@ -0,0 +1,79 @@
|
||||
package commands
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"log"
|
||||
"net"
|
||||
"net/http"
|
||||
"os"
|
||||
"time"
|
||||
|
||||
"messenger/src/backend/app"
|
||||
"messenger/src/backend/repositories"
|
||||
"messenger/src/backend/workers"
|
||||
)
|
||||
|
||||
// Run dispatches the existing executable modes; the binary and CLI stay the same.
|
||||
func Run(args []string) error {
|
||||
if len(args) == 0 {
|
||||
if environment := os.Getenv("APP_ENV"); environment != "" {
|
||||
log.SetPrefix("[" + environment + "] ")
|
||||
}
|
||||
return app.RunHTTP()
|
||||
}
|
||||
switch args[0] {
|
||||
case "image-process":
|
||||
return workers.RunImageChild(args[1:])
|
||||
case "jobs":
|
||||
return runJobAdmin(args[1:])
|
||||
case "worker", "dispatcher", "broker-publish":
|
||||
return workers.RunPipeline(args[0], args[1:])
|
||||
}
|
||||
if environment := os.Getenv("APP_ENV"); environment != "" {
|
||||
log.SetPrefix("[" + environment + "] ")
|
||||
}
|
||||
switch {
|
||||
case args[0] == "s3-bootstrap" && len(args) == 1:
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
if err := repositories.BootstrapS3(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
log.Print("private S3 bucket ready")
|
||||
return nil
|
||||
case args[0] == "attachments-cleanup":
|
||||
return runUploadCleanup(args[1:])
|
||||
case args[0] == "healthcheck" && len(args) == 1:
|
||||
return healthcheck()
|
||||
case args[0] == "migrate" && len(args) == 2 && (args[1] == "up" || args[1] == "down"):
|
||||
url, err := repositories.DatabaseURL()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err = repositories.RunMigrations(url, args[1]); err != nil {
|
||||
return err
|
||||
}
|
||||
log.Printf("migration %s complete", args[1])
|
||||
return nil
|
||||
default:
|
||||
return errors.New("usage: messenger migrate up|down")
|
||||
}
|
||||
}
|
||||
|
||||
func healthcheck() error {
|
||||
client := http.Client{Timeout: 2 * time.Second}
|
||||
port := os.Getenv("APP_PORT")
|
||||
if port == "" {
|
||||
port = "8080"
|
||||
}
|
||||
response, err := client.Get("http://" + net.JoinHostPort("127.0.0.1", port) + "/health/ready")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
response.Body.Close()
|
||||
if response.StatusCode != http.StatusOK {
|
||||
return errors.New("API is not ready")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,165 @@
|
||||
package handlers
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"github.com/google/uuid"
|
||||
"io"
|
||||
"log"
|
||||
"messenger/src/backend/repositories"
|
||||
"messenger/src/backend/services"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
func uploadAttachmentHandler(chats *services.ChatService, attachments *services.AttachmentService) http.HandlerFunc {
|
||||
return func(w http.ResponseWriter, r *http.Request) {
|
||||
requestID := newRequestID()
|
||||
w.Header().Set("X-Request-Id", requestID)
|
||||
if r.Method != http.MethodPost {
|
||||
writeError(w, 405, "method_not_allowed", "method not allowed", requestID)
|
||||
return
|
||||
}
|
||||
userID, ok := requestUserID(w, r, requestID)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
raw := strings.TrimSuffix(strings.TrimPrefix(r.URL.Path, "/api/v1/chats/"), "/attachments")
|
||||
chatID, err := uuid.Parse(raw)
|
||||
if err != nil {
|
||||
writeError(w, 400, "invalid_request", "invalid chat id", requestID)
|
||||
return
|
||||
}
|
||||
if _, err = chats.Get(r.Context(), userID, chatID.String()); err != nil {
|
||||
writeServiceError(w, err, requestID)
|
||||
return
|
||||
}
|
||||
query, err := urlQuery(r)
|
||||
if err != nil {
|
||||
writeServiceError(w, err, requestID)
|
||||
return
|
||||
}
|
||||
filenames := query["filename"]
|
||||
if len(filenames) != 1 {
|
||||
writeError(w, 400, "invalid_request", "exactly one filename is required", requestID)
|
||||
return
|
||||
}
|
||||
if err = services.ValidateFilename(filenames[0]); err != nil {
|
||||
writeServiceError(w, err, requestID)
|
||||
return
|
||||
}
|
||||
if r.ContentLength > services.MaxUploadBytes {
|
||||
writeServiceError(w, services.ErrPayloadTooLarge, requestID)
|
||||
return
|
||||
}
|
||||
r.Body = http.MaxBytesReader(w, r.Body, services.MaxUploadBytes)
|
||||
ctx, cancel := context.WithTimeout(r.Context(), 25*time.Second)
|
||||
defer cancel()
|
||||
attachment, err := attachments.Upload(ctx, userID, chatID.String(), filenames[0], r.Header.Get("Content-Type"), r.Body)
|
||||
var tooLarge *http.MaxBytesError
|
||||
if errors.As(err, &tooLarge) {
|
||||
err = services.ErrPayloadTooLarge
|
||||
}
|
||||
if err != nil {
|
||||
writeServiceError(w, err, requestID)
|
||||
return
|
||||
}
|
||||
writeJSON(w, 202, attachment)
|
||||
}
|
||||
}
|
||||
|
||||
func attachmentHandler(attachments *services.AttachmentService) http.HandlerFunc {
|
||||
return func(w http.ResponseWriter, r *http.Request) {
|
||||
requestID := newRequestID()
|
||||
w.Header().Set("X-Request-Id", requestID)
|
||||
if r.Method != http.MethodGet {
|
||||
writeError(w, 405, "method_not_allowed", "method not allowed", requestID)
|
||||
return
|
||||
}
|
||||
userID, ok := requestUserID(w, r, requestID)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
path := strings.TrimPrefix(r.URL.Path, "/api/v1/attachments/")
|
||||
parts := strings.Split(path, "/")
|
||||
if len(parts) > 2 || (len(parts) == 2 && parts[1] != "content") {
|
||||
writeServiceError(w, services.ErrAttachmentNotFound, requestID)
|
||||
return
|
||||
}
|
||||
id, err := uuid.Parse(parts[0])
|
||||
if err != nil {
|
||||
writeError(w, 400, "invalid_request", "invalid attachment id", requestID)
|
||||
return
|
||||
}
|
||||
a, err := attachments.Get(r.Context(), id.String(), userID)
|
||||
if err != nil {
|
||||
writeServiceError(w, err, requestID)
|
||||
return
|
||||
}
|
||||
if len(parts) == 1 {
|
||||
writeJSON(w, 200, a)
|
||||
return
|
||||
}
|
||||
query, err := urlQuery(r)
|
||||
if err != nil {
|
||||
writeServiceError(w, err, requestID)
|
||||
return
|
||||
}
|
||||
if a.Status != "ready" {
|
||||
writeServiceError(w, services.ErrConflict, requestID)
|
||||
return
|
||||
}
|
||||
if variants, exists := query["variant"]; exists {
|
||||
if len(variants) != 1 {
|
||||
writeError(w, 400, "invalid_request", "invalid variant", requestID)
|
||||
return
|
||||
}
|
||||
if variants[0] == "thumbnail" || variants[0] == "crop" || variants[0] == "optimized" {
|
||||
found := false
|
||||
for _, raw := range a.Variants {
|
||||
if v, ok := raw.(map[string]any); ok && v["name"] == variants[0] {
|
||||
a.ObjectKey = repositories.VariantObjectKey(a.ID, a.PipelineVersion, variants[0])
|
||||
a.ContentType = v["content_type"].(string)
|
||||
a.SizeBytes = int64(v["size_bytes"].(float64))
|
||||
found = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
writeServiceError(w, services.ErrAttachmentNotFound, requestID)
|
||||
return
|
||||
}
|
||||
} else if variants[0] != "original" {
|
||||
writeServiceError(w, services.ErrAttachmentNotFound, requestID)
|
||||
return
|
||||
}
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(r.Context(), 25*time.Second)
|
||||
defer cancel()
|
||||
content, err := attachments.Content(ctx, a)
|
||||
if err != nil {
|
||||
writeServiceError(w, err, requestID)
|
||||
return
|
||||
}
|
||||
defer content.Close()
|
||||
w.Header().Set("Content-Type", a.ContentType)
|
||||
w.Header().Set("Content-Length", strconv.FormatInt(a.SizeBytes, 10))
|
||||
w.Header().Set("X-Content-Type-Options", "nosniff")
|
||||
w.Header().Set("Cache-Control", "private, no-store")
|
||||
w.WriteHeader(200)
|
||||
if _, err = io.Copy(w, content); err != nil {
|
||||
log.Printf("attachment download interrupted, request %s", requestID)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func urlQuery(r *http.Request) (url.Values, error) {
|
||||
q, err := url.ParseQuery(r.URL.RawQuery)
|
||||
if err != nil {
|
||||
return nil, services.InvalidInput{Message: "invalid query"}
|
||||
}
|
||||
return q, nil
|
||||
}
|
||||
@@ -0,0 +1,137 @@
|
||||
package handlers
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"strings"
|
||||
|
||||
"messenger/src/backend/services"
|
||||
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
|
||||
type createChatRequest struct {
|
||||
Title string `json:"title"`
|
||||
MemberIDs []string `json:"member_ids"`
|
||||
}
|
||||
|
||||
func createChatHandler(chats *services.ChatService) http.HandlerFunc {
|
||||
return func(w http.ResponseWriter, r *http.Request) {
|
||||
requestID := newRequestID()
|
||||
w.Header().Set("X-Request-Id", requestID)
|
||||
if r.Method != http.MethodPost {
|
||||
writeError(w, 405, "method_not_allowed", "method not allowed", requestID)
|
||||
return
|
||||
}
|
||||
userID, ok := requestUserID(w, r, requestID)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
if !strings.HasPrefix(r.Header.Get("Content-Type"), "application/json") {
|
||||
writeError(w, 415, "unsupported_media_type", "content type must be application/json", requestID)
|
||||
return
|
||||
}
|
||||
var request createChatRequest
|
||||
if err := decodeJSONBody(r, &request); err != nil {
|
||||
writeError(w, 400, "invalid_request", "invalid JSON body", requestID)
|
||||
return
|
||||
}
|
||||
chat, err := chats.Create(r.Context(), userID, request.Title, request.MemberIDs)
|
||||
if err != nil {
|
||||
writeServiceError(w, err, requestID)
|
||||
return
|
||||
}
|
||||
writeJSON(w, 201, chat)
|
||||
}
|
||||
}
|
||||
|
||||
func getChatHandler(chats *services.ChatService) http.HandlerFunc {
|
||||
return func(w http.ResponseWriter, r *http.Request) {
|
||||
requestID := newRequestID()
|
||||
w.Header().Set("X-Request-Id", requestID)
|
||||
if r.Method != http.MethodGet {
|
||||
writeError(w, 405, "method_not_allowed", "method not allowed", requestID)
|
||||
return
|
||||
}
|
||||
userID, ok := requestUserID(w, r, requestID)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
chatID := strings.TrimPrefix(r.URL.Path, "/api/v1/chats/")
|
||||
if chatID == "" || strings.Contains(chatID, "/") {
|
||||
writeError(w, 400, "invalid_request", "invalid chat id", requestID)
|
||||
return
|
||||
}
|
||||
if _, err := uuid.Parse(chatID); err != nil {
|
||||
writeError(w, 400, "invalid_request", "invalid chat id", requestID)
|
||||
return
|
||||
}
|
||||
chat, err := chats.Get(r.Context(), userID, chatID)
|
||||
if err != nil {
|
||||
writeServiceError(w, err, requestID)
|
||||
return
|
||||
}
|
||||
writeJSON(w, 200, chat)
|
||||
}
|
||||
}
|
||||
|
||||
func listChatsHandler(chats *services.ChatService) http.HandlerFunc {
|
||||
return func(w http.ResponseWriter, r *http.Request) {
|
||||
requestID := newRequestID()
|
||||
w.Header().Set("X-Request-Id", requestID)
|
||||
if r.Method != http.MethodGet {
|
||||
writeError(w, 405, "method_not_allowed", "method not allowed", requestID)
|
||||
return
|
||||
}
|
||||
userID, ok := requestUserID(w, r, requestID)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
limit, cursor, ok := pageParams(w, r, requestID)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
page, err := chats.List(r.Context(), userID, limit, cursor)
|
||||
if err != nil {
|
||||
writeServiceError(w, err, requestID)
|
||||
return
|
||||
}
|
||||
writeJSON(w, 200, page)
|
||||
}
|
||||
}
|
||||
|
||||
func chatCollectionHandler(chats *services.ChatService) http.HandlerFunc {
|
||||
create := createChatHandler(chats)
|
||||
list := listChatsHandler(chats)
|
||||
return func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Method == http.MethodGet {
|
||||
list(w, r)
|
||||
} else {
|
||||
create(w, r)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func chatSubresourceHandler(chats *services.ChatService, messages *services.MessageService, attachments *services.AttachmentService) http.HandlerFunc {
|
||||
get := getChatHandler(chats)
|
||||
createMessage := createMessageHandler(chats, messages)
|
||||
listMessages := listMessagesHandler(chats, messages)
|
||||
return func(w http.ResponseWriter, r *http.Request) {
|
||||
if attachments != nil && strings.Contains(r.URL.Path, "/attachment-uploads") {
|
||||
directUploadHandler(chats, attachments)(w, r)
|
||||
return
|
||||
}
|
||||
if attachments != nil && strings.HasSuffix(r.URL.Path, "/attachments") {
|
||||
uploadAttachmentHandler(chats, attachments)(w, r)
|
||||
return
|
||||
}
|
||||
if strings.HasSuffix(r.URL.Path, "/messages") {
|
||||
if r.Method == http.MethodGet {
|
||||
listMessages(w, r)
|
||||
} else {
|
||||
createMessage(w, r)
|
||||
}
|
||||
return
|
||||
}
|
||||
get(w, r)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,81 @@
|
||||
package handlers
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"io"
|
||||
"net/http"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"messenger/src/backend/services"
|
||||
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
|
||||
func directUploadHandler(chats *services.ChatService, attachments *services.AttachmentService) http.HandlerFunc {
|
||||
return func(w http.ResponseWriter, r *http.Request) {
|
||||
requestID := newRequestID()
|
||||
w.Header().Set("X-Request-Id", requestID)
|
||||
if r.Method != http.MethodPost {
|
||||
writeError(w, 405, "method_not_allowed", "method not allowed", requestID)
|
||||
return
|
||||
}
|
||||
user, ok := requestUserID(w, r, requestID)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
parts := strings.Split(strings.TrimPrefix(r.URL.Path, "/api/v1/chats/"), "/")
|
||||
if (len(parts) != 2 && (len(parts) != 4 || parts[3] != "complete")) || parts[1] != "attachment-uploads" {
|
||||
writeError(w, 404, "not_found", "upload endpoint not found", requestID)
|
||||
return
|
||||
}
|
||||
chat, err := uuid.Parse(parts[0])
|
||||
if err != nil {
|
||||
writeError(w, 400, "invalid_request", "invalid chat id", requestID)
|
||||
return
|
||||
}
|
||||
if _, err = chats.Get(r.Context(), user, chat.String()); err != nil {
|
||||
writeServiceError(w, err, requestID)
|
||||
return
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(r.Context(), 25*time.Second)
|
||||
defer cancel()
|
||||
if len(parts) == 4 {
|
||||
id, err := uuid.Parse(parts[2])
|
||||
if err != nil {
|
||||
writeError(w, 400, "invalid_request", "invalid upload id", requestID)
|
||||
return
|
||||
}
|
||||
a, err := attachments.CompleteDirect(ctx, user, chat.String(), id.String())
|
||||
if err != nil {
|
||||
writeServiceError(w, err, requestID)
|
||||
return
|
||||
}
|
||||
writeJSON(w, 202, a)
|
||||
return
|
||||
}
|
||||
r.Body = http.MaxBytesReader(w, r.Body, 4096)
|
||||
var input services.DirectUploadInput
|
||||
decoder := json.NewDecoder(r.Body)
|
||||
decoder.DisallowUnknownFields()
|
||||
err = decoder.Decode(&input)
|
||||
if err == nil {
|
||||
var extra any
|
||||
if next := decoder.Decode(&extra); !errors.Is(next, io.EOF) {
|
||||
err = errors.New("expected one JSON object")
|
||||
}
|
||||
}
|
||||
if err != nil {
|
||||
writeError(w, 400, "invalid_request", "invalid upload JSON (maximum 4 KiB)", requestID)
|
||||
return
|
||||
}
|
||||
result, err := attachments.InitDirect(ctx, user, chat.String(), input)
|
||||
if err != nil {
|
||||
writeServiceError(w, err, requestID)
|
||||
return
|
||||
}
|
||||
writeJSON(w, 201, result)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,163 @@
|
||||
package handlers
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"log"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"messenger/src/backend/repositories"
|
||||
"messenger/src/backend/services"
|
||||
)
|
||||
|
||||
type healthResponse struct {
|
||||
Status string `json:"status"`
|
||||
}
|
||||
type apiError struct {
|
||||
Code string `json:"code"`
|
||||
Message string `json:"message"`
|
||||
RequestID string `json:"request_id"`
|
||||
}
|
||||
type errorResponse struct {
|
||||
Error apiError `json:"error"`
|
||||
}
|
||||
|
||||
func newRequestID() string {
|
||||
bytes := make([]byte, 16)
|
||||
if _, err := rand.Read(bytes); err != nil {
|
||||
return "request-id-generation-failed"
|
||||
}
|
||||
return hex.EncodeToString(bytes)
|
||||
}
|
||||
|
||||
func writeError(w http.ResponseWriter, status int, code, message, requestID string) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.WriteHeader(status)
|
||||
response := errorResponse{Error: apiError{Code: code, Message: message, RequestID: requestID}}
|
||||
if err := json.NewEncoder(w).Encode(response); err != nil {
|
||||
log.Printf("failed to encode error response: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func writeServiceError(w http.ResponseWriter, err error, requestID string) {
|
||||
var upload repositories.UploadError
|
||||
if errors.As(err, &upload) {
|
||||
writeError(w, upload.Status, upload.Code, upload.Message, requestID)
|
||||
return
|
||||
}
|
||||
var invalid services.InvalidInput
|
||||
switch {
|
||||
case errors.As(err, &invalid):
|
||||
writeError(w, 400, "invalid_request", invalid.Message, requestID)
|
||||
case errors.Is(err, services.ErrPayloadTooLarge):
|
||||
writeError(w, 413, "payload_too_large", "image exceeds upload limits", requestID)
|
||||
case errors.Is(err, services.ErrUnsupportedMedia):
|
||||
writeError(w, 415, "unsupported_media_type", "a valid PNG or JPEG matching Content-Type is required", requestID)
|
||||
case errors.Is(err, services.ErrAttachmentNotFound):
|
||||
writeError(w, 404, "not_found", "attachment not found", requestID)
|
||||
case errors.Is(err, services.ErrConflict):
|
||||
writeError(w, 409, "conflict", "request conflicts with an existing resource", requestID)
|
||||
case errors.Is(err, services.ErrUnknownUser):
|
||||
writeError(w, 401, "unauthorized", "unknown user", requestID)
|
||||
case errors.Is(err, services.ErrChatNotFound):
|
||||
writeError(w, 404, "not_found", "chat not found", requestID)
|
||||
default:
|
||||
writeError(w, 503, "unavailable", "dependency unavailable", requestID)
|
||||
}
|
||||
}
|
||||
|
||||
func writeJSON(w http.ResponseWriter, status int, value any) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.WriteHeader(status)
|
||||
if err := json.NewEncoder(w).Encode(value); err != nil {
|
||||
log.Printf("failed to encode response: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func requestUserID(w http.ResponseWriter, r *http.Request, requestID string) (string, bool) {
|
||||
id := currentUserID(r)
|
||||
if id == "" {
|
||||
writeError(w, 401, "unauthorized", "valid Bearer session is required", requestID)
|
||||
return "", false
|
||||
}
|
||||
return id, true
|
||||
}
|
||||
|
||||
func pageParams(w http.ResponseWriter, r *http.Request, requestID string) (int, string, bool) {
|
||||
limit := 50
|
||||
if values, exists := r.URL.Query()["limit"]; exists {
|
||||
if len(values) != 1 || values[0] == "" {
|
||||
writeError(w, 400, "invalid_request", "invalid limit", requestID)
|
||||
return 0, "", false
|
||||
}
|
||||
parsed, err := strconv.Atoi(values[0])
|
||||
if err != nil {
|
||||
writeError(w, 400, "invalid_request", "limit must be between 1 and 100", requestID)
|
||||
return 0, "", false
|
||||
}
|
||||
limit = parsed
|
||||
}
|
||||
var cursor string
|
||||
if values, exists := r.URL.Query()["cursor"]; exists {
|
||||
if len(values) != 1 || values[0] == "" {
|
||||
writeError(w, 400, "invalid_request", "invalid cursor", requestID)
|
||||
return 0, "", false
|
||||
}
|
||||
cursor = values[0]
|
||||
}
|
||||
return limit, cursor, true
|
||||
}
|
||||
|
||||
func healthHandler(w http.ResponseWriter, r *http.Request) {
|
||||
requestID := newRequestID()
|
||||
w.Header().Set("X-Request-Id", requestID)
|
||||
if r.Method != http.MethodGet {
|
||||
writeError(w, 405, "method_not_allowed", "method not allowed", requestID)
|
||||
return
|
||||
}
|
||||
writeJSON(w, 200, healthResponse{"ok"})
|
||||
}
|
||||
|
||||
func readyHandler(health *services.HealthService) http.HandlerFunc {
|
||||
return func(w http.ResponseWriter, r *http.Request) {
|
||||
requestID := newRequestID()
|
||||
w.Header().Set("X-Request-Id", requestID)
|
||||
if r.Method != http.MethodGet {
|
||||
writeError(w, 405, "method_not_allowed", "method not allowed", requestID)
|
||||
return
|
||||
}
|
||||
// Readiness must finish before HAProxy's one-second check read timeout.
|
||||
ctx, cancel := context.WithTimeout(r.Context(), 500*time.Millisecond)
|
||||
defer cancel()
|
||||
if err := health.Ready(ctx); err != nil {
|
||||
writeError(w, 503, "unavailable", "dependency unavailable", requestID)
|
||||
return
|
||||
}
|
||||
writeJSON(w, 200, healthResponse{"ok"})
|
||||
}
|
||||
}
|
||||
|
||||
func NewRouter(health *services.HealthService, users *services.UserService, sessions *services.SessionService, chats *services.ChatService, messages *services.MessageService, attachmentServices ...*services.AttachmentService) http.Handler {
|
||||
var attachments *services.AttachmentService
|
||||
if len(attachmentServices) > 0 {
|
||||
attachments = attachmentServices[0]
|
||||
}
|
||||
mux := http.NewServeMux()
|
||||
mux.HandleFunc("/health/live", healthHandler)
|
||||
mux.HandleFunc("/health/ready", readyHandler(health))
|
||||
mux.Handle("/api/v1/users", createUserHandler(users))
|
||||
mux.Handle("/api/v1/auth/sessions", sessionHandler(sessions))
|
||||
mux.Handle("/api/v1/auth/sessions/current", authRequired(sessions, logoutHandler(sessions)))
|
||||
mux.Handle("/api/v1/users/me", authRequired(sessions, currentUserHandler(users)))
|
||||
mux.Handle("/api/v1/chats", authRequired(sessions, chatCollectionHandler(chats)))
|
||||
mux.Handle("/api/v1/chats/", authRequired(sessions, chatSubresourceHandler(chats, messages, attachments)))
|
||||
if attachments != nil {
|
||||
mux.Handle("/api/v1/attachments/", authRequired(sessions, attachmentHandler(attachments)))
|
||||
}
|
||||
return withInstance(mux)
|
||||
}
|
||||
@@ -0,0 +1,34 @@
|
||||
package handlers
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"os"
|
||||
"sync/atomic"
|
||||
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
|
||||
// Docker's hostname lets the operator map an API response to its container.
|
||||
var instanceID = func() string {
|
||||
id, err := os.Hostname()
|
||||
if err != nil || id == "" {
|
||||
return uuid.NewString()
|
||||
}
|
||||
return id
|
||||
}()
|
||||
|
||||
var draining atomic.Bool
|
||||
|
||||
func BeginShutdown() { draining.Store(true) }
|
||||
|
||||
func withInstance(next http.Handler) http.Handler {
|
||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("X-Instance-Id", instanceID)
|
||||
w.Header().Set("X-Request-Id", newRequestID())
|
||||
if draining.Load() && r.URL.Path != "/health/live" {
|
||||
writeError(w, 503, "unavailable", "instance is shutting down", w.Header().Get("X-Request-Id"))
|
||||
return
|
||||
}
|
||||
next.ServeHTTP(w, r)
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,127 @@
|
||||
package handlers
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"strings"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"messenger/src/backend/services"
|
||||
)
|
||||
|
||||
type createMessageRequest struct {
|
||||
Text *string `json:"text"`
|
||||
AttachmentIDs json.RawMessage `json:"attachment_ids"`
|
||||
}
|
||||
|
||||
func messageChatID(path string) (string, bool) {
|
||||
const prefix = "/api/v1/chats/"
|
||||
const suffix = "/messages"
|
||||
if !strings.HasPrefix(path, prefix) || !strings.HasSuffix(path, suffix) {
|
||||
return "", false
|
||||
}
|
||||
chatID := strings.TrimSuffix(strings.TrimPrefix(path, prefix), suffix)
|
||||
if chatID == "" || strings.Contains(chatID, "/") {
|
||||
return "", false
|
||||
}
|
||||
return chatID, true
|
||||
}
|
||||
|
||||
func createMessageHandler(chats *services.ChatService, messages *services.MessageService) http.HandlerFunc {
|
||||
return func(w http.ResponseWriter, r *http.Request) {
|
||||
requestID := newRequestID()
|
||||
w.Header().Set("X-Request-Id", requestID)
|
||||
if r.Method != http.MethodPost {
|
||||
writeError(w, 405, "method_not_allowed", "method not allowed", requestID)
|
||||
return
|
||||
}
|
||||
userID, ok := requestUserID(w, r, requestID)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
chatID, ok := messageChatID(r.URL.Path)
|
||||
if !ok {
|
||||
writeError(w, 400, "invalid_request", "invalid chat id", requestID)
|
||||
return
|
||||
}
|
||||
if _, err := uuid.Parse(chatID); err != nil {
|
||||
writeError(w, 400, "invalid_request", "invalid chat id", requestID)
|
||||
return
|
||||
}
|
||||
chat, err := chats.Get(r.Context(), userID, chatID)
|
||||
if err != nil {
|
||||
writeServiceError(w, err, requestID)
|
||||
return
|
||||
}
|
||||
if !strings.HasPrefix(r.Header.Get("Content-Type"), "application/json") {
|
||||
writeError(w, 415, "unsupported_media_type", "content type must be application/json", requestID)
|
||||
return
|
||||
}
|
||||
var request createMessageRequest
|
||||
if err := decodeJSONBody(r, &request); err != nil {
|
||||
writeError(w, 400, "invalid_request", "invalid JSON body", requestID)
|
||||
return
|
||||
}
|
||||
ids := []string{}
|
||||
if request.Text == nil {
|
||||
writeError(w, 400, "invalid_request", "text must be a string", requestID)
|
||||
return
|
||||
}
|
||||
if len(request.AttachmentIDs) > 0 {
|
||||
if err := json.Unmarshal(request.AttachmentIDs, &ids); err != nil || ids == nil {
|
||||
writeError(w, 400, "invalid_request", "attachment_ids must be an array of UUIDs", requestID)
|
||||
return
|
||||
}
|
||||
}
|
||||
keys := r.Header.Values("Idempotency-Key")
|
||||
if len(keys) != 1 {
|
||||
writeError(w, 400, "invalid_request", "exactly one Idempotency-Key is required", requestID)
|
||||
return
|
||||
}
|
||||
message, err := messages.CreateIdempotent(r.Context(), userID, chat, *request.Text, keys[0], ids)
|
||||
if err != nil {
|
||||
writeServiceError(w, err, requestID)
|
||||
return
|
||||
}
|
||||
writeJSON(w, 201, message)
|
||||
}
|
||||
}
|
||||
|
||||
func listMessagesHandler(chats *services.ChatService, messages *services.MessageService) http.HandlerFunc {
|
||||
return func(w http.ResponseWriter, r *http.Request) {
|
||||
requestID := newRequestID()
|
||||
w.Header().Set("X-Request-Id", requestID)
|
||||
if r.Method != http.MethodGet {
|
||||
writeError(w, 405, "method_not_allowed", "method not allowed", requestID)
|
||||
return
|
||||
}
|
||||
userID, ok := requestUserID(w, r, requestID)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
chatID, ok := messageChatID(r.URL.Path)
|
||||
if !ok {
|
||||
writeError(w, 400, "invalid_request", "invalid chat id", requestID)
|
||||
return
|
||||
}
|
||||
if _, err := uuid.Parse(chatID); err != nil {
|
||||
writeError(w, 400, "invalid_request", "invalid chat id", requestID)
|
||||
return
|
||||
}
|
||||
chat, err := chats.Get(r.Context(), userID, chatID)
|
||||
if err != nil {
|
||||
writeServiceError(w, err, requestID)
|
||||
return
|
||||
}
|
||||
limit, cursor, ok := pageParams(w, r, requestID)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
page, err := messages.List(r.Context(), userID, chat, limit, cursor)
|
||||
if err != nil {
|
||||
writeServiceError(w, err, requestID)
|
||||
return
|
||||
}
|
||||
writeJSON(w, 200, page)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,98 @@
|
||||
package handlers
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"net/http"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"messenger/src/backend/services"
|
||||
)
|
||||
|
||||
type authenticatedUserKey struct{}
|
||||
|
||||
func currentUserID(r *http.Request) string {
|
||||
id, _ := r.Context().Value(authenticatedUserKey{}).(string)
|
||||
return id
|
||||
}
|
||||
|
||||
func authRequired(sessions *services.SessionService, next http.Handler) http.Handler {
|
||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
authorization := r.Header.Get("Authorization")
|
||||
if !strings.HasPrefix(authorization, "Bearer ") || strings.TrimSpace(strings.TrimPrefix(authorization, "Bearer ")) == "" {
|
||||
unauthorized(w, "Bearer", newRequestID())
|
||||
return
|
||||
}
|
||||
user, err := sessions.User(r.Context(), strings.TrimPrefix(authorization, "Bearer "))
|
||||
if errors.Is(err, services.ErrInvalidSession) {
|
||||
unauthorized(w, "Bearer", newRequestID())
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
requestID := newRequestID()
|
||||
w.Header().Set("X-Request-Id", requestID)
|
||||
writeError(w, 503, "unavailable", "dependency unavailable", requestID)
|
||||
return
|
||||
}
|
||||
next.ServeHTTP(w, r.WithContext(context.WithValue(r.Context(), authenticatedUserKey{}, user.ID)))
|
||||
})
|
||||
}
|
||||
|
||||
func unauthorized(w http.ResponseWriter, scheme, requestID string) {
|
||||
w.Header().Set("X-Request-Id", requestID)
|
||||
if scheme == "Basic" {
|
||||
w.Header().Set("WWW-Authenticate", `Basic realm="messenger", charset="UTF-8"`)
|
||||
} else {
|
||||
w.Header().Set("WWW-Authenticate", `Bearer realm="messenger"`)
|
||||
}
|
||||
writeError(w, 401, "unauthorized", "invalid credentials", requestID)
|
||||
}
|
||||
|
||||
func sessionHandler(sessions *services.SessionService) http.HandlerFunc {
|
||||
return func(w http.ResponseWriter, r *http.Request) {
|
||||
requestID := newRequestID()
|
||||
w.Header().Set("X-Request-Id", requestID)
|
||||
if r.Method != http.MethodPost {
|
||||
writeError(w, 405, "method_not_allowed", "method not allowed", requestID)
|
||||
return
|
||||
}
|
||||
username, password, ok := r.BasicAuth()
|
||||
if !ok || r.ContentLength > 0 {
|
||||
unauthorized(w, "Basic", requestID)
|
||||
return
|
||||
}
|
||||
user, token, expires, valid, err := sessions.Login(r.Context(), username, password)
|
||||
if err != nil {
|
||||
writeError(w, 503, "unavailable", "dependency unavailable", requestID)
|
||||
return
|
||||
}
|
||||
if !valid {
|
||||
unauthorized(w, "Basic", requestID)
|
||||
return
|
||||
}
|
||||
writeJSON(w, 201, struct {
|
||||
Token string `json:"token"`
|
||||
TokenType string `json:"token_type"`
|
||||
ExpiresAt time.Time `json:"expires_at"`
|
||||
User any `json:"user"`
|
||||
}{token, "Bearer", expires, user})
|
||||
}
|
||||
}
|
||||
|
||||
func logoutHandler(sessions *services.SessionService) http.HandlerFunc {
|
||||
return func(w http.ResponseWriter, r *http.Request) {
|
||||
requestID := newRequestID()
|
||||
w.Header().Set("X-Request-Id", requestID)
|
||||
if r.Method != http.MethodDelete {
|
||||
writeError(w, 405, "method_not_allowed", "method not allowed", requestID)
|
||||
return
|
||||
}
|
||||
token := strings.TrimPrefix(r.Header.Get("Authorization"), "Bearer ")
|
||||
if err := sessions.Revoke(r.Context(), token); err != nil {
|
||||
writeError(w, 503, "unavailable", "dependency unavailable", requestID)
|
||||
return
|
||||
}
|
||||
w.WriteHeader(204)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,103 @@
|
||||
package handlers
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"reflect"
|
||||
"strings"
|
||||
|
||||
"messenger/src/backend/services"
|
||||
)
|
||||
|
||||
type createUserRequest struct {
|
||||
Username string `json:"username"`
|
||||
DisplayName string `json:"display_name"`
|
||||
Password string `json:"password"`
|
||||
}
|
||||
|
||||
func decodeJSONBody(r *http.Request, destination any) error {
|
||||
decoder := json.NewDecoder(r.Body)
|
||||
var body json.RawMessage
|
||||
if err := decoder.Decode(&body); err != nil {
|
||||
return err
|
||||
}
|
||||
var extra any
|
||||
if err := decoder.Decode(&extra); err != io.EOF {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return fmt.Errorf("more than one JSON value")
|
||||
}
|
||||
var fields map[string]json.RawMessage
|
||||
if err := json.Unmarshal(body, &fields); err != nil {
|
||||
return err
|
||||
}
|
||||
if fields == nil {
|
||||
return fmt.Errorf("JSON body must be an object")
|
||||
}
|
||||
// encoding/json matches struct fields case-insensitively, even with
|
||||
// DisallowUnknownFields. Check exact contract names before decoding values.
|
||||
typeOfRequest := reflect.TypeOf(destination).Elem()
|
||||
allowed := make(map[string]bool, typeOfRequest.NumField())
|
||||
for i := 0; i < typeOfRequest.NumField(); i++ {
|
||||
name := strings.Split(typeOfRequest.Field(i).Tag.Get("json"), ",")[0]
|
||||
if name != "" && name != "-" {
|
||||
allowed[name] = true
|
||||
}
|
||||
}
|
||||
for name := range fields {
|
||||
if !allowed[name] {
|
||||
return fmt.Errorf("unknown JSON field %q", name)
|
||||
}
|
||||
}
|
||||
return json.Unmarshal(body, destination)
|
||||
}
|
||||
|
||||
func createUserHandler(users *services.UserService) http.HandlerFunc {
|
||||
return func(w http.ResponseWriter, r *http.Request) {
|
||||
requestID := newRequestID()
|
||||
w.Header().Set("X-Request-Id", requestID)
|
||||
if r.Method != http.MethodPost {
|
||||
writeError(w, 405, "method_not_allowed", "method not allowed", requestID)
|
||||
return
|
||||
}
|
||||
if !strings.HasPrefix(r.Header.Get("Content-Type"), "application/json") {
|
||||
writeError(w, 415, "unsupported_media_type", "content type must be application/json", requestID)
|
||||
return
|
||||
}
|
||||
var request createUserRequest
|
||||
if err := decodeJSONBody(r, &request); err != nil {
|
||||
writeError(w, 400, "invalid_request", "invalid JSON body", requestID)
|
||||
return
|
||||
}
|
||||
user, err := users.Create(r.Context(), request.Username, request.DisplayName, request.Password)
|
||||
if err != nil {
|
||||
writeServiceError(w, err, requestID)
|
||||
return
|
||||
}
|
||||
writeJSON(w, 201, user)
|
||||
}
|
||||
}
|
||||
|
||||
func currentUserHandler(users *services.UserService) http.HandlerFunc {
|
||||
return func(w http.ResponseWriter, r *http.Request) {
|
||||
requestID := newRequestID()
|
||||
w.Header().Set("X-Request-Id", requestID)
|
||||
if r.Method != http.MethodGet {
|
||||
writeError(w, 405, "method_not_allowed", "method not allowed", requestID)
|
||||
return
|
||||
}
|
||||
userID, ok := requestUserID(w, r, requestID)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
user, err := users.Get(r.Context(), userID)
|
||||
if err != nil {
|
||||
writeServiceError(w, err, requestID)
|
||||
return
|
||||
}
|
||||
writeJSON(w, 200, user)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,64 @@
|
||||
package faults
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
|
||||
// Only an operator-mounted directory enables faults; clients cannot set them.
|
||||
// Rename makes each injected crash one-shot across automatic container restarts.
|
||||
func Take(point, id string) bool {
|
||||
directory := os.Getenv("PIPELINE_FAULT_DIR")
|
||||
if directory == "" {
|
||||
return false
|
||||
}
|
||||
if _, err := uuid.Parse(id); err != nil {
|
||||
return false
|
||||
}
|
||||
name := filepath.Join(directory, point+"."+id)
|
||||
return os.Rename(name, name+".hit") == nil
|
||||
}
|
||||
func Crash(point, id string) {
|
||||
if Take(point, id) {
|
||||
log.Printf("injected pipeline crash: %s, attachment %s", point, id)
|
||||
os.Exit(86)
|
||||
}
|
||||
}
|
||||
|
||||
// Observe records a test-only branch without exposing an API or a lease token.
|
||||
func Observe(point, id string) {
|
||||
directory := os.Getenv("PIPELINE_FAULT_DIR")
|
||||
if directory == "" {
|
||||
return
|
||||
}
|
||||
if _, err := uuid.Parse(id); err != nil {
|
||||
return
|
||||
}
|
||||
_ = os.WriteFile(filepath.Join(directory, point+"."+id+".observed"), []byte("observed\n"), 0644)
|
||||
}
|
||||
|
||||
// Pause is operator-only and ends on release or context expiry.
|
||||
func Pause(ctx context.Context, point, id string) {
|
||||
if !Take(point, id) {
|
||||
return
|
||||
}
|
||||
release := filepath.Join(os.Getenv("PIPELINE_FAULT_DIR"), point+"."+id+".release")
|
||||
ticker := time.NewTicker(50 * time.Millisecond)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-ticker.C:
|
||||
if _, err := os.Stat(release); err == nil {
|
||||
_ = os.WriteFile(release+".observed", []byte("released\n"), 0644)
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"log"
|
||||
"os"
|
||||
|
||||
"messenger/src/backend/commands"
|
||||
)
|
||||
|
||||
func main() {
|
||||
if err := commands.Run(os.Args[1:]); err != nil {
|
||||
if len(os.Args) > 1 && (os.Args[1] == "image-process" || os.Args[1] == "healthcheck") {
|
||||
os.Exit(1)
|
||||
}
|
||||
log.Fatal(err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,48 @@
|
||||
package repositories
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
"time"
|
||||
)
|
||||
|
||||
type Attachment struct {
|
||||
ID string `json:"id"`
|
||||
ChatID string `json:"chat_id"`
|
||||
OwnerID string `json:"owner_id"`
|
||||
Filename string `json:"filename"`
|
||||
ContentType string `json:"content_type"`
|
||||
SizeBytes int64 `json:"size_bytes"`
|
||||
SHA256 string `json:"sha256"`
|
||||
Status string `json:"status"`
|
||||
Variants []any `json:"variants"`
|
||||
ErrorCode *string `json:"error_code"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
UpdatedAt time.Time `json:"updated_at"`
|
||||
ObjectKey string `json:"-"`
|
||||
PipelineVersion int `json:"-"`
|
||||
}
|
||||
|
||||
type AttachmentStore struct{ db *pgxpool.Pool }
|
||||
|
||||
func NewAttachmentStore(db *pgxpool.Pool) *AttachmentStore { return &AttachmentStore{db: db} }
|
||||
func (s *AttachmentStore) Create(ctx context.Context, a Attachment) error {
|
||||
ctx, cancel := boundedQuery(ctx)
|
||||
defer cancel()
|
||||
_, err := s.db.Exec(ctx, `INSERT INTO attachments(id,chat_id,owner_id,filename,object_key,content_type,size_bytes,sha256,created_at,updated_at) VALUES($1,$2,$3,$4,$5,$6,$7,$8,$9,$10)`, a.ID, a.ChatID, a.OwnerID, a.Filename, a.ObjectKey, a.ContentType, a.SizeBytes, a.SHA256, a.CreatedAt, a.UpdatedAt)
|
||||
return err
|
||||
}
|
||||
func (s *AttachmentStore) GetForMember(ctx context.Context, id, userID string) (Attachment, error) {
|
||||
ctx, cancel := boundedQuery(ctx)
|
||||
defer cancel()
|
||||
a := Attachment{Variants: []any{}}
|
||||
var variants []byte
|
||||
err := s.db.QueryRow(ctx, `SELECT a.id,a.chat_id,a.owner_id,a.filename,a.object_key,a.content_type,a.size_bytes,a.sha256,a.created_at,a.updated_at,a.status,a.variants,a.error_code,a.published_pipeline_version FROM attachments a JOIN chat_members cm ON cm.chat_id=a.chat_id WHERE a.id=$1 AND cm.user_id=$2`, id, userID).Scan(&a.ID, &a.ChatID, &a.OwnerID, &a.Filename, &a.ObjectKey, &a.ContentType, &a.SizeBytes, &a.SHA256, &a.CreatedAt, &a.UpdatedAt, &a.Status, &variants, &a.ErrorCode, &a.PipelineVersion)
|
||||
if err == nil {
|
||||
err = json.Unmarshal(variants, &a.Variants)
|
||||
}
|
||||
a.CreatedAt = a.CreatedAt.UTC()
|
||||
a.UpdatedAt = a.UpdatedAt.UTC()
|
||||
return a, err
|
||||
}
|
||||
@@ -0,0 +1,83 @@
|
||||
package repositories
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"errors"
|
||||
"messenger/src/backend/telemetry"
|
||||
"net"
|
||||
"time"
|
||||
|
||||
amqp "github.com/rabbitmq/amqp091-go"
|
||||
)
|
||||
|
||||
const jobQueue = "attachments.v1"
|
||||
|
||||
type JobBroker struct {
|
||||
connection *amqp.Connection
|
||||
channel *amqp.Channel
|
||||
returned chan amqp.Return
|
||||
}
|
||||
|
||||
func NewJobBroker(ctx context.Context, address string) (*JobBroker, error) {
|
||||
connection, err := amqp.DialConfig(address, amqp.Config{
|
||||
Heartbeat: 10 * time.Second,
|
||||
Dial: func(network, address string) (net.Conn, error) {
|
||||
return (&net.Dialer{Timeout: 5 * time.Second}).DialContext(ctx, network, address)
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
return nil, errors.New("RabbitMQ connection unavailable")
|
||||
}
|
||||
channel, err := connection.Channel()
|
||||
if err != nil {
|
||||
connection.Close()
|
||||
return nil, errors.New("RabbitMQ channel unavailable")
|
||||
}
|
||||
b := &JobBroker{connection: connection, channel: channel, returned: make(chan amqp.Return, 1)}
|
||||
// Queue metadata and message bodies both survive a broker restart.
|
||||
_, err = channel.QueueDeclare(jobQueue, true, false, false, false, nil)
|
||||
if err == nil {
|
||||
err = channel.Confirm(false)
|
||||
}
|
||||
if err == nil {
|
||||
err = channel.Qos(1, 0, false)
|
||||
}
|
||||
if err != nil {
|
||||
b.Close()
|
||||
return nil, errors.New("RabbitMQ queue initialization failed")
|
||||
}
|
||||
channel.NotifyReturn(b.returned)
|
||||
return b, nil
|
||||
}
|
||||
|
||||
func (b *JobBroker) Close() { b.channel.Close(); b.connection.Close() }
|
||||
|
||||
func (b *JobBroker) Publish(ctx context.Context, body []byte) error {
|
||||
// mandatory catches an unroutable message; confirms alone do not prove routing.
|
||||
headers := amqp.Table{}
|
||||
for key, value := range telemetry.TraceHeaders(ctx) {
|
||||
headers[key] = value
|
||||
}
|
||||
confirmation, err := b.channel.PublishWithDeferredConfirmWithContext(ctx, "", jobQueue, true, false, amqp.Publishing{
|
||||
DeliveryMode: amqp.Persistent, ContentType: "application/json", Body: body, Headers: headers,
|
||||
})
|
||||
if err != nil {
|
||||
return errors.New("RabbitMQ publication failed")
|
||||
}
|
||||
confirmed, err := confirmation.WaitContext(ctx)
|
||||
if err != nil || !confirmed {
|
||||
return errors.New("RabbitMQ publication not confirmed")
|
||||
}
|
||||
// basic.return precedes basic.ack on the channel; NotifyReturn is buffered.
|
||||
select {
|
||||
case <-b.returned:
|
||||
return errors.New("RabbitMQ message was not routed")
|
||||
default:
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
func (b *JobBroker) Consume() (<-chan amqp.Delivery, error) {
|
||||
return b.channel.Consume(jobQueue, "", false, false, false, false, nil)
|
||||
}
|
||||
@@ -0,0 +1,129 @@
|
||||
package repositories
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
)
|
||||
|
||||
type Chat struct {
|
||||
ID string `json:"id"`
|
||||
Title string `json:"title"`
|
||||
MemberIDs []string `json:"member_ids"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
}
|
||||
|
||||
type ChatStore struct {
|
||||
db *pgxpool.Pool
|
||||
}
|
||||
|
||||
func NewChatStore(db *pgxpool.Pool) *ChatStore {
|
||||
return &ChatStore{db: db}
|
||||
}
|
||||
|
||||
var ErrUnknownMember = errors.New("unknown member")
|
||||
|
||||
func (s *ChatStore) Create(ctx context.Context, title string, memberIDs []string) (Chat, error) {
|
||||
ctx, cancel := boundedQuery(ctx)
|
||||
defer cancel()
|
||||
membersCopy := append([]string(nil), memberIDs...)
|
||||
chat := Chat{
|
||||
ID: uuid.New().String(),
|
||||
Title: title,
|
||||
MemberIDs: membersCopy,
|
||||
CreatedAt: time.Now().UTC().Truncate(time.Microsecond),
|
||||
}
|
||||
|
||||
tx, err := s.db.Begin(ctx)
|
||||
if err != nil {
|
||||
return Chat{}, err
|
||||
}
|
||||
defer tx.Rollback(ctx)
|
||||
for _, id := range membersCopy {
|
||||
var exists bool
|
||||
if err = tx.QueryRow(ctx, "SELECT EXISTS(SELECT 1 FROM users WHERE id=$1)", id).Scan(&exists); err != nil {
|
||||
return Chat{}, err
|
||||
}
|
||||
if !exists {
|
||||
return Chat{}, ErrUnknownMember
|
||||
}
|
||||
}
|
||||
if _, err = tx.Exec(ctx, "INSERT INTO chats(id,title,created_at) VALUES($1,$2,$3)", chat.ID, title, chat.CreatedAt); err != nil {
|
||||
return Chat{}, err
|
||||
}
|
||||
for _, id := range membersCopy {
|
||||
if _, err = tx.Exec(ctx, "INSERT INTO chat_members(chat_id,user_id) VALUES($1,$2)", chat.ID, id); err != nil {
|
||||
return Chat{}, err
|
||||
}
|
||||
}
|
||||
if err = tx.Commit(ctx); err != nil {
|
||||
return Chat{}, err
|
||||
}
|
||||
return chat, nil
|
||||
}
|
||||
|
||||
func (s *ChatStore) GetByID(ctx context.Context, id string) (Chat, error) {
|
||||
ctx, cancel := boundedQuery(ctx)
|
||||
defer cancel()
|
||||
var chat Chat
|
||||
err := s.db.QueryRow(ctx, "SELECT id,title,created_at FROM chats WHERE id=$1", id).Scan(&chat.ID, &chat.Title, &chat.CreatedAt)
|
||||
if err != nil {
|
||||
return Chat{}, err
|
||||
}
|
||||
rows, err := s.db.Query(ctx, "SELECT user_id FROM chat_members WHERE chat_id=$1 ORDER BY user_id", id)
|
||||
if err != nil {
|
||||
return Chat{}, err
|
||||
}
|
||||
defer rows.Close()
|
||||
chat.MemberIDs = []string{}
|
||||
for rows.Next() {
|
||||
var member string
|
||||
if err = rows.Scan(&member); err != nil {
|
||||
return Chat{}, err
|
||||
}
|
||||
chat.MemberIDs = append(chat.MemberIDs, member)
|
||||
}
|
||||
return chat, rows.Err()
|
||||
}
|
||||
|
||||
func (s *ChatStore) ListPageForUser(ctx context.Context, userID string, cursor *ChatCursor, limit int) ([]Chat, bool, error) {
|
||||
ctx, cancel := boundedQuery(ctx)
|
||||
defer cancel()
|
||||
afterTime := time.Time{}
|
||||
afterID := "00000000-0000-0000-0000-000000000000"
|
||||
if cursor != nil {
|
||||
afterTime = cursor.CreatedAt
|
||||
afterID = cursor.ChatID
|
||||
}
|
||||
rows, err := s.db.Query(ctx, `SELECT c.id,c.title,c.created_at FROM chats c JOIN chat_members m ON m.chat_id=c.id WHERE m.user_id=$1 AND (c.created_at,c.id)>($2,$3::uuid) ORDER BY c.created_at,c.id LIMIT $4`, userID, afterTime, afterID, limit+1)
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
defer rows.Close()
|
||||
chats := []Chat{}
|
||||
for rows.Next() {
|
||||
var c Chat
|
||||
if err = rows.Scan(&c.ID, &c.Title, &c.CreatedAt); err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
chats = append(chats, c)
|
||||
}
|
||||
if err = rows.Err(); err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
more := len(chats) > limit
|
||||
if more {
|
||||
chats = chats[:limit]
|
||||
}
|
||||
for i := range chats {
|
||||
chat, err := s.GetByID(ctx, chats[i].ID)
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
chats[i] = chat
|
||||
}
|
||||
return chats, more, nil
|
||||
}
|
||||
@@ -0,0 +1,20 @@
|
||||
package repositories
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"os"
|
||||
"strconv"
|
||||
"time"
|
||||
)
|
||||
|
||||
func UploadCleanupGrace() (time.Duration, error) {
|
||||
value := os.Getenv("UPLOAD_CLEANUP_GRACE_SECONDS")
|
||||
if value == "" {
|
||||
value = "600"
|
||||
}
|
||||
seconds, err := strconv.Atoi(value)
|
||||
if err != nil || seconds < 120 || seconds > 86400 {
|
||||
return 0, errors.New("UPLOAD_CLEANUP_GRACE_SECONDS must be between 120 and 86400")
|
||||
}
|
||||
return time.Duration(seconds) * time.Second, nil
|
||||
}
|
||||
@@ -0,0 +1,47 @@
|
||||
package repositories
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/redis/go-redis/v9"
|
||||
)
|
||||
|
||||
func RedisClientFromEnvironment() (*redis.Client, error) {
|
||||
if path := os.Getenv("CACHE_URL_FILE"); path != "" {
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
options, err := redis.ParseURL(strings.TrimSpace(string(data)))
|
||||
if err != nil {
|
||||
return nil, errors.New("invalid Redis URL in CACHE_URL_FILE")
|
||||
}
|
||||
options.PoolSize = 10
|
||||
options.MaxRetries = -1
|
||||
options.DialTimeout = time.Second
|
||||
options.ReadTimeout = time.Second
|
||||
options.WriteTimeout = time.Second
|
||||
return redis.NewClient(options), nil
|
||||
}
|
||||
address := os.Getenv("REDIS_ADDR")
|
||||
if address == "" {
|
||||
address = "redis:6379"
|
||||
}
|
||||
return NewRedisClient(address), nil
|
||||
}
|
||||
|
||||
func IdempotencyTTL() (time.Duration, error) {
|
||||
value := os.Getenv("IDEMPOTENCY_TTL_SECONDS")
|
||||
if value == "" {
|
||||
value = "86400"
|
||||
}
|
||||
seconds, err := strconv.ParseInt(value, 10, 64)
|
||||
if err != nil || seconds < 1 || seconds > 31536000 {
|
||||
return 0, errors.New("IDEMPOTENCY_TTL_SECONDS must be between 1 and 31536000")
|
||||
}
|
||||
return time.Duration(seconds) * time.Second, nil
|
||||
}
|
||||
@@ -0,0 +1,15 @@
|
||||
package repositories
|
||||
|
||||
import "time"
|
||||
|
||||
type ChatCursor struct {
|
||||
UserID string `json:"user_id"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
ChatID string `json:"chat_id"`
|
||||
}
|
||||
|
||||
type MessageCursor struct {
|
||||
ChatID string `json:"chat_id"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
MessageID string `json:"message_id"`
|
||||
}
|
||||
@@ -0,0 +1,100 @@
|
||||
package repositories
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"embed"
|
||||
"errors"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"messenger/src/backend/telemetry"
|
||||
|
||||
"github.com/golang-migrate/migrate/v4"
|
||||
pgxmigrate "github.com/golang-migrate/migrate/v4/database/pgx/v5"
|
||||
"github.com/golang-migrate/migrate/v4/source/iofs"
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
_ "github.com/jackc/pgx/v5/stdlib"
|
||||
)
|
||||
|
||||
//go:embed migrations/*.sql
|
||||
var migrationFiles embed.FS
|
||||
|
||||
func DatabaseURL() (string, error) {
|
||||
if path := os.Getenv("DATABASE_URL_FILE"); path != "" {
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return strings.TrimSpace(string(data)), nil
|
||||
}
|
||||
if url := os.Getenv("DATABASE_URL"); url != "" {
|
||||
return url, nil
|
||||
}
|
||||
return "", errors.New("DATABASE_URL_FILE or DATABASE_URL is required")
|
||||
}
|
||||
|
||||
func OpenDatabase(ctx context.Context) (*pgxpool.Pool, error) {
|
||||
url, err := DatabaseURL()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
cfg, err := pgxpool.ParseConfig(url)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
cfg.ConnConfig.Tracer = telemetry.PostgresTracer{}
|
||||
cfg.MaxConns = 10
|
||||
if value := os.Getenv("DATABASE_MAX_CONNS"); value != "" {
|
||||
maxConns, parseErr := strconv.Atoi(value)
|
||||
if parseErr != nil || maxConns < 1 || maxConns > 100 {
|
||||
return nil, errors.New("DATABASE_MAX_CONNS must be between 1 and 100")
|
||||
}
|
||||
cfg.MaxConns = int32(maxConns)
|
||||
}
|
||||
cfg.ConnConfig.ConnectTimeout = 5 * time.Second
|
||||
db, err := pgxpool.NewWithConfig(ctx, cfg)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err = db.Ping(ctx); err != nil {
|
||||
db.Close()
|
||||
return nil, err
|
||||
}
|
||||
return db, nil
|
||||
}
|
||||
|
||||
// RunMigrations uses versioned up/down SQL embedded in the API binary.
|
||||
// "down" rolls back exactly one version and is an explicit operator command.
|
||||
func RunMigrations(url, direction string) error {
|
||||
source, err := iofs.New(migrationFiles, "migrations")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
sqlDB, err := sql.Open("pgx", url)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
driver, err := pgxmigrate.WithInstance(sqlDB, &pgxmigrate.Config{})
|
||||
if err != nil {
|
||||
sqlDB.Close()
|
||||
return err
|
||||
}
|
||||
m, err := migrate.NewWithInstance("iofs", source, "pgx5", driver)
|
||||
if err != nil {
|
||||
sqlDB.Close()
|
||||
return err
|
||||
}
|
||||
defer m.Close()
|
||||
if direction == "down" {
|
||||
err = m.Steps(-1)
|
||||
} else {
|
||||
err = m.Up()
|
||||
}
|
||||
if errors.Is(err, migrate.ErrNoChange) {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
@@ -0,0 +1,106 @@
|
||||
package repositories
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/jackc/pgx/v5"
|
||||
)
|
||||
|
||||
func (s *AttachmentStore) CleanupDirectUploads(ctx context.Context, dryRun bool, remove func(context.Context, string) error) (CleanupResult, error) {
|
||||
result := CleanupResult{DryRun: dryRun}
|
||||
rows, err := s.db.Query(ctx, `SELECT id::text FROM direct_uploads WHERE cleanup_after<=clock_timestamp() ORDER BY cleanup_after,id LIMIT 100`)
|
||||
if err != nil {
|
||||
return result, err
|
||||
}
|
||||
ids := []string{}
|
||||
for rows.Next() {
|
||||
var id string
|
||||
if err = rows.Scan(&id); err != nil {
|
||||
rows.Close()
|
||||
return result, err
|
||||
}
|
||||
ids = append(ids, id)
|
||||
}
|
||||
err = rows.Err()
|
||||
rows.Close()
|
||||
if err != nil {
|
||||
return result, err
|
||||
}
|
||||
result.Candidates = len(ids)
|
||||
for _, id := range ids {
|
||||
outcome, err := s.cleanupDirectUpload(ctx, id, dryRun, remove)
|
||||
if err != nil {
|
||||
result.Failed++
|
||||
continue
|
||||
}
|
||||
switch outcome {
|
||||
case "deleted":
|
||||
result.Deleted++
|
||||
case "eligible":
|
||||
result.Eligible++
|
||||
default:
|
||||
result.Skipped++
|
||||
}
|
||||
}
|
||||
if result.Failed > 0 {
|
||||
return result, fmt.Errorf("%d direct upload cleanup operations failed", result.Failed)
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func (s *AttachmentStore) cleanupDirectUpload(ctx context.Context, id string, dryRun bool, remove func(context.Context, string) error) (string, error) {
|
||||
ctx, cancel := context.WithTimeout(ctx, 10*time.Second)
|
||||
defer cancel()
|
||||
tx, err := s.db.Begin(ctx)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer tx.Rollback(context.Background())
|
||||
var key, state string
|
||||
var barrier, complete, purge, now time.Time
|
||||
err = tx.QueryRow(ctx, `SELECT staging_key,state,complete_before+interval '600 seconds',complete_before,purge_after,clock_timestamp() FROM direct_uploads WHERE id=$1 AND cleanup_after<=clock_timestamp() FOR UPDATE SKIP LOCKED`, id).Scan(&key, &state, &barrier, &complete, &purge, &now)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return "skipped", nil
|
||||
}
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
parsed, err := uuid.Parse(id)
|
||||
if err != nil || key != "staging/direct/"+parsed.String() {
|
||||
return "", errors.New("invalid direct staging key")
|
||||
}
|
||||
if state == "pending" && now.Before(barrier) {
|
||||
return "skipped", nil
|
||||
}
|
||||
if dryRun {
|
||||
return "eligible", nil
|
||||
}
|
||||
if err = remove(ctx, key); err != nil {
|
||||
return "", err
|
||||
}
|
||||
if now.After(purge) && !now.Before(barrier) {
|
||||
_, err = tx.Exec(ctx, `DELETE FROM direct_uploads WHERE id=$1`, id)
|
||||
} else {
|
||||
if state == "pending" && !now.Before(complete) {
|
||||
state = "expired"
|
||||
}
|
||||
// An early cleanup is not final: a still valid POST can recreate staging.
|
||||
// Keep a second cleanup after the safety barrier, then retain the receipt 24h.
|
||||
next := barrier
|
||||
if !now.Before(barrier) {
|
||||
next = purge
|
||||
}
|
||||
_, err = tx.Exec(ctx, `UPDATE direct_uploads SET state=$2,cleanup_after=$3 WHERE id=$1`, id, state, next)
|
||||
}
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if err = tx.Commit(ctx); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return "deleted", nil
|
||||
}
|
||||
@@ -0,0 +1,150 @@
|
||||
package repositories
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
)
|
||||
|
||||
// UploadError is also persisted for terminal validation failures, so retries agree.
|
||||
type UploadError struct {
|
||||
Status int
|
||||
Code, Message string
|
||||
}
|
||||
|
||||
func (e UploadError) Error() string { return e.Message }
|
||||
|
||||
func DirectError(code string) UploadError {
|
||||
switch code {
|
||||
case "upload_not_uploaded":
|
||||
return UploadError{409, code, "file has not been uploaded"}
|
||||
case "upload_expired":
|
||||
return UploadError{409, code, "upload confirmation period has expired"}
|
||||
case "upload_size_mismatch":
|
||||
return UploadError{400, code, "uploaded size differs from declared size"}
|
||||
case "upload_checksum_mismatch":
|
||||
return UploadError{400, code, "uploaded SHA-256 differs from declared checksum"}
|
||||
case "payload_too_large":
|
||||
return UploadError{413, code, "image exceeds upload limits"}
|
||||
default:
|
||||
return UploadError{415, "unsupported_media_type", "a valid PNG or JPEG matching Content-Type is required"}
|
||||
}
|
||||
}
|
||||
|
||||
type DirectUpload struct {
|
||||
ID string `json:"upload_id"`
|
||||
AttachmentID string `json:"-"`
|
||||
OwnerID string `json:"-"`
|
||||
ChatID string `json:"-"`
|
||||
Filename string `json:"-"`
|
||||
ContentType string `json:"-"`
|
||||
SizeBytes int64 `json:"-"`
|
||||
SHA256 string `json:"-"`
|
||||
StagingKey string `json:"-"`
|
||||
FinalKey string `json:"-"`
|
||||
State string `json:"-"`
|
||||
ErrorCode *string `json:"-"`
|
||||
ExpiresAt time.Time `json:"expires_at"`
|
||||
CompleteBefore time.Time `json:"complete_before"`
|
||||
}
|
||||
|
||||
const directColumns = `d.id,d.attachment_id,d.owner_id,d.chat_id,d.filename,d.content_type,d.size_bytes,d.sha256,d.staging_key,d.final_key,d.state,d.error_code,d.expires_at,d.complete_before`
|
||||
|
||||
func scanDirect(row pgx.Row) (DirectUpload, error) {
|
||||
var u DirectUpload
|
||||
err := row.Scan(&u.ID, &u.AttachmentID, &u.OwnerID, &u.ChatID, &u.Filename, &u.ContentType, &u.SizeBytes, &u.SHA256, &u.StagingKey, &u.FinalKey, &u.State, &u.ErrorCode, &u.ExpiresAt, &u.CompleteBefore)
|
||||
return u, err
|
||||
}
|
||||
|
||||
func (s *AttachmentStore) CreateDirect(ctx context.Context, u DirectUpload) (DirectUpload, error) {
|
||||
ctx, cancel := boundedQuery(ctx)
|
||||
defer cancel()
|
||||
return scanDirect(s.db.QueryRow(ctx, `INSERT INTO direct_uploads AS d(id,attachment_id,owner_id,chat_id,filename,content_type,size_bytes,sha256,staging_key,final_key) VALUES($1,$2,$3,$4,$5,$6,$7,$8,$9,$10) RETURNING `+directColumns, u.ID, u.AttachmentID, u.OwnerID, u.ChatID, u.Filename, u.ContentType, u.SizeBytes, u.SHA256, u.StagingKey, u.FinalKey))
|
||||
}
|
||||
|
||||
func (s *AttachmentStore) GetDirect(ctx context.Context, id, chat, user string) (DirectUpload, error) {
|
||||
ctx, cancel := boundedQuery(ctx)
|
||||
defer cancel()
|
||||
return scanDirect(s.db.QueryRow(ctx, `SELECT `+directColumns+` FROM direct_uploads d JOIN chat_members cm ON cm.chat_id=d.chat_id AND cm.user_id=$3 WHERE d.id=$1 AND d.chat_id=$2 AND d.owner_id=$3`, id, chat, user))
|
||||
}
|
||||
|
||||
// The session lock serializes complete across replicas. A separately committed
|
||||
// reservation survives rollback/crash, while its row lock protects the final key.
|
||||
func (s *AttachmentStore) CompleteDirect(ctx context.Context, id, chat, user string, validate func(context.Context, DirectUpload) error) (Attachment, error) {
|
||||
var a Attachment
|
||||
tx, err := s.db.Begin(ctx)
|
||||
if err != nil {
|
||||
return a, err
|
||||
}
|
||||
defer tx.Rollback(context.Background())
|
||||
u, err := scanDirect(tx.QueryRow(ctx, `SELECT `+directColumns+` FROM direct_uploads d JOIN chat_members cm ON cm.chat_id=d.chat_id AND cm.user_id=$3 WHERE d.id=$1 AND d.chat_id=$2 AND d.owner_id=$3 FOR UPDATE OF d`, id, chat, user))
|
||||
if err != nil {
|
||||
return a, err
|
||||
}
|
||||
if u.StagingKey != "staging/direct/"+u.ID || u.FinalKey != "originals/"+u.AttachmentID {
|
||||
return a, errors.New("invalid direct upload object keys")
|
||||
}
|
||||
if u.State == "ready" {
|
||||
return s.GetForMember(ctx, u.AttachmentID, user)
|
||||
}
|
||||
if u.State == "rejected" && u.ErrorCode != nil {
|
||||
return a, DirectError(*u.ErrorCode)
|
||||
}
|
||||
var expired bool
|
||||
if err = tx.QueryRow(ctx, `SELECT clock_timestamp()>=$1`, u.CompleteBefore).Scan(&expired); err != nil {
|
||||
return a, err
|
||||
}
|
||||
if expired || u.State == "expired" {
|
||||
return a, DirectError("upload_expired")
|
||||
}
|
||||
grace, err := UploadCleanupGrace()
|
||||
if err != nil {
|
||||
return a, err
|
||||
}
|
||||
// ON CONFLICT handles retry after an interrupted COPY without losing tracking.
|
||||
_, err = s.db.Exec(ctx, `INSERT INTO upload_attempts(id,object_key,cleanup_after) VALUES($1,$2,clock_timestamp()+$3*interval '1 second') ON CONFLICT(id) DO UPDATE SET cleanup_after=GREATEST(upload_attempts.cleanup_after,EXCLUDED.cleanup_after) WHERE upload_attempts.object_key=EXCLUDED.object_key`, u.AttachmentID, u.FinalKey, grace.Seconds())
|
||||
if err != nil {
|
||||
return a, err
|
||||
}
|
||||
var key string
|
||||
if err = tx.QueryRow(ctx, `SELECT object_key FROM upload_attempts WHERE id=$1 AND cleanup_after>clock_timestamp() FOR UPDATE`, u.AttachmentID).Scan(&key); err != nil {
|
||||
return a, err
|
||||
}
|
||||
if key != u.FinalKey {
|
||||
return a, errors.New("direct upload reservation mismatch")
|
||||
}
|
||||
if err = validate(ctx, u); err != nil {
|
||||
var invalid UploadError
|
||||
if errors.As(err, &invalid) && (invalid.Status == 400 || invalid.Status == 413 || invalid.Status == 415) {
|
||||
_, saveErr := tx.Exec(ctx, `UPDATE direct_uploads SET state='rejected',error_code=$2,cleanup_after=clock_timestamp(),purge_after=clock_timestamp()+interval '24 hours' WHERE id=$1`, id, invalid.Code)
|
||||
if saveErr != nil {
|
||||
return a, saveErr
|
||||
}
|
||||
if saveErr = tx.Commit(ctx); saveErr != nil {
|
||||
return a, saveErr
|
||||
}
|
||||
}
|
||||
return a, err
|
||||
}
|
||||
now := time.Now().UTC().Truncate(time.Microsecond)
|
||||
a = Attachment{ID: u.AttachmentID, ChatID: u.ChatID, OwnerID: u.OwnerID, Filename: u.Filename, ContentType: u.ContentType, SizeBytes: u.SizeBytes, SHA256: u.SHA256, ObjectKey: u.FinalKey, Status: "queued", Variants: []any{}, CreatedAt: now, UpdatedAt: now}
|
||||
_, err = tx.Exec(ctx, `INSERT INTO attachments(id,chat_id,owner_id,filename,object_key,content_type,size_bytes,sha256,created_at,updated_at) VALUES($1,$2,$3,$4,$5,$6,$7,$8,$9,$10)`, a.ID, a.ChatID, a.OwnerID, a.Filename, a.ObjectKey, a.ContentType, a.SizeBytes, a.SHA256, a.CreatedAt, a.UpdatedAt)
|
||||
if err != nil {
|
||||
return Attachment{}, err
|
||||
}
|
||||
if err = enqueueJob(ctx, tx, a, a.ObjectKey); err != nil {
|
||||
return Attachment{}, err
|
||||
}
|
||||
if _, err = tx.Exec(ctx, `UPDATE direct_uploads SET state='ready',cleanup_after=clock_timestamp(),purge_after=clock_timestamp()+interval '24 hours' WHERE id=$1`, id); err != nil {
|
||||
return Attachment{}, err
|
||||
}
|
||||
if _, err = tx.Exec(ctx, `DELETE FROM upload_attempts WHERE id=$1`, a.ID); err != nil {
|
||||
return Attachment{}, err
|
||||
}
|
||||
if err = tx.Commit(ctx); err != nil {
|
||||
return Attachment{}, err
|
||||
}
|
||||
return a, nil
|
||||
}
|
||||
@@ -0,0 +1,21 @@
|
||||
package repositories
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"github.com/jackc/pgx/v5"
|
||||
"github.com/jackc/pgx/v5/pgconn"
|
||||
)
|
||||
|
||||
func NotFound(err error) bool {
|
||||
return errors.Is(err, pgx.ErrNoRows)
|
||||
}
|
||||
|
||||
func CheckViolation(err error) bool {
|
||||
var pgerr *pgconn.PgError
|
||||
return errors.As(err, &pgerr) && pgerr.Code == "23514"
|
||||
}
|
||||
|
||||
func UniqueViolation(err error) bool {
|
||||
var pgerr *pgconn.PgError
|
||||
return errors.As(err, &pgerr) && pgerr.Code == "23505"
|
||||
}
|
||||
@@ -0,0 +1,21 @@
|
||||
package repositories
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
)
|
||||
|
||||
type HealthStore struct {
|
||||
db *pgxpool.Pool
|
||||
}
|
||||
|
||||
func NewHealthStore(db *pgxpool.Pool) *HealthStore {
|
||||
return &HealthStore{db: db}
|
||||
}
|
||||
|
||||
func (s *HealthStore) Ping(ctx context.Context) error {
|
||||
ctx, cancel := boundedQuery(ctx)
|
||||
defer cancel()
|
||||
return s.db.Ping(ctx)
|
||||
}
|
||||
@@ -0,0 +1,95 @@
|
||||
package repositories
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"reflect"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/jackc/pgx/v5"
|
||||
)
|
||||
|
||||
var ErrAttachmentNotFound = errors.New("attachment not found")
|
||||
var ErrAttachmentFailed = errors.New("attachment failed")
|
||||
|
||||
var ErrIdempotencyConflict = errors.New("idempotency key belongs to a different message")
|
||||
|
||||
// CreateIdempotent commits the message and its replay record together.
|
||||
func (s *MessageStore) CreateIdempotent(ctx context.Context, chatID, senderID, text, key string, attachments ...[]string) (Message, error) {
|
||||
ids := []string{}
|
||||
if len(attachments) > 0 && attachments[0] != nil {
|
||||
ids = attachments[0]
|
||||
}
|
||||
ctx, cancel := boundedQuery(ctx)
|
||||
defer cancel()
|
||||
tx, err := s.db.Begin(ctx)
|
||||
if err != nil {
|
||||
return Message{}, err
|
||||
}
|
||||
defer tx.Rollback(context.Background())
|
||||
|
||||
// The transaction-scoped lock is shared by all API replicas. Hash collisions
|
||||
// only serialize unrelated requests; the full scope is checked by the query.
|
||||
scope := senderID + ":POST:/api/v1/chats/" + chatID + "/messages:" + key
|
||||
if _, err = tx.Exec(ctx, "SELECT pg_advisory_xact_lock(hashtextextended($1, 0))", scope); err != nil {
|
||||
return Message{}, err
|
||||
}
|
||||
// A shared row lock prevents failure publication racing with a new link.
|
||||
hasFailed := false
|
||||
for _, id := range ids {
|
||||
var status string
|
||||
err = tx.QueryRow(ctx, "SELECT status FROM attachments WHERE id=$1 AND chat_id=$2 FOR SHARE", id, chatID).Scan(&status)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return Message{}, ErrAttachmentNotFound
|
||||
}
|
||||
if err != nil {
|
||||
return Message{}, err
|
||||
}
|
||||
hasFailed = hasFailed || status == "failed"
|
||||
}
|
||||
|
||||
var message Message
|
||||
err = tx.QueryRow(ctx, `
|
||||
SELECT m.id, m.chat_id, m.sender_id, m.text, m.created_at, ARRAY(SELECT attachment_id::text FROM message_attachments ma WHERE ma.message_id=m.id ORDER BY position)
|
||||
FROM message_idempotency i JOIN messages m ON m.id = i.message_id
|
||||
WHERE i.sender_id=$1 AND i.chat_id=$2 AND i.key=$3
|
||||
AND i.expires_at > clock_timestamp()`, senderID, chatID, key).
|
||||
Scan(&message.ID, &message.ChatID, &message.SenderID, &message.Text, &message.CreatedAt, &message.AttachmentIDs)
|
||||
if err == nil {
|
||||
message.CreatedAt = message.CreatedAt.UTC()
|
||||
if message.Text != text || !reflect.DeepEqual(message.AttachmentIDs, ids) {
|
||||
return Message{}, ErrIdempotencyConflict
|
||||
}
|
||||
return message, tx.Commit(ctx)
|
||||
}
|
||||
if !errors.Is(err, pgx.ErrNoRows) {
|
||||
return Message{}, err
|
||||
}
|
||||
|
||||
if hasFailed {
|
||||
return Message{}, ErrAttachmentFailed
|
||||
}
|
||||
|
||||
// Expiry permits key reuse; it never deletes the original message.
|
||||
if _, err = tx.Exec(ctx, "DELETE FROM message_idempotency WHERE sender_id=$1 AND chat_id=$2 AND key=$3", senderID, chatID, key); err != nil {
|
||||
return Message{}, err
|
||||
}
|
||||
message = Message{
|
||||
AttachmentIDs: ids, ID: uuid.NewString(), ChatID: chatID, SenderID: senderID, Text: text,
|
||||
CreatedAt: time.Now().UTC().Truncate(time.Microsecond),
|
||||
}
|
||||
if _, err = tx.Exec(ctx, "INSERT INTO messages(id,chat_id,sender_id,text,created_at) VALUES($1,$2,$3,$4,$5)", message.ID, chatID, senderID, text, message.CreatedAt); err != nil {
|
||||
return Message{}, err
|
||||
}
|
||||
for position, id := range ids {
|
||||
if _, err = tx.Exec(ctx, "INSERT INTO message_attachments(message_id,attachment_id,position) VALUES($1,$2,$3)", message.ID, id, position); err != nil {
|
||||
return Message{}, err
|
||||
}
|
||||
}
|
||||
if _, err = tx.Exec(ctx, `INSERT INTO message_idempotency(sender_id,chat_id,key,message_id,expires_at)
|
||||
VALUES($1,$2,$3,$4,clock_timestamp() + $5 * interval '1 second')`, senderID, chatID, key, message.ID, s.idempotencyTTL.Seconds()); err != nil {
|
||||
return Message{}, err
|
||||
}
|
||||
return message, tx.Commit(ctx)
|
||||
}
|
||||
@@ -0,0 +1,142 @@
|
||||
package repositories
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"github.com/jackc/pgx/v5"
|
||||
"strings"
|
||||
"time"
|
||||
"unicode/utf8"
|
||||
)
|
||||
|
||||
type FailedJob struct {
|
||||
ID string `json:"job_id"`
|
||||
AttachmentID string `json:"attachment_id"`
|
||||
Code string `json:"error_code"`
|
||||
Attempts int `json:"attempts"`
|
||||
FailedAt time.Time `json:"failed_at"`
|
||||
}
|
||||
|
||||
func (s *AttachmentStore) FailedJobs(ctx context.Context) ([]FailedJob, error) {
|
||||
rows, err := s.db.Query(ctx, `SELECT j.id,j.attachment_id,f.error_code,f.attempts,f.failed_at FROM attachment_job_failures f JOIN attachment_jobs j ON j.id=f.job_id JOIN attachments a ON a.id=j.attachment_id WHERE a.status='failed' AND j.pipeline_version=a.pipeline_version AND j.superseded_at IS NULL ORDER BY f.failed_at DESC LIMIT 100`)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
result := []FailedJob{}
|
||||
for rows.Next() {
|
||||
var j FailedJob
|
||||
if err = rows.Scan(&j.ID, &j.AttachmentID, &j.Code, &j.Attempts, &j.FailedAt); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
result = append(result, j)
|
||||
}
|
||||
return result, rows.Err()
|
||||
}
|
||||
|
||||
func (s *AttachmentStore) ReplayJob(ctx context.Context, id, reason string, exists func(context.Context, string) error) (bool, error) {
|
||||
reason = strings.TrimSpace(reason)
|
||||
if reason == "" || utf8.RuneCountInString(reason) > 200 {
|
||||
return false, errors.New("replay reason must contain 1-200 characters")
|
||||
}
|
||||
tx, err := s.db.Begin(ctx)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
defer tx.Rollback(context.Background())
|
||||
var status, key, attachment string
|
||||
var deleted, obsolete bool
|
||||
err = tx.QueryRow(ctx, `SELECT a.status,j.staging_key,a.id,j.staging_deleted_at IS NOT NULL,j.pipeline_version<>a.pipeline_version OR j.superseded_at IS NOT NULL FROM attachment_jobs j JOIN attachments a ON a.id=j.attachment_id WHERE j.id=$1 FOR UPDATE OF j,a`, id).Scan(&status, &key, &attachment, &deleted, &obsolete)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
if obsolete || status != "failed" {
|
||||
return false, nil
|
||||
}
|
||||
if deleted {
|
||||
return false, errors.New("staging has expired; replay unavailable")
|
||||
}
|
||||
if key != "staging/jobs/"+attachment && key != "originals/"+attachment {
|
||||
return false, errors.New("invalid staging key")
|
||||
}
|
||||
if err = exists(ctx, key); err != nil {
|
||||
return false, errors.New("staging unavailable; replay not scheduled")
|
||||
}
|
||||
if _, err = tx.Exec(ctx, `INSERT INTO attachment_job_replays(job_id,reason) VALUES($1,$2)`, id, reason); err != nil {
|
||||
return false, err
|
||||
}
|
||||
if _, err = tx.Exec(ctx, `UPDATE attachment_jobs SET attempts=0,generation=generation+1,next_attempt_at=now(),lease_until=NULL,lease_token=NULL,published_at=NULL WHERE id=$1`, id); err != nil {
|
||||
return false, err
|
||||
}
|
||||
if _, err = tx.Exec(ctx, `UPDATE attachments SET status='queued',error_code=NULL,updated_at=now() WHERE id=$1`, attachment); err != nil {
|
||||
return false, err
|
||||
}
|
||||
return true, tx.Commit(ctx)
|
||||
}
|
||||
|
||||
// Replay and cleanup lock the same rows. A ready source is removed only after
|
||||
// final publication, failed sources expire after retention; queued work is safe.
|
||||
func (s *AttachmentStore) CleanupJobStaging(ctx context.Context, id string, retention time.Duration, dryRun bool, remove func(context.Context, string) error) (bool, error) {
|
||||
tx, err := s.db.Begin(ctx)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
defer tx.Rollback(context.Background())
|
||||
var key, attachment, status string
|
||||
var eligible bool
|
||||
err = tx.QueryRow(ctx, `SELECT j.staging_key,a.id,a.status,j.staging_deleted_at IS NULL AND (a.status='ready' AND jsonb_array_length(a.variants)>=3 OR a.status='failed' AND j.pipeline_version=a.pipeline_version AND COALESCE(f.failed_at,a.updated_at)<now()-$2*interval '1 second') FROM attachment_jobs j JOIN attachments a ON a.id=j.attachment_id LEFT JOIN attachment_job_failures f ON f.job_id=j.id WHERE j.id=$1 FOR UPDATE OF j,a SKIP LOCKED`, id, retention.Seconds()).Scan(&key, &attachment, &status, &eligible)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return false, nil
|
||||
}
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
if !eligible || key == "originals/"+attachment {
|
||||
return false, nil
|
||||
}
|
||||
if key != "staging/jobs/"+attachment {
|
||||
return false, errors.New("invalid staging key")
|
||||
}
|
||||
if dryRun {
|
||||
return true, nil
|
||||
}
|
||||
if err = remove(ctx, key); err != nil {
|
||||
return false, err
|
||||
}
|
||||
if _, err = tx.Exec(ctx, `UPDATE attachment_jobs SET staging_deleted_at=now() WHERE id=$1`, id); err != nil {
|
||||
return false, err
|
||||
}
|
||||
return true, tx.Commit(ctx)
|
||||
}
|
||||
|
||||
func (s *AttachmentStore) CleanupJobs(ctx context.Context, retention time.Duration, dryRun bool, remove func(context.Context, string) error) (int, error) {
|
||||
rows, err := s.db.Query(ctx, `SELECT j.id FROM attachment_jobs j JOIN attachments a ON a.id=j.attachment_id LEFT JOIN attachment_job_failures f ON f.job_id=j.id WHERE j.staging_deleted_at IS NULL AND j.staging_key='staging/jobs/'||j.attachment_id::text AND (a.status='ready' AND jsonb_array_length(a.variants)>=3 OR a.status='failed' AND j.pipeline_version=a.pipeline_version AND COALESCE(f.failed_at,a.updated_at)<now()-$1*interval '1 second') ORDER BY j.created_at LIMIT 100`, retention.Seconds())
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
ids := []string{}
|
||||
for rows.Next() {
|
||||
var id string
|
||||
if err = rows.Scan(&id); err != nil {
|
||||
rows.Close()
|
||||
return 0, err
|
||||
}
|
||||
ids = append(ids, id)
|
||||
}
|
||||
err = rows.Err()
|
||||
rows.Close()
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
count := 0
|
||||
for _, id := range ids {
|
||||
done, e := s.CleanupJobStaging(ctx, id, retention, dryRun, remove)
|
||||
if e != nil {
|
||||
return count, e
|
||||
}
|
||||
if done {
|
||||
count++
|
||||
}
|
||||
}
|
||||
return count, nil
|
||||
}
|
||||
@@ -0,0 +1,232 @@
|
||||
package repositories
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"math/big"
|
||||
"time"
|
||||
|
||||
"messenger/src/backend/telemetry"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/jackc/pgx/v5"
|
||||
"go.opentelemetry.io/otel/trace"
|
||||
)
|
||||
|
||||
type Variant struct {
|
||||
Name string `json:"name"`
|
||||
ContentType string `json:"content_type"`
|
||||
SizeBytes int64 `json:"size_bytes"`
|
||||
Width int `json:"width"`
|
||||
Height int `json:"height"`
|
||||
SHA256 string `json:"sha256"`
|
||||
}
|
||||
type Job struct {
|
||||
SchemaVersion int `json:"schema_version"`
|
||||
ID string `json:"job_id"`
|
||||
Type string `json:"type"`
|
||||
AttachmentID string `json:"attachment_id"`
|
||||
PipelineVersion int `json:"pipeline_version"`
|
||||
Attempt int `json:"attempt"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
Traceparent string `json:"-"`
|
||||
Tracestate string `json:"-"`
|
||||
}
|
||||
|
||||
// QueuedJobs is also the recovery source when publish/confirm is uncertain.
|
||||
// Re-publication is intentional: a durable intent is never removed by publish.
|
||||
func (s *AttachmentStore) QueuedJobs(ctx context.Context) ([]Job, error) {
|
||||
rows, err := s.db.Query(ctx, `SELECT j.id,j.attachment_id,j.pipeline_version,j.attempts+1,j.created_at,j.traceparent,j.tracestate FROM attachment_jobs j JOIN attachments a ON a.id=j.attachment_id WHERE j.superseded_at IS NULL AND j.pipeline_version=a.pipeline_version AND a.status IN ('queued','processing') AND j.next_attempt_at<=now() AND (j.lease_until IS NULL OR j.lease_until<now()) AND (j.published_at IS NULL OR j.published_at<now()-interval '30 seconds') ORDER BY j.created_at LIMIT 100`)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
jobs := []Job{}
|
||||
for rows.Next() {
|
||||
j := Job{SchemaVersion: 1, Type: "attachment.process.v1"}
|
||||
if err = rows.Scan(&j.ID, &j.AttachmentID, &j.PipelineVersion, &j.Attempt, &j.CreatedAt, &j.Traceparent, &j.Tracestate); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
j.CreatedAt = j.CreatedAt.UTC()
|
||||
jobs = append(jobs, j)
|
||||
}
|
||||
return jobs, rows.Err()
|
||||
}
|
||||
func (s *AttachmentStore) MarkPublished(ctx context.Context, id string) error {
|
||||
_, err := s.db.Exec(ctx, `UPDATE attachment_jobs SET published_at=now() WHERE id=$1`, id)
|
||||
return err
|
||||
}
|
||||
|
||||
// StartAttempt commits the lease and attempt before decoding. A token fences a
|
||||
// consumer that was paused between claim and processing past its lease deadline.
|
||||
func (s *AttachmentStore) StartAttempt(ctx context.Context, j Job, timeout time.Duration, maxAttempts int) (string, error) {
|
||||
tx, err := s.db.Begin(ctx)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer tx.Rollback(context.Background())
|
||||
var status string
|
||||
var attempts, generation int
|
||||
var available bool
|
||||
err = tx.QueryRow(ctx, `SELECT a.status,j.attempts,j.generation,j.next_attempt_at<=now() AND (j.lease_until IS NULL OR j.lease_until<now()) FROM attachment_jobs j JOIN attachments a ON a.id=j.attachment_id WHERE j.id=$1 AND j.attachment_id=$2 AND j.pipeline_version=$3 AND j.superseded_at IS NULL AND a.pipeline_version=j.pipeline_version FOR UPDATE OF j,a SKIP LOCKED`, j.ID, j.AttachmentID, j.PipelineVersion).Scan(&status, &attempts, &generation, &available)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return "", nil
|
||||
}
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if status == "ready" || status == "failed" || !available {
|
||||
return "", nil
|
||||
}
|
||||
// A crashed attempt remains visible in the audit trail.
|
||||
if _, err = tx.Exec(ctx, `UPDATE attachment_job_attempts SET finished_at=clock_timestamp(),error_code='processing_timeout' WHERE job_id=$1 AND finished_at IS NULL`, j.ID); err != nil {
|
||||
return "", err
|
||||
}
|
||||
if attempts >= maxAttempts {
|
||||
if err = failJob(ctx, tx, j, "retry_exhausted", attempts); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return "", tx.Commit(ctx)
|
||||
}
|
||||
token := uuid.NewString()
|
||||
_, err = tx.Exec(ctx, `UPDATE attachment_jobs SET attempts=attempts+1,lease_token=$2,lease_until=now()+$3*interval '1 second' WHERE id=$1`, j.ID, token, timeout.Seconds()+5)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if _, err = tx.Exec(ctx, `UPDATE attachments SET status='processing',updated_at=now() WHERE id=$1`, j.AttachmentID); err != nil {
|
||||
return "", err
|
||||
}
|
||||
if _, err = tx.Exec(ctx, `INSERT INTO attachment_job_attempts(job_id,generation,attempt) VALUES($1,$2,$3)`, j.ID, generation, attempts+1); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return token, tx.Commit(ctx)
|
||||
}
|
||||
|
||||
var ErrStaleJob = errors.New("stale job lease or version")
|
||||
|
||||
var ErrProcessingTimeout = errors.New("processing timeout")
|
||||
|
||||
// ProcessJob holds the row lock across deterministic object writes. The lock and
|
||||
// token check prevent overlapping/stale attempts from publishing metadata.
|
||||
func (s *AttachmentStore) ProcessJob(ctx context.Context, j Job, token string, maxAttempts int, process func(Attachment, string) ([]Variant, string, error)) (result error) {
|
||||
ctx, finish := telemetry.Operation(ctx, "postgresql finalize attachment", trace.SpanKindClient)
|
||||
defer func() { finish(result) }()
|
||||
tx, err := s.db.Begin(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer tx.Rollback(context.Background())
|
||||
var staging string
|
||||
var attempts, generation int
|
||||
var a Attachment
|
||||
err = tx.QueryRow(ctx, `SELECT a.id,a.object_key,a.content_type,a.size_bytes,a.sha256,a.status,j.staging_key,j.attempts,j.generation FROM attachment_jobs j JOIN attachments a ON a.id=j.attachment_id WHERE j.id=$1 AND j.attachment_id=$2 AND j.pipeline_version=$3 AND j.lease_token=$4 AND j.lease_until>now() AND j.superseded_at IS NULL AND a.pipeline_version=j.pipeline_version FOR UPDATE OF j,a`, j.ID, j.AttachmentID, j.PipelineVersion, token).Scan(&a.ID, &a.ObjectKey, &a.ContentType, &a.SizeBytes, &a.SHA256, &a.Status, &staging, &attempts, &generation)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return ErrStaleJob
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if a.Status != "processing" {
|
||||
return errors.New("job is not processing")
|
||||
}
|
||||
a.PipelineVersion = j.PipelineVersion
|
||||
variants, code, workErr := process(a, staging)
|
||||
var next *time.Time
|
||||
var auditCode *string
|
||||
if workErr == nil {
|
||||
data, e := json.Marshal(variants)
|
||||
if e != nil {
|
||||
return e
|
||||
}
|
||||
_, err = tx.Exec(ctx, `UPDATE attachments SET status='ready',variants=$2,published_pipeline_version=$3,error_code=NULL,updated_at=now() WHERE id=$1 AND pipeline_version=$3`, a.ID, data, j.PipelineVersion)
|
||||
if err == nil {
|
||||
_, err = tx.Exec(ctx, `INSERT INTO attachment_publications(attachment_id,job_id,pipeline_version,variants) VALUES($1,$2,$3,$4)`, a.ID, j.ID, j.PipelineVersion, data)
|
||||
}
|
||||
} else {
|
||||
reason := "temporary_failure"
|
||||
if errors.Is(workErr, ErrProcessingTimeout) {
|
||||
reason = "processing_timeout"
|
||||
}
|
||||
if code != "" {
|
||||
reason = code
|
||||
}
|
||||
auditCode = &reason
|
||||
if code != "" || attempts >= maxAttempts {
|
||||
if code == "" {
|
||||
code = "retry_exhausted"
|
||||
}
|
||||
err = failJob(ctx, tx, j, code, attempts)
|
||||
} else {
|
||||
due := time.Now().UTC().Add(RetryDelay(attempts))
|
||||
next = &due
|
||||
_, err = tx.Exec(ctx, `UPDATE attachments SET status='queued',error_code=NULL,updated_at=now() WHERE id=$1`, a.ID)
|
||||
}
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = tx.Exec(ctx, `UPDATE attachment_job_attempts SET finished_at=clock_timestamp(),error_code=$4,next_attempt_at=$5 WHERE job_id=$1 AND generation=$2 AND attempt=$3`, j.ID, generation, attempts, auditCode, next)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = tx.Exec(ctx, `UPDATE attachment_jobs SET lease_until=NULL,lease_token=NULL,published_at=NULL,next_attempt_at=COALESCE($2,now()) WHERE id=$1`, j.ID, next)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return tx.Commit(ctx)
|
||||
}
|
||||
|
||||
func failJob(ctx context.Context, tx pgx.Tx, j Job, code string, attempts int) error {
|
||||
if _, err := tx.Exec(ctx, `UPDATE attachments SET status='failed',error_code=$2,updated_at=now() WHERE id=$1`, j.AttachmentID, code); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := tx.Exec(ctx, `UPDATE attachment_jobs SET lease_token=NULL,lease_until=NULL WHERE id=$1`, j.ID); err != nil {
|
||||
return err
|
||||
}
|
||||
_, err := tx.Exec(ctx, `INSERT INTO attachment_job_failures(job_id,error_code,attempts) VALUES($1,$2,$3) ON CONFLICT(job_id) DO UPDATE SET error_code=EXCLUDED.error_code,attempts=EXCLUDED.attempts,failed_at=clock_timestamp()`, j.ID, code, attempts)
|
||||
return err
|
||||
}
|
||||
|
||||
// Equal jitter: exponential ceiling 2,4,8,...,30 seconds; delay in [ceiling/2,ceiling).
|
||||
func RetryDelay(attempt int) time.Duration {
|
||||
ceiling := 2 * time.Second
|
||||
for i := 1; i < attempt && ceiling < 30*time.Second; i++ {
|
||||
ceiling *= 2
|
||||
}
|
||||
if ceiling > 30*time.Second {
|
||||
ceiling = 30 * time.Second
|
||||
}
|
||||
n, err := rand.Int(rand.Reader, big.NewInt(int64(ceiling/2)))
|
||||
if err != nil {
|
||||
return ceiling
|
||||
}
|
||||
return ceiling/2 + time.Duration(n.Int64())
|
||||
}
|
||||
|
||||
func enqueueJob(ctx context.Context, tx pgx.Tx, a Attachment, staging string) error {
|
||||
_, err := tx.Exec(ctx, `UPDATE attachments SET status='queued' WHERE id=$1`, a.ID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
headers := telemetry.TraceHeaders(ctx)
|
||||
_, err = tx.Exec(ctx, `INSERT INTO attachment_jobs(id,attachment_id,staging_key,traceparent,tracestate) VALUES($1,$2,$3,$4,$5)`, uuid.NewString(), a.ID, staging, headers["traceparent"], headers["tracestate"])
|
||||
return err
|
||||
}
|
||||
|
||||
func (s *AttachmentStore) JobTerminal(ctx context.Context, id string) (bool, error) {
|
||||
var done bool
|
||||
err := s.db.QueryRow(ctx, `SELECT j.superseded_at IS NOT NULL OR j.pipeline_version<a.pipeline_version OR a.status IN ('ready','failed') FROM attachment_jobs j JOIN attachments a ON a.id=j.attachment_id WHERE j.id=$1`, id).Scan(&done)
|
||||
return done, err
|
||||
}
|
||||
|
||||
// PendingJobs counts durable outstanding work, including leases and scheduled retries.
|
||||
func (s *AttachmentStore) PendingJobs(ctx context.Context) (int64, float64, error) {
|
||||
var count int64
|
||||
var age float64
|
||||
err := s.db.QueryRow(ctx, `SELECT count(*), COALESCE(EXTRACT(EPOCH FROM now()-min(j.created_at)),0)::double precision
|
||||
FROM attachment_jobs j JOIN attachments a ON a.id=j.attachment_id
|
||||
WHERE j.superseded_at IS NULL AND j.pipeline_version=a.pipeline_version AND a.status IN ('queued','processing')`).Scan(&count, &age)
|
||||
return count, age, err
|
||||
}
|
||||
@@ -0,0 +1,89 @@
|
||||
package repositories
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
)
|
||||
|
||||
type Message struct {
|
||||
ID string `json:"id"`
|
||||
ChatID string `json:"chat_id"`
|
||||
SenderID string `json:"sender_id"`
|
||||
Text string `json:"text"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
AttachmentIDs []string `json:"attachment_ids"`
|
||||
}
|
||||
|
||||
type MessageStore struct {
|
||||
db *pgxpool.Pool
|
||||
idempotencyTTL time.Duration
|
||||
}
|
||||
|
||||
func NewMessageStore(db *pgxpool.Pool) *MessageStore {
|
||||
return NewMessageStoreWithTTL(db, 24*time.Hour)
|
||||
}
|
||||
|
||||
func NewMessageStoreWithTTL(db *pgxpool.Pool, ttl time.Duration) *MessageStore {
|
||||
return &MessageStore{db: db, idempotencyTTL: ttl}
|
||||
}
|
||||
|
||||
func (s *MessageStore) Create(
|
||||
ctx context.Context,
|
||||
chatID string,
|
||||
senderID string,
|
||||
text string,
|
||||
) (Message, error) {
|
||||
ctx, cancel := boundedQuery(ctx)
|
||||
defer cancel()
|
||||
message := Message{
|
||||
AttachmentIDs: []string{},
|
||||
ID: uuid.New().String(),
|
||||
ChatID: chatID,
|
||||
SenderID: senderID,
|
||||
Text: text,
|
||||
CreatedAt: time.Now().UTC().Truncate(time.Microsecond),
|
||||
}
|
||||
|
||||
_, err := s.db.Exec(ctx, "INSERT INTO messages(id,chat_id,sender_id,text,created_at) VALUES($1,$2,$3,$4,$5)", message.ID, message.ChatID, message.SenderID, message.Text, message.CreatedAt)
|
||||
return message, err
|
||||
}
|
||||
|
||||
func (s *MessageStore) ListPageByChat(
|
||||
ctx context.Context,
|
||||
chatID string,
|
||||
cursor *MessageCursor,
|
||||
limit int,
|
||||
) ([]Message, bool, error) {
|
||||
ctx, cancel := boundedQuery(ctx)
|
||||
defer cancel()
|
||||
afterTime := time.Time{}
|
||||
afterID := "00000000-0000-0000-0000-000000000000"
|
||||
if cursor != nil {
|
||||
afterTime = cursor.CreatedAt
|
||||
afterID = cursor.MessageID
|
||||
}
|
||||
rows, err := s.db.Query(ctx, `SELECT id,chat_id,sender_id,text,created_at,ARRAY(SELECT attachment_id::text FROM message_attachments ma WHERE ma.message_id=messages.id ORDER BY position) FROM messages WHERE chat_id=$1 AND (created_at,id)>($2,$3::uuid) ORDER BY created_at,id LIMIT $4`, chatID, afterTime, afterID, limit+1)
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
defer rows.Close()
|
||||
items := []Message{}
|
||||
for rows.Next() {
|
||||
var m Message
|
||||
if err = rows.Scan(&m.ID, &m.ChatID, &m.SenderID, &m.Text, &m.CreatedAt, &m.AttachmentIDs); err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
items = append(items, m)
|
||||
}
|
||||
if err = rows.Err(); err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
more := len(items) > limit
|
||||
if more {
|
||||
items = items[:limit]
|
||||
}
|
||||
return items, more, nil
|
||||
}
|
||||
@@ -0,0 +1,5 @@
|
||||
DROP TABLE IF EXISTS sessions;
|
||||
DROP TABLE IF EXISTS messages;
|
||||
DROP TABLE IF EXISTS chat_members;
|
||||
DROP TABLE IF EXISTS chats;
|
||||
DROP TABLE IF EXISTS users;
|
||||
@@ -0,0 +1,32 @@
|
||||
CREATE TABLE IF NOT EXISTS users (
|
||||
id uuid PRIMARY KEY,
|
||||
username varchar(32) NOT NULL UNIQUE CHECK (username ~ '^[a-z0-9_]{3,32}$'),
|
||||
display_name text NOT NULL CHECK (char_length(display_name) BETWEEN 1 AND 100),
|
||||
password_hash bytea NOT NULL,
|
||||
created_at timestamptz NOT NULL
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS chats (
|
||||
id uuid PRIMARY KEY,
|
||||
title text NOT NULL CHECK (char_length(title) BETWEEN 1 AND 100),
|
||||
created_at timestamptz NOT NULL
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS chat_members (
|
||||
chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE,
|
||||
user_id uuid NOT NULL REFERENCES users(id),
|
||||
PRIMARY KEY (chat_id,user_id)
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS chat_members_user_idx ON chat_members(user_id,chat_id);
|
||||
CREATE TABLE IF NOT EXISTS messages (
|
||||
id uuid PRIMARY KEY,
|
||||
chat_id uuid NOT NULL REFERENCES chats(id),
|
||||
sender_id uuid NOT NULL REFERENCES users(id),
|
||||
text text NOT NULL CHECK (char_length(text) BETWEEN 1 AND 4000),
|
||||
created_at timestamptz NOT NULL
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS messages_history_idx ON messages(chat_id,created_at,id);
|
||||
CREATE TABLE IF NOT EXISTS sessions (
|
||||
token_hash bytea PRIMARY KEY,
|
||||
user_id uuid NOT NULL REFERENCES users(id),
|
||||
expires_at timestamptz NOT NULL
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS sessions_expiry_idx ON sessions(expires_at);
|
||||
@@ -0,0 +1,6 @@
|
||||
CREATE TABLE sessions (
|
||||
token_hash bytea PRIMARY KEY,
|
||||
user_id uuid NOT NULL REFERENCES users(id),
|
||||
expires_at timestamptz NOT NULL
|
||||
);
|
||||
CREATE INDEX sessions_expiry_idx ON sessions(expires_at);
|
||||
@@ -0,0 +1 @@
|
||||
DROP TABLE sessions;
|
||||
@@ -0,0 +1 @@
|
||||
DROP TABLE message_idempotency;
|
||||
@@ -0,0 +1,8 @@
|
||||
CREATE TABLE message_idempotency (
|
||||
sender_id uuid NOT NULL REFERENCES users(id),
|
||||
chat_id uuid NOT NULL REFERENCES chats(id),
|
||||
key varchar(128) NOT NULL CHECK (key ~ '^[A-Za-z0-9._:-]{1,128}$'),
|
||||
message_id uuid NOT NULL REFERENCES messages(id),
|
||||
expires_at timestamptz NOT NULL,
|
||||
PRIMARY KEY (sender_id, chat_id, key)
|
||||
);
|
||||
@@ -0,0 +1,5 @@
|
||||
-- An operator must first remove attachment-only messages before this rollback.
|
||||
ALTER TABLE messages DROP CONSTRAINT messages_text_check;
|
||||
ALTER TABLE messages ADD CONSTRAINT messages_text_check CHECK (char_length(text) BETWEEN 1 AND 4000);
|
||||
DROP TABLE message_attachments;
|
||||
DROP TABLE attachments;
|
||||
@@ -0,0 +1,22 @@
|
||||
CREATE TABLE attachments (
|
||||
id uuid PRIMARY KEY,
|
||||
chat_id uuid NOT NULL REFERENCES chats(id),
|
||||
owner_id uuid NOT NULL REFERENCES users(id),
|
||||
filename text NOT NULL,
|
||||
object_key text NOT NULL UNIQUE,
|
||||
content_type text NOT NULL CHECK (content_type IN ('image/png','image/jpeg')),
|
||||
size_bytes bigint NOT NULL CHECK (size_bytes BETWEEN 1 AND 10485760),
|
||||
sha256 text NOT NULL CHECK (sha256 ~ '^[a-f0-9]{64}$'),
|
||||
created_at timestamptz NOT NULL,
|
||||
updated_at timestamptz NOT NULL
|
||||
);
|
||||
CREATE INDEX attachments_chat_idx ON attachments(chat_id);
|
||||
CREATE TABLE message_attachments (
|
||||
message_id uuid NOT NULL REFERENCES messages(id) ON DELETE CASCADE,
|
||||
attachment_id uuid NOT NULL REFERENCES attachments(id),
|
||||
position smallint NOT NULL CHECK (position BETWEEN 0 AND 9),
|
||||
PRIMARY KEY (message_id, attachment_id),
|
||||
UNIQUE(message_id, position)
|
||||
);
|
||||
ALTER TABLE messages DROP CONSTRAINT messages_text_check;
|
||||
ALTER TABLE messages ADD CONSTRAINT messages_text_check CHECK (char_length(text) <= 4000);
|
||||
@@ -0,0 +1 @@
|
||||
DROP TABLE upload_attempts;
|
||||
@@ -0,0 +1,7 @@
|
||||
CREATE TABLE upload_attempts (
|
||||
id uuid PRIMARY KEY,
|
||||
object_key text NOT NULL UNIQUE,
|
||||
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
|
||||
cleanup_after timestamptz NOT NULL
|
||||
);
|
||||
CREATE INDEX upload_attempts_cleanup_idx ON upload_attempts(cleanup_after,id);
|
||||
@@ -0,0 +1 @@
|
||||
DROP TABLE direct_uploads;
|
||||
@@ -0,0 +1,20 @@
|
||||
CREATE TABLE direct_uploads (
|
||||
id uuid PRIMARY KEY,
|
||||
attachment_id uuid NOT NULL UNIQUE,
|
||||
owner_id uuid NOT NULL REFERENCES users(id),
|
||||
chat_id uuid NOT NULL REFERENCES chats(id),
|
||||
filename text NOT NULL,
|
||||
content_type text NOT NULL CHECK (content_type IN ('image/png','image/jpeg')),
|
||||
size_bytes bigint NOT NULL CHECK (size_bytes BETWEEN 1 AND 10485760),
|
||||
sha256 text NOT NULL CHECK (sha256 ~ '^[a-f0-9]{64}$'),
|
||||
staging_key text NOT NULL UNIQUE,
|
||||
final_key text NOT NULL UNIQUE,
|
||||
state text NOT NULL DEFAULT 'pending' CHECK (state IN ('pending','ready','rejected','expired')),
|
||||
error_code text,
|
||||
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
|
||||
expires_at timestamptz NOT NULL DEFAULT clock_timestamp()+interval '120 seconds',
|
||||
complete_before timestamptz NOT NULL DEFAULT clock_timestamp()+interval '720 seconds',
|
||||
cleanup_after timestamptz NOT NULL DEFAULT clock_timestamp()+interval '1320 seconds',
|
||||
purge_after timestamptz NOT NULL DEFAULT clock_timestamp()+interval '24 hours'
|
||||
);
|
||||
CREATE INDEX direct_uploads_cleanup_idx ON direct_uploads(cleanup_after,id);
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user