ЛР 6 — БВТ2303 — Гришко — на 4 #24

Closed
TAskMAster339 wants to merge 6 commits from TAskMAster339/mtusi2026lab6-fork:solution/Grishko-BVT2303 into main
183 changed files with 15069 additions and 18 deletions
+91
View File
@@ -4,11 +4,28 @@ CA_FILE ?=
ACTION_SCRIPT ?=
TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",)
CLEANUP_ARGS ?= --dry-run
SERVICES ?=
JOB_ID ?=
ATTACHMENT_ID ?=
REASON ?=
export JOB_ID ATTACHMENT_ID REASON
COMPOSE = docker compose --project-name highload-lab6 --env-file .env -f compose.yml
.PHONY: help check validate test load test-persistence test-failover
.PHONY: app-init app-build app-up app-start app-stop app-down app-restart app-logs app-status
.PHONY: app-migrate app-s3-bootstrap worker-start worker-stop queue-status attachments-cleanup
.PHONY: jobs-failed jobs-replay jobs-upgrade jobs-cleanup-staging
.PHONY: test-backend test-pipeline frontend-build
help:
@echo "check: offline structure; validate: full OpenAPI (requirements-dev.txt); test: running API; load: synthetic writes"
@echo "app-init / app-build / app-up: configure, build, start the stack"
@echo "app-status / app-logs / app-down: inspect or stop the stack"
check:
$(PYTHON) scripts/check_contract.py
@@ -28,3 +45,77 @@ test-persistence:
test-failover:
@test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable stop-one hook"; exit 2)
$(PYTHON) tests/resilience.py --mode failover --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS)
app-init:
$(PYTHON) scripts/init_env.py
app-build: app-init
$(COMPOSE) build
app-up: app-init
$(COMPOSE) up -d --build --wait
app-start: app-init
$(COMPOSE) up -d --no-build --wait
app-stop:
$(COMPOSE) stop
app-down:
$(COMPOSE) down
app-restart: app-init
./scripts/restart.sh
app-logs:
$(COMPOSE) logs -f --tail=100 $(SERVICES)
app-status:
$(COMPOSE) ps --all
app-migrate: app-init
$(COMPOSE) run --rm migrate
app-s3-bootstrap: app-init
$(COMPOSE) run --rm s3-bootstrap
worker-start:
$(COMPOSE) start worker
worker-stop:
$(COMPOSE) stop worker
queue-status:
$(COMPOSE) exec -T broker rabbitmqctl -q list_queues name messages_ready messages_unacknowledged --formatter json
attachments-cleanup:
$(COMPOSE) run --rm --no-deps upload-cleanup ./messenger attachments-cleanup $(CLEANUP_ARGS)
jobs-failed:
$(COMPOSE) exec -T dispatcher ./messenger jobs list-failed
jobs-replay:
@test -n "$$JOB_ID" && test -n "$$REASON" || { echo "Set JOB_ID=... REASON='...'"; exit 2; }
$(COMPOSE) exec -T dispatcher ./messenger jobs replay "$$JOB_ID" "$$REASON"
jobs-upgrade:
@test -n "$$ATTACHMENT_ID" && test -n "$$REASON" || { echo "Set ATTACHMENT_ID=... REASON='...'"; exit 2; }
$(COMPOSE) exec -T dispatcher ./messenger jobs upgrade "$$ATTACHMENT_ID" 2 "$$REASON"
jobs-cleanup-staging:
@test -n "$$JOB_ID" || { echo "Set JOB_ID=..."; exit 2; }
$(COMPOSE) exec -T dispatcher ./messenger jobs cleanup-staging "$$JOB_ID"
test-backend: app-init
$(COMPOSE) --profile test run --rm --build backend-tests
test-pipeline: app-init
$(COMPOSE) --profile test run --rm --build backend-tests go test -race -count=1 -v ./backend/tests ./backend/workers -run 'TestPipeline|TestVersionedRendering|TestThumbnail|TestOptimized'
frontend-build:
$(COMPOSE) exec -T frontend npm run build
.PHONY: evidence
evidence:
$(PYTHON) scripts/generate_evidence.py --base-url "$(BASE_URL)"
+94 -18
View File
@@ -1,37 +1,113 @@
# Отчёт — лабораторная 6
- Автор, группа: TODO
- Целевая оценка; выбранный трек 5, если нужен: TODO
- Commit/PR: TODO после публикации
- Использованные источники, библиотеки и помощь ИИ: TODO
- Автор — Гришко Дмитрий Михайлович, БВТ2303.
- Целевая оценка — 4.
- PR: https://gitea.privcloud.tech/mtusi2026/mtusi2026lab6/pulls/24.
- Источники — [задание](README.md), [контракт наблюдаемости](contracts/observability.md), [OpenAPI](contracts/openapi.json), [OpenTelemetry Go](https://opentelemetry.io/docs/languages/go/instrumentation/), [OTLP в Loki](https://grafana.com/docs/loki/latest/send-data/otel/). Зависимости закреплены в [go.mod](src/go.mod). Помощь ИИ — OpenAI Codex, реализация, проверки и оформление.
## Запуск из чистого clone
## Запуск из чистого клона
TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных.
Нужны Linux, Git, Make, Python 3.9+, Docker Engine с Compose и доступ к реестрам образов и зависимостей при первой сборке. Склонируйте ветку решения и перейдите в каталог проекта.
```sh
git clone --branch solution/Grishko-BVT2303 --single-branch \
https://gitea.privcloud.tech/TAskMAster339/mtusi2026lab6-fork.git
cd mtusi2026lab6-fork
```
Создайте локальную конфигурацию, проверьте Compose и запустите стенд.
```sh
make app-init
docker compose config --quiet
make app-up
curl --fail http://127.0.0.1:8080/health/ready
```
После ответа `{"status":"ok"}` запустите проверки.
```sh
make check
make test BASE_URL=http://127.0.0.1:8080
make evidence BASE_URL=http://127.0.0.1:8080
```
`app-init` создаёт локальные `.env` и `secrets/`. Compose применяет миграции и создаёт приватный S3 bucket до запуска API и worker. По умолчанию API доступен на 8080, Grafana на 3000, UI на 5173, S3 на 9000 и статистика HAProxy на 8404. Dashboard открывается в гостевом режиме. Для Explore при первом входе используются стандартные локальные данные Grafana с последующей сменой пароля.
`make app-down` останавливает стенд с сохранением томов. Для намеренного удаления учебных данных используется `docker compose down --volumes`.
Приложение собирается Go 1.22.2 в Debian 12.15. Frontend использует Node 22. В Compose закреплены PostgreSQL 16.10, Redis 7.4.11, RabbitMQ 4.2.0, HAProxy 3.2.25, Collector Contrib 0.120.0, Prometheus 3.2.1, Loki 3.4.2, Tempo 2.7.1 и Grafana 11.5.2. MinIO собирается из закреплённого коммита [Dockerfile](infra/minio.Dockerfile). OpenTelemetry Go использует 1.34.0 для трасс и метрик и 0.10.0 для логов.
## Архитектура и решения
TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab5, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации.
```text
Клиент → HAProxy → API ×2 → PostgreSQL / Redis / S3 staging
PostgreSQL outbox → dispatcher → RabbitMQ → worker → S3 / PostgreSQL
API / dispatcher / worker → OTLP HTTP → Collector
├→ Prometheus (scrape)
├→ Loki (OTLP)
└→ Tempo (OTLP)
Grafana → Prometheus / Loki / Tempo
```
Из №5 перенесены сессии Redis, балансировка, идемпотентность сообщений, приватный S3 и фоновая обработка. API отвечает 202 после записи оригинала в устойчивый staging и фиксации метаданных и задания в PostgreSQL. Dispatcher подтверждает публикацию через RabbitMQ. Worker завершает обработку и фиксирует результат или намерение повторной попытки перед ACK. PostgreSQL хранит бизнес-данные и состояние задания. Готовность API зависит от PostgreSQL, Redis и S3.
Миграция 10 добавляет W3C `traceparent` и `tracestate` в outbox. Dispatcher восстанавливает контекст, создаёт producer span и переносит его в заголовки AMQP. Worker создаёт дочерний consumer span с номером попытки. Повторная публикация восстанавливает тот же контекст upload. В Tempo доступны спаны S3 и PostgreSQL, а Loki хранит trace ID, span ID, исход операции и технические поля корреляции. SQL, параметры запросов, заголовки авторизации, тексты и содержимое файлов не включаются в события. В метриках маршруты представлены шаблонами.
Prometheus хранит временные ряды, Loki — журналы, Tempo — трассы. У хранилищ отдельные тома. Grafana загружает источники и dashboard из [provisioning](infra/otel/grafana/provisioning). Collector использует batch, memory limiter 128 МиБ, лимит контейнера 192 МиБ и очереди экспорта Loki/Tempo по 64 пакета с retry до 30 секунд. Экспорт выполняется асинхронно.
Compose использует `APP_*`, `API_REPLICAS`, `SESSION_TTL_SECONDS`, `IDEMPOTENCY_TTL_SECONDS`, параметры S3, файлы подключения и `JOB_MAX_ATTEMPTS` / `JOB_TIMEOUT_SECONDS`. Сессии используют Redis. Конкурентность задаётся количеством процессов worker, по одному слоту на экземпляр. Пределы API — 10 МиБ и 40 млн пикселей, worker — 8 млн по умолчанию через `WORKER_MAX_PIXELS`. Failed staging хранится 24 часа. Код задаёт ресурсные атрибуты и имя сервиса по роли процесса, использует OTLP HTTP/protobuf. Endpoint задаётся через `OTEL_EXPORTER_OTLP_ENDPOINT`.
## Доказательства критериев
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
| Критерий | Команда или сценарий | Наблюдение | Доказательство |
| --- | --- | --- | --- |
| Публичный smoke | make test | TODO | TODO |
| Остальные пункты уровня 3 | TODO | TODO | TODO |
| Все пункты уровня 4, если заявлен | TODO | TODO | TODO |
| Выбранный трек 5, если заявлен | TODO | TODO | TODO |
| 3. Минимум №5 | `make test` | Сессии, ACL, сообщения, идемпотентность, 202 → ready, оригинал и thumbnail | [Smoke](evidence/criteria/grade3-smoke.md) |
| 3. Единый стенд, хранилища, provisioning | Compose, Collector validate, Grafana API | Тома и три источника данных, импортированный dashboard | [Стенд](evidence/criteria/grade3-stand.md) |
| 3. OTLP и корреляция сигналов | `check_observability.py signals` | Метрики API и worker, совпадение trace/span ID в Loki и Tempo, проверка контрольных чувствительных значений | [Сигналы](evidence/criteria/grade3-signals.md) |
| 4. Контекст через брокер и retry | `pipeline`, затем временная остановка S3 | Родительская цепочка до worker и операции S3/DB, обе попытки в одной трассе | [Трассировка](evidence/criteria/grade4-pipeline.md) |
| 4. Диагностические метрики и SLO | `check_observability.py dashboard`, `promtool check rules` | PromQL для панелей, SLI за 30 минут, два валидных правила | [Dashboard и SLO](evidence/criteria/grade4-dashboard.md) |
| 4. Алерт возраста очереди | Остановка worker | `MessengerQueueLag` firing → resolved, принятое вложение стало ready | [Очередь](evidence/criteria/grade4-queue.md) |
| 4. Алерт доли 5xx | Временная блокировка таблицы чатов | `MessengerHighErrorRate` firing → resolved, задержка видна в дочернем спане PostgreSQL | [PostgreSQL](evidence/criteria/grade4-database.md) |
| 4. Недоступность хранилища сигналов | Остановка Tempo | API принимает сообщения, worker обрабатывает вложение, Collector ограничивает буфер и регистрирует потерю спанов | [Tempo](evidence/criteria/grade4-telemetry.md) |
TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены.
SLI — доля бизнес-запросов без 5xx, SLO ≥99% за 30 минут при наличии трафика. Health исключён. Перед итоговыми инцидентами SLI составил 99,93%. p50/p95/p99 вычисляются из агрегированных histogram buckets в секундах. Глубина очереди включает все незавершённые задания, в том числе ожидающие retry. Подробные запросы находятся в [dashboard JSON](infra/otel/grafana/dashboards/messenger.json), порядок диагностики — в [runbook](infra/otel/runbook.md).
## Эксперименты и отказы
TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения.
Сценарии запускаются последовательно на локальном стенде. Проверяется, можно ли по сохранённым сигналам различить остановку обработчика и задержку в БД, а также сохраняется ли бизнес-путь при недоступности хранилища трасс.
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
| --- | --- | --- | --- |
| TODO | TODO | TODO | TODO |
| Воздействие | Наблюдение | Восстановление |
| --- | --- | --- |
| Остановка worker | Возраст заданий растёт, alert firing после превышения 20 секунд в течение 15 секунд | Запуск worker, то же вложение ready, alert resolved |
| Блокировка таблицы чатов | 5xx и долгий дочерний спан PostgreSQL. Правило проверяет порог 5% за минуту с выдержкой 15 секунд | Снятие блокировки, успешное чтение чата, alert resolved после выхода ошибок из окна |
| Остановка S3 | Первая попытка обработки откладывается. Следующая видна в той же трассе | Запуск S3, исходное вложение ready |
| Остановка Tempo | Сообщения приняты, вложение достигло ready. Collector отбросил 54 спана. Память по снимку — 56,43 МиБ из 192 МиБ | Tempo запущен после проверки |
Один короткий профиль нагрузки выполнен на AMD Ryzen 9 PRO 7940HS с 16 логическими CPU и 14 ГиБ RAM. Две API-реплики ограничены 0,5 CPU и 512 МиБ каждая. Worker ограничен 1 CPU и 512 МиБ. Профиль — 30 секунд, четыре параллельных клиента, closed-loop POST сообщений, один прогон без отдельного прогрева. В итоговом прогоне подтверждено 37 708 запросов, ошибок 0, пропускная способность 1256,75 RPS. Задержки p50/p95/p99 составили 2,828 / 4,327 / 7,211 мс. Полный вывод и снимок ресурсов после прогона сохранены в [результатах нагрузки](evidence/criteria/grade4-load.md).
### Скриншоты Grafana
![Дашборд API во время нагрузки](evidence/grafana_load.png)
*Рисунок 1. Во время нагрузочного прогона частота запросов выросла, а частота ответов `5xx` осталась на нуле. Панели p50, p95 и p99 показывают задержку API.*
![Сквозная трасса загрузки с повторной обработкой](evidence/s3_fall.png)
*Рисунок 2. В одной трассе Tempo видны HTTP-ответ `202`, публикации в RabbitMQ и спаны worker. Первая обработка завершилась ошибкой, затем worker получил задание повторно. Trace ID — `9718ad3f6a66598ef72ed8ec1cb5101f`.*
![Глубина и возраст очереди при остановке worker](evidence/queue_fall.png)
*Рисунок 3. При остановке worker в очереди оставалось одно незавершённое задание. Возраст старейшего задания превысил 30 секунд. После возобновления обработки оба показателя вернулись к нулю.*
![Состояния алерта MessengerQueueLag в Prometheus](evidence/alerts.png)
*Рисунок 4. В Grafana Explore запрос к метрике `ALERTS` показывает переход `MessengerQueueLag` из `pending` в `firing` во время роста возраста задания.*
![Дочерний спан PostgreSQL при ответе API 503](evidence/postgres_fall.png)
*Рисунок 5. При блокировке таблицы чатов API вернул `503`. Ошибочный дочерний спан `postgresql query` занял около пяти секунд, почти всю длительность HTTP-запроса.*
## Ограничения и следующие шаги
TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза.
Стенд локальный, PostgreSQL, S3, RabbitMQ и балансировщик развёрнуты в одном экземпляре. Отказы этих компонентов могут прерывать соответствующие бизнес-операции. Снимок памяти Collector относится к короткому отказу. При переполнении очереди или исчерпании повторов возможна потеря телеметрии. Результат closed-loop прогона зависит от клиента и не оценивает предел пропускной способности. Заявленная оценка — 4.
+386
View File
@@ -0,0 +1,386 @@
name: highload-lab6
services:
redis:
cpus: 0.25
mem_limit: 128m
networks: [data]
image: redis:7.4.11-bookworm
command:
[
"redis-server",
"--appendonly",
"yes",
"--appendfsync",
"everysec",
"--dir",
"/data",
]
volumes:
- redis_data:/data
healthcheck:
test: ["CMD", "redis-cli", "ping"]
interval: 3s
timeout: 3s
retries: 20
db:
cpus: 1.0
mem_limit: 512m
networks: [data]
image: postgres:16.10-bookworm
environment:
POSTGRES_DB: ${POSTGRES_DB}
POSTGRES_USER: ${POSTGRES_USER}
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD}
volumes:
- postgres_data:/var/lib/postgresql/data
healthcheck:
test: ["CMD-SHELL", "pg_isready -U $$POSTGRES_USER -d $$POSTGRES_DB"]
interval: 3s
timeout: 3s
retries: 20
migrate:
cpus: 0.5
mem_limit: 128m
networks: [data]
build: ./src
environment:
DATABASE_URL_FILE: ${DATABASE_URL_FILE:-/run/secrets/database_url}
DATABASE_URL: postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@db:5432/${POSTGRES_DB}?sslmode=disable
volumes:
- ./secrets/database_url:${DATABASE_URL_FILE:-/run/secrets/database_url}:ro
command: ["./messenger", "migrate", "up"]
depends_on:
db:
condition: service_healthy
s3:
cpus: 1.0
mem_limit: 512m
networks: [edge, data]
ports:
- "127.0.0.1:9000:9000"
build:
context: ./infra
dockerfile: minio.Dockerfile
environment:
MINIO_ROOT_USER_FILE: /run/secrets/s3_access_key_id
MINIO_ROOT_PASSWORD_FILE: /run/secrets/s3_secret_access_key
MINIO_BROWSER: "off"
MINIO_API_CORS_ALLOW_ORIGIN: ${S3_CORS_ORIGINS:-http://127.0.0.1:5173,http://localhost:5173}
volumes:
- s3_data:/data
- ./secrets/s3_access_key_id:/run/secrets/s3_access_key_id:ro
- ./secrets/s3_secret_access_key:/run/secrets/s3_secret_access_key:ro
healthcheck:
test: ["CMD", "curl", "-f", "http://127.0.0.1:9000/minio/health/ready"]
interval: 3s
timeout: 2s
retries: 30
s3-bootstrap:
cpus: 0.5
mem_limit: 128m
networks: [data]
build: ./src
environment: &s3-environment
S3_ROOT_ACCESS_KEY_ID_FILE: /run/secrets/s3_root_access_key_id
S3_ROOT_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_root_secret_access_key
S3_ENDPOINT: ${S3_ENDPOINT:-http://s3:9000}
S3_PUBLIC_ENDPOINT: ${S3_PUBLIC_ENDPOINT:-http://127.0.0.1:9000}
S3_REGION: ${S3_REGION:-us-east-1}
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
S3_ACCESS_KEY_ID_FILE: ${S3_ACCESS_KEY_ID_FILE:-/run/secrets/s3_access_key_id}
S3_SECRET_ACCESS_KEY_FILE: ${S3_SECRET_ACCESS_KEY_FILE:-/run/secrets/s3_secret_access_key}
volumes:
- ./secrets/s3_access_key_id:/run/secrets/s3_root_access_key_id:ro
- ./secrets/s3_secret_access_key:/run/secrets/s3_root_secret_access_key:ro
- ./secrets/s3_api_access_key_id:${S3_ACCESS_KEY_ID_FILE:-/run/secrets/s3_access_key_id}:ro
- ./secrets/s3_api_secret_access_key:${S3_SECRET_ACCESS_KEY_FILE:-/run/secrets/s3_secret_access_key}:ro
command: ["./messenger", "s3-bootstrap"]
depends_on:
s3:
condition: service_healthy
api:
cpus: 0.5
mem_limit: 512m
networks: [proxy, data, telemetry]
build: ./src
deploy:
replicas: ${API_REPLICAS:-2}
stop_grace_period: 15s
environment: &api-environment
<<: *s3-environment
OTEL_EXPORTER_OTLP_ENDPOINT: ${OTEL_EXPORTER_OTLP_ENDPOINT-http://otel-collector:4318}
S3_ROOT_ACCESS_KEY_ID_FILE: ""
S3_ROOT_SECRET_ACCESS_KEY_FILE: ""
BROKER_URL_FILE: /run/secrets/broker_url
JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5}
JOB_TIMEOUT_SECONDS: ${JOB_TIMEOUT_SECONDS:-60}
WORKER_MAX_PIXELS: ${WORKER_MAX_PIXELS:-8000000}
UPLOAD_TMP_DIR: /tmp/uploads
UPLOAD_CLEANUP_GRACE_SECONDS: ${UPLOAD_CLEANUP_GRACE_SECONDS:-600}
# Go 1.22 sees host CPUs; keep parallelism suitable for this 0.5 CPU limit.
GOMAXPROCS: ${API_GOMAXPROCS:-1}
GOMEMLIMIT: 384MiB
APP_HOST: ${APP_HOST:-0.0.0.0}
APP_PORT: ${APP_PORT:-8080}
APP_ENV: ${APP_ENV:-lab}
IDEMPOTENCY_TTL_SECONDS: ${IDEMPOTENCY_TTL_SECONDS:-86400}
DATABASE_URL_FILE: ${DATABASE_URL_FILE:-/run/secrets/database_url}
CACHE_URL_FILE: ${CACHE_URL_FILE:-/run/secrets/cache_url}
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600}
REDIS_ADDR: redis:6379
DATABASE_URL: postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@db:5432/${POSTGRES_DB}?sslmode=disable
tmpfs:
- /tmp/uploads:rw,noexec,nosuid,size=16m,mode=1777
volumes: &api-volumes
- ./secrets/s3_api_access_key_id:${S3_ACCESS_KEY_ID_FILE:-/run/secrets/s3_access_key_id}:ro
- ./secrets/s3_api_secret_access_key:${S3_SECRET_ACCESS_KEY_FILE:-/run/secrets/s3_secret_access_key}:ro
- ./secrets/broker_url:/run/secrets/broker_url:ro
- ./secrets/database_url:${DATABASE_URL_FILE:-/run/secrets/database_url}:ro
- ./secrets/cache_url:${CACHE_URL_FILE:-/run/secrets/cache_url}:ro
depends_on:
s3-bootstrap:
condition: service_completed_successfully
migrate:
condition: service_completed_successfully
db:
condition: service_healthy
redis:
condition: service_healthy
healthcheck:
test: ["CMD", "./messenger", "healthcheck"]
interval: 5s
timeout: 3s
retries: 20
broker:
image: rabbitmq:4.2.0-management
hostname: broker
environment:
RABBITMQ_SERVER_ADDITIONAL_ERL_ARGS: "+S 2:2"
RABBITMQ_CTL_ERL_ARGS: "+S 1:1"
cpus: 0.5
mem_limit: 512m
networks: [data]
command:
- sh
- -ec
- |
export RABBITMQ_DEFAULT_USER="$$(cat /run/secrets/broker_user)"
export RABBITMQ_DEFAULT_PASS="$$(cat /run/secrets/broker_password)"
exec docker-entrypoint.sh rabbitmq-server
volumes:
- rabbitmq_data:/var/lib/rabbitmq
- ./secrets/broker_user:/run/secrets/broker_user:ro
- ./secrets/broker_password:/run/secrets/broker_password:ro
healthcheck:
test: ["CMD-SHELL", "rabbitmq-diagnostics -q check_running && rabbitmq-diagnostics -q check_port_connectivity"]
interval: 5s
timeout: 10s
start_period: 20s
retries: 30
worker:
tmpfs:
- /tmp:rw,noexec,nosuid,size=64m,mode=1777
build: ./src
restart: unless-stopped
cpus: 1.0
mem_limit: 512m
networks: [data, telemetry]
environment: *api-environment
volumes: *api-volumes
command: ["./messenger", "worker"]
healthcheck:
disable: true
depends_on:
migrate:
condition: service_completed_successfully
s3-bootstrap:
condition: service_completed_successfully
broker:
condition: service_healthy
dispatcher:
build: ./src
restart: unless-stopped
cpus: 0.25
mem_limit: 128m
networks: [data, telemetry]
environment: *api-environment
volumes: *api-volumes
command: ["./messenger", "dispatcher"]
healthcheck:
disable: true
depends_on:
migrate:
condition: service_completed_successfully
broker:
condition: service_healthy
upload-cleanup:
restart: unless-stopped
cpus: 0.25
mem_limit: 128m
networks: [data]
build: ./src
command: ["./messenger", "attachments-cleanup", "--loop"]
environment:
<<: *s3-environment
S3_ROOT_ACCESS_KEY_ID_FILE: ""
S3_ROOT_SECRET_ACCESS_KEY_FILE: ""
DATABASE_URL_FILE: ${DATABASE_URL_FILE:-/run/secrets/database_url}
volumes:
- ./secrets/s3_api_access_key_id:${S3_ACCESS_KEY_ID_FILE:-/run/secrets/s3_access_key_id}:ro
- ./secrets/s3_api_secret_access_key:${S3_SECRET_ACCESS_KEY_FILE:-/run/secrets/s3_secret_access_key}:ro
- ./secrets/database_url:${DATABASE_URL_FILE:-/run/secrets/database_url}:ro
healthcheck:
disable: true
depends_on:
migrate:
condition: service_completed_successfully
s3-bootstrap:
condition: service_completed_successfully
haproxy:
environment:
APP_PORT: ${APP_PORT:-8080}
API_REPLICAS: ${API_REPLICAS:-2}
cpus: 0.5
mem_limit: 128m
networks: [edge, proxy]
image: haproxy:3.2.25-alpine
ports:
- "127.0.0.1:${API_HTTP_PORT:-8080}:8080"
- "127.0.0.1:8404:8404"
volumes:
- ./infra/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro
depends_on:
api:
condition: service_healthy
healthcheck:
test:
[
"CMD",
"wget",
"-q",
"-O",
"/dev/null",
"http://127.0.0.1:8080/health/ready",
]
interval: 2s
timeout: 3s
retries: 20
backend-tests:
cpus: 2.0
mem_limit: 1g
networks: [data]
profiles: ["test"]
build:
context: ./src
target: builder
environment:
<<: *s3-environment
S3_PUBLIC_ENDPOINT: http://s3:9000
TEST_DATABASE_URL: postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@db:5432/${POSTGRES_DB}?sslmode=disable
TEST_REDIS_ADDR: redis:6379
volumes:
- ./secrets/s3_api_access_key_id:${S3_ACCESS_KEY_ID_FILE:-/run/secrets/s3_access_key_id}:ro
- ./secrets/s3_api_secret_access_key:${S3_SECRET_ACCESS_KEY_FILE:-/run/secrets/s3_secret_access_key}:ro
- ./secrets/s3_access_key_id:/run/secrets/s3_root_access_key_id:ro
- ./secrets/s3_secret_access_key:/run/secrets/s3_root_secret_access_key:ro
depends_on:
s3-bootstrap:
condition: service_completed_successfully
db:
condition: service_healthy
redis:
condition: service_healthy
command: ["go", "test", "-race", "-count=1", "-v", "./backend/..."]
frontend:
cpus: 0.5
mem_limit: 512m
networks: [edge]
build:
context: ./src/frontend
environment:
API_PROXY_TARGET: http://haproxy:8080
ports:
- "127.0.0.1:5173:5173"
volumes:
- ./src/frontend:/app
- /app/node_modules
depends_on:
haproxy:
condition: service_healthy
healthcheck:
test:
[
"CMD",
"node",
"-e",
"fetch('http://127.0.0.1:5173/').then(r => process.exit(r.ok ? 0 : 1)).catch(() => process.exit(1))",
]
interval: 5s
timeout: 3s
retries: 20
otel-collector:
image: otel/opentelemetry-collector-contrib:0.120.0
networks: [telemetry]
mem_limit: 192m
command: ["--config=/etc/otelcol-contrib/config.yaml"]
volumes:
- ./infra/otel/collector.yaml:/etc/otelcol-contrib/config.yaml:ro
prometheus:
image: prom/prometheus:v3.2.1
networks: [telemetry]
command: ["--config.file=/etc/prometheus/prometheus.yml", "--storage.tsdb.path=/prometheus", "--storage.tsdb.retention.time=7d"]
volumes:
- ./infra/otel/prometheus.yaml:/etc/prometheus/prometheus.yml:ro
- ./infra/otel/alerts.yaml:/etc/prometheus/alerts.yaml:ro
- prometheus_data:/prometheus
loki:
image: grafana/loki:3.4.2
networks: [telemetry]
command: ["-config.file=/etc/loki/lab.yaml"]
volumes:
- ./infra/otel/loki.yaml:/etc/loki/lab.yaml:ro
- loki_data:/loki
tempo:
image: grafana/tempo:2.7.1
networks: [telemetry]
command: ["-config.file=/etc/tempo/lab.yaml"]
volumes:
- ./infra/otel/tempo.yaml:/etc/tempo/lab.yaml:ro
- tempo_data:/var/tempo
grafana:
image: grafana/grafana:11.5.2
networks: [telemetry, edge]
environment:
GF_AUTH_ANONYMOUS_ENABLED: "true"
GF_AUTH_ANONYMOUS_ORG_ROLE: Viewer
ports:
- "127.0.0.1:${GRAFANA_HTTP_PORT:-3000}:3000"
volumes:
- ./infra/otel/grafana/provisioning:/etc/grafana/provisioning:ro
- ./infra/otel/grafana/dashboards:/var/lib/grafana/dashboards:ro
- grafana_data:/var/lib/grafana
volumes:
prometheus_data:
loki_data:
tempo_data:
grafana_data:
rabbitmq_data:
postgres_data:
redis_data:
s3_data:
networks:
edge:
proxy:
internal: true
data:
internal: true
telemetry:
internal: true
Binary file not shown.

After

Width:  |  Height:  |  Size: 306 KiB

+13
View File
@@ -0,0 +1,13 @@
# Доказательства критериев
[Сводка прогона 20261113T155722.910201Z](runs/20261113T155722.910201Z/summary.json)
- [grade3-stand](grade3-stand.md) — PASS
- [grade3-smoke](grade3-smoke.md) — PASS
- [grade3-signals](grade3-signals.md) — PASS
- [grade4-pipeline](grade4-pipeline.md) — PASS
- [grade4-dashboard](grade4-dashboard.md) — PASS
- [grade4-queue](grade4-queue.md) — PASS
- [grade4-database](grade4-database.md) — PASS
- [grade4-telemetry](grade4-telemetry.md) — PASS
- [grade4-load](grade4-load.md) — PASS
+22
View File
@@ -0,0 +1,22 @@
# Метрики, структурные логи и трассы API и worker
Предпосылки — локальный синтетический стенд после make app-up и доступ к Docker.
Контрольный POST сопоставляется с логом по X-Request-Id. Оба сервиса сопоставляются с Tempo по trace_id и span_id. Prometheus возвращает счётчики и histogram buckets. Проверка ищет утечки синтетических значений в полученных сигналах.
Команда `/usr/bin/python3 scripts/check_observability.py --base-url http://127.0.0.1:8080 --grafana-url http://127.0.0.1:3000 signals`. Код выхода **0**. Время 7.2 с.
[Вывод](runs/20261113T155722.910201Z/grade3-signals-1.txt)
````text
Worker: изображение 3000×3000 завершилось в состоянии failed.
messenger-api: http.request.completed, trace=91735be6f8944614b886c2a34bfc95a8, span=7aca4e9abb3620f6
messenger-worker: job.processing.completed, trace=8f05225357fb027e222f890074249eed, span=49632776d49e21fe
messenger-worker: job.processing.completed, trace=c958e0f21140e39dddb4192b5982442e, span=f459a5c48084c984
Prometheus: messenger_http_requests_total, рядов=40
Prometheus: messenger_http_duration_seconds_bucket, рядов=560
Prometheus: messenger_worker_jobs_total{job_outcome="success"}, рядов=1
Prometheus: messenger_worker_jobs_total{job_outcome="error"}, рядов=1
Prometheus: messenger_worker_duration_seconds_bucket, рядов=42
Проверенные сигналы не содержат контрольных credentials, текста, имени/байтов файла и бизнес-ID.
````
+27
View File
@@ -0,0 +1,27 @@
# Совместимость API при включённой телеметрии
Предпосылки — локальный синтетический стенд после make app-up и доступ к Docker.
Smoke проверяет сессии, сообщения, ACL, идемпотентность и готовность вложения с оригиналом и thumbnail.
Команда `make check`. Код выхода **0**. Время 0.05 с.
[Вывод](runs/20261113T155722.910201Z/grade3-smoke-1.txt)
````text
make[1]: Entering directory '/home/TAskMAster339/Projects/highload-systems/mtusi2026lab6-fork'
python3 scripts/check_contract.py
PASS: offline skeleton structure, local refs and Python syntax; application NOT tested.
make[1]: Leaving directory '/home/TAskMAster339/Projects/highload-systems/mtusi2026lab6-fork'
````
Команда `make test BASE_URL=http://127.0.0.1:8080`. Код выхода **0**. Время 3.51 с.
[Вывод](runs/20261113T155722.910201Z/grade3-smoke-2.txt)
````text
make[1]: Entering directory '/home/TAskMAster339/Projects/highload-systems/mtusi2026lab6-fork'
python3 tests/smoke.py --base-url "http://127.0.0.1:8080"
PASS: lab6 public API scenario; infrastructure and grading evidence remain separate.
make[1]: Leaving directory '/home/TAskMAster339/Projects/highload-systems/mtusi2026lab6-fork'
````
+74
View File
@@ -0,0 +1,74 @@
# Стенд, тома и provisioning
Предпосылки — локальный синтетический стенд после make app-up и доступ к Docker.
Prometheus хранит временные ряды, Loki — журналы, Tempo — трассы. Collector принимает и маршрутизирует OTLP. Grafana читает данные и загружает dashboard из JSON.
Команда `docker compose config --quiet`. Код выхода **0**. Время 0.08 с.
[Вывод](runs/20261113T155722.910201Z/grade3-stand-1.txt)
````text
(пусто)
````
Команда `docker compose ps --all`. Код выхода **0**. Время 0.11 с.
[Вывод](runs/20261113T155722.910201Z/grade3-stand-2.txt)
````text
NAME IMAGE COMMAND SERVICE CREATED STATUS PORTS
highload-lab6-api-1 highload-lab6-api "./messenger" api 31 minutes ago Up 31 minutes (healthy) 8080/tcp
highload-lab6-api-2 highload-lab6-api "./messenger" api 31 minutes ago Up 31 minutes (healthy) 8080/tcp
highload-lab6-broker-1 rabbitmq:4.2.0-management "docker-entrypoint.s…" broker About an hour ago Up About an hour (healthy) 4369/tcp, 5671-5672/tcp, 15671-15672/tcp, 15691-15692/tcp, 25672/tcp
highload-lab6-db-1 postgres:16.10-bookworm "docker-entrypoint.s…" db About an hour ago Up About an hour (healthy) 5432/tcp
highload-lab6-dispatcher-1 highload-lab6-dispatcher "./messenger dispatc…" dispatcher 31 minutes ago Up 31 minutes 8080/tcp
highload-lab6-frontend-1 sha256:6f2bacde48d02aca251e04c4707e11d6707c797af56bf6623a671b2e0c20da30 "docker-entrypoint.s…" frontend About an hour ago Up About an hour (healthy) 127.0.0.1:5173->5173/tcp
highload-lab6-grafana-1 grafana/grafana:11.5.2 "/run.sh" grafana About an hour ago Up About an hour 127.0.0.1:3000->3000/tcp
highload-lab6-haproxy-1 haproxy:3.2.25-alpine "docker-entrypoint.s…" haproxy About an hour ago Up About an hour (healthy) 127.0.0.1:8080->8080/tcp, 127.0.0.1:8404->8404/tcp
highload-lab6-loki-1 grafana/loki:3.4.2 "/usr/bin/loki -conf…" loki About an hour ago Up About an hour 3100/tcp
highload-lab6-migrate-1 highload-lab6-migrate "./messenger migrate…" migrate 31 minutes ago Exited (0) 3 minutes ago
highload-lab6-otel-collector-1 otel/opentelemetry-collector-contrib:0.120.0 "/otelcol-contrib --…" otel-collector About an hour ago Up About an hour 4317-4318/tcp, 55678-55679/tcp
highload-lab6-prometheus-1 prom/prometheus:v3.2.1 "/bin/prometheus --c…" prometheus 31 minutes ago Up 31 minutes 9090/tcp
highload-lab6-redis-1 redis:7.4.11-bookworm "docker-entrypoint.s…" redis About an hour ago Up About an hour (healthy) 6379/tcp
highload-lab6-s3-1 sha256:b891ef2bb2e30a5a240ee0d267c8938ab43f2ce5e5c89347e580e7c1b1cf4e5e "minio server /data …" s3 About an hour ago Up 3 minutes (healthy) 127.0.0.1:9000->9000/tcp
highload-lab6-s3-bootstrap-1 highload-lab6-s3-bootstrap "./messenger s3-boot…" s3-bootstrap 31 minutes ago Exited (0) 3 minutes ago
highload-lab6-tempo-1 grafana/tempo:2.7.1 "/tempo -config.file…" tempo About an hour ago Up 34 seconds
highload-lab6-upload-cleanup-1 highload-lab6-upload-cleanup "./messenger attachm…" upload-cleanup 31 minutes ago Up 31 minutes 8080/tcp
highload-lab6-worker-1 highload-lab6-worker "./messenger worker" worker 31 minutes ago Up 3 minutes 8080/tcp
````
Команда `docker compose config --volumes`. Код выхода **0**. Время 0.08 с.
[Вывод](runs/20261113T155722.910201Z/grade3-stand-3.txt)
````text
s3_data
loki_data
postgres_data
prometheus_data
redis_data
tempo_data
grafana_data
rabbitmq_data
````
Команда `docker compose run --rm --no-deps otel-collector validate --config=/etc/otelcol-contrib/config.yaml`. Код выхода **0**. Время 1.28 с.
[Вывод](runs/20261113T155722.910201Z/grade3-stand-4.txt)
````text
Container highload-lab6-otel-collector-run-0fb806dd4ede Creating
Container highload-lab6-otel-collector-run-0fb806dd4ede Created
````
Команда `/usr/bin/python3 scripts/check_observability.py --base-url http://127.0.0.1:8080 --grafana-url http://127.0.0.1:3000 provisioning`. Код выхода **0**. Время 0.13 с.
[Вывод](runs/20261113T155722.910201Z/grade3-stand-5.txt)
````text
Источник Grafana: prometheus
Источник Grafana: loki
Источник Grafana: tempo
Загружен dashboard messenger-lab6, панелей: 17
````
+38
View File
@@ -0,0 +1,38 @@
# Метрики диагностики и SLO
Предпосылки — локальный синтетический стенд после make app-up и доступ к Docker.
Проверяются запросы панелей, SLI за 30 минут и синтаксис двух правил.
Команда `/usr/bin/python3 scripts/check_observability.py --base-url http://127.0.0.1:8080 --grafana-url http://127.0.0.1:3000 dashboard`. Код выхода **0**. Время 0.16 с.
[Вывод](runs/20261113T155722.910201Z/grade4-dashboard-1.txt)
````text
API · запросы в секунду — sum(rate(messenger_http_requests_total[1m])) — рядов: 1
API · 5xx в секунду — sum(rate(messenger_http_requests_total{http_status_class="5xx"}[1m])) or 0 * sum(rate(messenger_http_requests_total[1m])) — рядов: 1
API · p95 длительности — histogram_quantile(0.95, sum by (le) (rate(messenger_http_duration_seconds_bucket[1m]))) — рядов: 1
Worker · задания в минуту — 60 * sum by (job_outcome) (rate(messenger_worker_jobs_total[1m])) — рядов: 3
Worker · p95 длительности — histogram_quantile(0.95, sum by (le,job_outcome) (rate(messenger_worker_duration_seconds_bucket[1m]))) — рядов: 3
API · p50 длительности — histogram_quantile(0.5, sum by (le) (rate(messenger_http_duration_seconds_bucket[1m]))) — рядов: 1
API · p99 длительности — histogram_quantile(0.99, sum by (le) (rate(messenger_http_duration_seconds_bucket[1m]))) — рядов: 1
API · доля 5xx — (sum(rate(messenger_http_requests_total{http_status_class="5xx"}[1m])) or 0 * sum(rate(messenger_http_requests_total[1m]))) / sum(rate(messenger_http_requests_total[1m])) — рядов: 1
Очередь attachments.v1 · незавершённые задания — max(messenger_queue_depth) — рядов: 1
Очередь attachments.v1 · возраст старейшего задания — max(messenger_queue_oldest_seconds) — рядов: 1
PostgreSQL · занятые соединения — sum by (service_name) (messenger_db_pool_used) — рядов: 3
PostgreSQL · запросы к пулу, ожидавшие соединение, в секунду — sum by (service_name) (rate(messenger_db_pool_waited_total[1m])) — рядов: 3
Worker · загрузка — sum(messenger_worker_busy) / sum(messenger_worker_capacity) — рядов: 1
Вложения · p95 до ready — histogram_quantile(0.95, sum by (le) (rate(messenger_attachment_ready_seconds_bucket[1m]))) — рядов: 1
Процессы · загрузка CPU — sum by (service_name) (rate(messenger_process_cpu_seconds_total[1m])) — рядов: 3
Процессы · резидентная память — sum by (service_name) (messenger_process_rss_bytes) — рядов: 3
SLI за 30 минут: 0.9992789095149922 цель: >= 0.99
````
Команда `docker compose exec -T prometheus promtool check rules /etc/prometheus/alerts.yaml`. Код выхода **0**. Время 0.23 с.
[Вывод](runs/20261113T155722.910201Z/grade4-dashboard-2.txt)
````text
Checking /etc/prometheus/alerts.yaml
SUCCESS: 2 rules found
````
+25
View File
@@ -0,0 +1,25 @@
# Задержка PostgreSQL и восстановление
Предпосылки — локальный синтетический стенд после make app-up и доступ к Docker.
При блокировке таблицы растёт доля 5xx и алерт срабатывает. Задержку показывает дочерний спан PostgreSQL. После снятия блокировки алерт переходит в resolved.
Команда `/usr/bin/python3 scripts/check_observability_incidents.py --base-url http://127.0.0.1:8080 --grafana-url http://127.0.0.1:3000 database`. Код выхода **0**. Время 104.65 с.
[Вывод](runs/20261113T155722.910201Z/grade4-database-1.txt)
````text
MessengerHighErrorRate firing
pg_terminate_backend
----------------------
t
(1 row)
FATAL: terminating connection due to administrator command
server closed the connection unexpectedly
This probably means the server terminated abnormally
before or while processing the request.
connection to server was lost
Медленный запрос PostgreSQL подтверждён в trace: 624d2df384e6bb8e27ad11ca29a41d0b
MessengerHighErrorRate resolved
````
+125
View File
@@ -0,0 +1,125 @@
# Короткая нагрузка на API
Предпосылки — локальный синтетический стенд после make app-up и доступ к Docker.
Один closed-loop прогон POST сообщений с четырьмя клиентами, отдельно от инцидентов. Снимок ресурсов сделан сразу после прогона.
Команда `lscpu`. Код выхода **0**. Время 0.01 с.
[Вывод](runs/20261113T155722.910201Z/grade4-load-1.txt)
````text
Architecture: x86_64
CPU op-mode(s): 32-bit, 64-bit
Address sizes: 48 bits physical, 48 bits virtual
Byte Order: Little Endian
CPU(s): 16
On-line CPU(s) list: 0-15
Vendor ID: AuthenticAMD
Model name: AMD Ryzen 9 PRO 7940HS w/ Radeon 780M Graphics
CPU family: 25
Model: 116
Thread(s) per core: 2
Core(s) per socket: 8
Socket(s): 1
Stepping: 1
Microcode version: 0xa70410a
Frequency boost: enabled
CPU(s) scaling MHz: 90%
CPU max MHz: 5263,0610
CPU min MHz: 402,7860
BogoMIPS: 7985,14
Flags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good amd_lbr_v2 nopl xtopology nonstop_tsc cpuid extd_apicid aperfmperf rapl pni pclmulqdq monitor ssse3 fma cx16 sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand lahf_lm cmp_legacy svm extapic cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw ibs skinit wdt tce topoext perfctr_core perfctr_nb bpext perfctr_llc mwaitx cpuid_fault cpb cat_l3 cdp_l3 hw_pstate ssbd mba perfmon_v2 ibrs ibpb stibp ibrs_enhanced vmmcall fsgsbase bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local user_shstk avx512_bf16 clzero irperf xsaveerptr rdpru wbnoinvd cppc arat npt lbrv svm_lock nrip_save tsc_scale vmcb_clean flushbyasid decodeassists pausefilter pfthreshold vgif x2avic v_spec_ctrl vnmi avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg avx512_vpopcntdq rdpid overflow_recov succor smca fsrm flush_l1d amd_lbr_pmc_freeze
Virtualization: AMD-V
L1d cache: 256 KiB (8 instances)
L1i cache: 256 KiB (8 instances)
L2 cache: 8 MiB (8 instances)
L3 cache: 16 MiB (1 instance)
NUMA node(s): 1
NUMA node0 CPU(s): 0-15
Vulnerability Gather data sampling: Not affected
Vulnerability Ghostwrite: Not affected
Vulnerability Indirect target selection: Not affected
Vulnerability Itlb multihit: Not affected
Vulnerability L1tf: Not affected
Vulnerability Mds: Not affected
Vulnerability Meltdown: Not affected
Vulnerability Mmio stale data: Not affected
Vulnerability Old microcode: Not affected
Vulnerability Reg file data sampling: Not affected
Vulnerability Retbleed: Not affected
Vulnerability Spec rstack overflow: Mitigation; Safe RET
Vulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl
Vulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization
Vulnerability Spectre v2: Mitigation; Enhanced / Automatic IBRS; IBPB conditional; STIBP always-on; PBRSB-eIBRS Not affected; BHI Not affected
Vulnerability Srbds: Not affected
Vulnerability Tsa: Mitigation; Clear CPU buffers
Vulnerability Tsx async abort: Not affected
Vulnerability Vmscape: Mitigation; IBPB before exit to userspace
````
Команда `free -h`. Код выхода **0**. Время 0.0 с.
[Вывод](runs/20261113T155722.910201Z/grade4-load-2.txt)
````text
total used free shared buff/cache available
Mem: 14Gi 11Gi 1,3Gi 139Mi 2,3Gi 3,1Gi
Swap: 8,0Gi 8,0Gi 548Ki
````
Команда `docker compose version`. Код выхода **0**. Время 0.04 с.
[Вывод](runs/20261113T155722.910201Z/grade4-load-3.txt)
````text
Docker Compose version v5.7.0
````
Команда `/usr/bin/python3 tests/load.py --base-url http://127.0.0.1:8080 --duration 30 --concurrency 4`. Код выхода **0**. Время 31.48 с.
[Вывод](runs/20261113T155722.910201Z/grade4-load-4.txt)
````text
{
"lab": 6,
"workload": "closed-loop POST messages; no automatic retries",
"requested_duration_seconds": 30.0,
"elapsed_seconds": 30.004,
"concurrency": 4,
"successful_requests": 37708,
"failed_requests": 0,
"error_ratio": 0.0,
"successful_rps": 1256.75,
"successful_latency_ms": {
"p50": 2.828,
"p95": 4.327,
"p99": 7.211
},
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
}
````
Команда `docker compose stats --no-stream`. Код выхода **0**. Время 2.17 с.
[Вывод](runs/20261113T155722.910201Z/grade4-load-5.txt)
````text
CONTAINER ID NAME CPU % MEM USAGE / LIMIT MEM % NET I/O BLOCK I/O PIDS
2ee09eab650d highload-lab6-worker-1 0.01% 10.86MiB / 512MiB 2.12% 274kB / 393kB 4.58MB / 0B 7
7fdc55311297 highload-lab6-dispatcher-1 0.03% 10.59MiB / 128MiB 8.27% 1.05MB / 1.41MB 1.33MB / 422kB 7
188cf5f1391d highload-lab6-api-2 2.56% 22.67MiB / 512MiB 4.43% 85.5MB / 161MB 8.96MB / 406kB 7
93535c5e7b67 highload-lab6-api-1 0.06% 21.67MiB / 512MiB 4.23% 86.4MB / 163MB 8.18MB / 389kB 7
ae279d3a717c highload-lab6-upload-cleanup-1 0.00% 4.543MiB / 128MiB 3.55% 35.6kB / 31.2kB 32.8kB / 532kB 12
db0e2da28fb1 highload-lab6-prometheus-1 0.23% 35.93MiB / 14.89GiB 0.24% 3.86MB / 1.91MB 426MB / 6.98MB 22
66b201cac5eb highload-lab6-grafana-1 0.68% 89.27MiB / 14.89GiB 0.59% 59.5MB / 116MB 1.32GB / 242MB 25
0d0412e6fe25 highload-lab6-tempo-1 0.02% 307.1MiB / 14.89GiB 2.01% 9.75MB / 202kB 58.6MB / 55.3MB 21
a8fc40b42323 highload-lab6-loki-1 0.92% 161.8MiB / 14.89GiB 1.06% 5.45MB / 32.4MB 534MB / 24.9MB 21
a2048ea9933c highload-lab6-otel-collector-1 0.19% 60.13MiB / 192MiB 31.32% 123MB / 28.3MB 489MB / 20.9MB 21
efeed09a36ec highload-lab6-frontend-1 0.27% 24.01MiB / 512MiB 4.69% 52.3kB / 7.76MB 562MB / 161MB 36
2ac44f5d15bb highload-lab6-haproxy-1 2.36% 10.5MiB / 128MiB 8.20% 104MB / 106MB 30.2MB / 14MB 17
3999d26f1e22 highload-lab6-s3-1 9.80% 354.4MiB / 512MiB 69.22% 1.07MB / 760kB 125MB / 10.2MB 21
8837cc38b561 highload-lab6-db-1 0.66% 80.51MiB / 512MiB 15.72% 160MB / 109MB 459MB / 5.37GB 18
3e7d2e507d3d highload-lab6-broker-1 23.55% 89.5MiB / 512MiB 17.48% 374kB / 333kB 267MB / 91.5MB 47
65ad13aee585 highload-lab6-redis-1 2.71% 4.18MiB / 128MiB 3.27% 13.8MB / 8.47MB 163MB / 7.52MB 6
````
+45
View File
@@ -0,0 +1,45 @@
# Сквозная трасса и повторная попытка
Предпосылки — локальный синтетический стенд после make app-up и доступ к Docker.
Tempo подтверждает родительскую связь HTTP, producer и consumer, операции S3 и PostgreSQL и две попытки одной загрузки.
Команда `/usr/bin/python3 scripts/check_observability.py --base-url http://127.0.0.1:8080 --grafana-url http://127.0.0.1:3000 pipeline`. Код выхода **0**. Время 4.92 с.
[Вывод](runs/20261113T155722.910201Z/grade4-pipeline-1.txt)
````text
Upload → producer → consumer → S3/PostgreSQL, trace: a1b887f518133aa40e303527d73f514e
````
Команда `/usr/bin/python3 scripts/check_observability_incidents.py --base-url http://127.0.0.1:8080 --grafana-url http://127.0.0.1:3000 retry`. Код выхода **0**. Время 15.62 с.
[Вывод](runs/20261113T155722.910201Z/grade4-pipeline-2.txt)
````text
Container highload-lab6-worker-1 Stopping
Container highload-lab6-worker-1 Stopped
Container highload-lab6-s3-1 Stopping
Container highload-lab6-s3-1 Stopped
Container highload-lab6-worker-1 Starting
Container highload-lab6-worker-1 Started
Container highload-lab6-s3-1 Starting
Container highload-lab6-s3-1 Started
Container highload-lab6-s3-1 Waiting
Container highload-lab6-s3-1 Healthy
Исходное вложение перешло в ready, обе попытки в trace: 762071e58a275355660808c10b605df8
Container highload-lab6-broker-1 Waiting
Container highload-lab6-db-1 Waiting
Container highload-lab6-s3-1 Waiting
Container highload-lab6-db-1 Healthy
Container highload-lab6-migrate-1 Starting
Container highload-lab6-broker-1 Healthy
Container highload-lab6-s3-1 Healthy
Container highload-lab6-s3-bootstrap-1 Starting
Container highload-lab6-migrate-1 Started
Container highload-lab6-migrate-1 Waiting
Container highload-lab6-s3-bootstrap-1 Started
Container highload-lab6-s3-bootstrap-1 Waiting
Container highload-lab6-migrate-1 Exited
Container highload-lab6-s3-bootstrap-1 Exited
````
+32
View File
@@ -0,0 +1,32 @@
# Остановка и восстановление worker
Предпосылки — локальный синтетический стенд после make app-up и доступ к Docker.
Возраст незавершённой работы вызывает firing. Вложение достигает ready после восстановления worker, алерт переходит в resolved.
Команда `/usr/bin/python3 scripts/check_observability_incidents.py --base-url http://127.0.0.1:8080 --grafana-url http://127.0.0.1:3000 worker`. Код выхода **0**. Время 55.56 с.
[Вывод](runs/20261113T155722.910201Z/grade4-queue-1.txt)
````text
Container highload-lab6-worker-1 Stopping
Container highload-lab6-worker-1 Stopped
MessengerQueueLag firing
Container highload-lab6-broker-1 Waiting
Container highload-lab6-db-1 Waiting
Container highload-lab6-s3-1 Waiting
Container highload-lab6-db-1 Healthy
Container highload-lab6-migrate-1 Starting
Container highload-lab6-s3-1 Healthy
Container highload-lab6-s3-bootstrap-1 Starting
Container highload-lab6-broker-1 Healthy
Container highload-lab6-migrate-1 Started
Container highload-lab6-migrate-1 Waiting
Container highload-lab6-s3-bootstrap-1 Started
Container highload-lab6-s3-bootstrap-1 Waiting
Container highload-lab6-migrate-1 Exited
Container highload-lab6-s3-bootstrap-1 Exited
Container highload-lab6-worker-1 Starting
Container highload-lab6-worker-1 Started
MessengerQueueLag resolved
````
+20
View File
@@ -0,0 +1,20 @@
# Недоступность Tempo
Предпосылки — локальный синтетический стенд после make app-up и доступ к Docker.
Проверяются сообщения и готовность вложения при остановленном Tempo, снимок памяти Collector и число отброшенных спанов по его журналу.
Команда `/usr/bin/python3 scripts/check_observability_incidents.py --base-url http://127.0.0.1:8080 --grafana-url http://127.0.0.1:3000 telemetry`. Код выхода **0**. Время 46.39 с.
[Вывод](runs/20261113T155722.910201Z/grade4-telemetry-1.txt)
````text
Container highload-lab6-tempo-1 Stopping
Container highload-lab6-tempo-1 Stopped
CONTAINER ID NAME CPU % MEM USAGE / LIMIT MEM % NET I/O BLOCK I/O PIDS
a2048ea9933c highload-lab6-otel-collector-1 0.42% 56.43MiB / 192MiB 29.39% 69.6MB / 15.8MB 466MB / 20.8MB 21
API и worker работали при остановленном Tempo.
Container highload-lab6-tempo-1 Starting
Container highload-lab6-tempo-1 Started
Спанов отброшено Collector за период проверки: 54
````
@@ -0,0 +1,10 @@
Worker: изображение 3000×3000 завершилось в состоянии failed.
messenger-api: http.request.completed, trace=91735be6f8944614b886c2a34bfc95a8, span=7aca4e9abb3620f6
messenger-worker: job.processing.completed, trace=8f05225357fb027e222f890074249eed, span=49632776d49e21fe
messenger-worker: job.processing.completed, trace=c958e0f21140e39dddb4192b5982442e, span=f459a5c48084c984
Prometheus: messenger_http_requests_total, рядов=40
Prometheus: messenger_http_duration_seconds_bucket, рядов=560
Prometheus: messenger_worker_jobs_total{job_outcome="success"}, рядов=1
Prometheus: messenger_worker_jobs_total{job_outcome="error"}, рядов=1
Prometheus: messenger_worker_duration_seconds_bucket, рядов=42
Проверенные сигналы не содержат контрольных credentials, текста, имени/байтов файла и бизнес-ID.
@@ -0,0 +1,4 @@
make[1]: Entering directory '/home/TAskMAster339/Projects/highload-systems/mtusi2026lab6-fork'
python3 scripts/check_contract.py
PASS: offline skeleton structure, local refs and Python syntax; application NOT tested.
make[1]: Leaving directory '/home/TAskMAster339/Projects/highload-systems/mtusi2026lab6-fork'
@@ -0,0 +1,4 @@
make[1]: Entering directory '/home/TAskMAster339/Projects/highload-systems/mtusi2026lab6-fork'
python3 tests/smoke.py --base-url "http://127.0.0.1:8080"
PASS: lab6 public API scenario; infrastructure and grading evidence remain separate.
make[1]: Leaving directory '/home/TAskMAster339/Projects/highload-systems/mtusi2026lab6-fork'
@@ -0,0 +1,19 @@
NAME IMAGE COMMAND SERVICE CREATED STATUS PORTS
highload-lab6-api-1 highload-lab6-api "./messenger" api 31 minutes ago Up 31 minutes (healthy) 8080/tcp
highload-lab6-api-2 highload-lab6-api "./messenger" api 31 minutes ago Up 31 minutes (healthy) 8080/tcp
highload-lab6-broker-1 rabbitmq:4.2.0-management "docker-entrypoint.s…" broker About an hour ago Up About an hour (healthy) 4369/tcp, 5671-5672/tcp, 15671-15672/tcp, 15691-15692/tcp, 25672/tcp
highload-lab6-db-1 postgres:16.10-bookworm "docker-entrypoint.s…" db About an hour ago Up About an hour (healthy) 5432/tcp
highload-lab6-dispatcher-1 highload-lab6-dispatcher "./messenger dispatc…" dispatcher 31 minutes ago Up 31 minutes 8080/tcp
highload-lab6-frontend-1 sha256:6f2bacde48d02aca251e04c4707e11d6707c797af56bf6623a671b2e0c20da30 "docker-entrypoint.s…" frontend About an hour ago Up About an hour (healthy) 127.0.0.1:5173->5173/tcp
highload-lab6-grafana-1 grafana/grafana:11.5.2 "/run.sh" grafana About an hour ago Up About an hour 127.0.0.1:3000->3000/tcp
highload-lab6-haproxy-1 haproxy:3.2.25-alpine "docker-entrypoint.s…" haproxy About an hour ago Up About an hour (healthy) 127.0.0.1:8080->8080/tcp, 127.0.0.1:8404->8404/tcp
highload-lab6-loki-1 grafana/loki:3.4.2 "/usr/bin/loki -conf…" loki About an hour ago Up About an hour 3100/tcp
highload-lab6-migrate-1 highload-lab6-migrate "./messenger migrate…" migrate 31 minutes ago Exited (0) 3 minutes ago
highload-lab6-otel-collector-1 otel/opentelemetry-collector-contrib:0.120.0 "/otelcol-contrib --…" otel-collector About an hour ago Up About an hour 4317-4318/tcp, 55678-55679/tcp
highload-lab6-prometheus-1 prom/prometheus:v3.2.1 "/bin/prometheus --c…" prometheus 31 minutes ago Up 31 minutes 9090/tcp
highload-lab6-redis-1 redis:7.4.11-bookworm "docker-entrypoint.s…" redis About an hour ago Up About an hour (healthy) 6379/tcp
highload-lab6-s3-1 sha256:b891ef2bb2e30a5a240ee0d267c8938ab43f2ce5e5c89347e580e7c1b1cf4e5e "minio server /data …" s3 About an hour ago Up 3 minutes (healthy) 127.0.0.1:9000->9000/tcp
highload-lab6-s3-bootstrap-1 highload-lab6-s3-bootstrap "./messenger s3-boot…" s3-bootstrap 31 minutes ago Exited (0) 3 minutes ago
highload-lab6-tempo-1 grafana/tempo:2.7.1 "/tempo -config.file…" tempo About an hour ago Up 34 seconds
highload-lab6-upload-cleanup-1 highload-lab6-upload-cleanup "./messenger attachm…" upload-cleanup 31 minutes ago Up 31 minutes 8080/tcp
highload-lab6-worker-1 highload-lab6-worker "./messenger worker" worker 31 minutes ago Up 3 minutes 8080/tcp
@@ -0,0 +1,8 @@
s3_data
loki_data
postgres_data
prometheus_data
redis_data
tempo_data
grafana_data
rabbitmq_data
@@ -0,0 +1,2 @@
Container highload-lab6-otel-collector-run-0fb806dd4ede Creating
Container highload-lab6-otel-collector-run-0fb806dd4ede Created
@@ -0,0 +1,4 @@
Источник Grafana: prometheus
Источник Grafana: loki
Источник Grafana: tempo
Загружен dashboard messenger-lab6, панелей: 17
@@ -0,0 +1,17 @@
API · запросы в секунду — sum(rate(messenger_http_requests_total[1m])) — рядов: 1
API · 5xx в секунду — sum(rate(messenger_http_requests_total{http_status_class="5xx"}[1m])) or 0 * sum(rate(messenger_http_requests_total[1m])) — рядов: 1
API · p95 длительности — histogram_quantile(0.95, sum by (le) (rate(messenger_http_duration_seconds_bucket[1m]))) — рядов: 1
Worker · задания в минуту — 60 * sum by (job_outcome) (rate(messenger_worker_jobs_total[1m])) — рядов: 3
Worker · p95 длительности — histogram_quantile(0.95, sum by (le,job_outcome) (rate(messenger_worker_duration_seconds_bucket[1m]))) — рядов: 3
API · p50 длительности — histogram_quantile(0.5, sum by (le) (rate(messenger_http_duration_seconds_bucket[1m]))) — рядов: 1
API · p99 длительности — histogram_quantile(0.99, sum by (le) (rate(messenger_http_duration_seconds_bucket[1m]))) — рядов: 1
API · доля 5xx — (sum(rate(messenger_http_requests_total{http_status_class="5xx"}[1m])) or 0 * sum(rate(messenger_http_requests_total[1m]))) / sum(rate(messenger_http_requests_total[1m])) — рядов: 1
Очередь attachments.v1 · незавершённые задания — max(messenger_queue_depth) — рядов: 1
Очередь attachments.v1 · возраст старейшего задания — max(messenger_queue_oldest_seconds) — рядов: 1
PostgreSQL · занятые соединения — sum by (service_name) (messenger_db_pool_used) — рядов: 3
PostgreSQL · запросы к пулу, ожидавшие соединение, в секунду — sum by (service_name) (rate(messenger_db_pool_waited_total[1m])) — рядов: 3
Worker · загрузка — sum(messenger_worker_busy) / sum(messenger_worker_capacity) — рядов: 1
Вложения · p95 до ready — histogram_quantile(0.95, sum by (le) (rate(messenger_attachment_ready_seconds_bucket[1m]))) — рядов: 1
Процессы · загрузка CPU — sum by (service_name) (rate(messenger_process_cpu_seconds_total[1m])) — рядов: 3
Процессы · резидентная память — sum by (service_name) (messenger_process_rss_bytes) — рядов: 3
SLI за 30 минут: 0.9992789095149922 цель: >= 0.99
@@ -0,0 +1,2 @@
Checking /etc/prometheus/alerts.yaml
SUCCESS: 2 rules found
@@ -0,0 +1,13 @@
MessengerHighErrorRate firing
pg_terminate_backend
----------------------
t
(1 row)
FATAL: terminating connection due to administrator command
server closed the connection unexpectedly
This probably means the server terminated abnormally
before or while processing the request.
connection to server was lost
Медленный запрос PostgreSQL подтверждён в trace: 624d2df384e6bb8e27ad11ca29a41d0b
MessengerHighErrorRate resolved
@@ -0,0 +1,47 @@
Architecture: x86_64
CPU op-mode(s): 32-bit, 64-bit
Address sizes: 48 bits physical, 48 bits virtual
Byte Order: Little Endian
CPU(s): 16
On-line CPU(s) list: 0-15
Vendor ID: AuthenticAMD
Model name: AMD Ryzen 9 PRO 7940HS w/ Radeon 780M Graphics
CPU family: 25
Model: 116
Thread(s) per core: 2
Core(s) per socket: 8
Socket(s): 1
Stepping: 1
Microcode version: 0xa70410a
Frequency boost: enabled
CPU(s) scaling MHz: 90%
CPU max MHz: 5263,0610
CPU min MHz: 402,7860
BogoMIPS: 7985,14
Flags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good amd_lbr_v2 nopl xtopology nonstop_tsc cpuid extd_apicid aperfmperf rapl pni pclmulqdq monitor ssse3 fma cx16 sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand lahf_lm cmp_legacy svm extapic cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw ibs skinit wdt tce topoext perfctr_core perfctr_nb bpext perfctr_llc mwaitx cpuid_fault cpb cat_l3 cdp_l3 hw_pstate ssbd mba perfmon_v2 ibrs ibpb stibp ibrs_enhanced vmmcall fsgsbase bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local user_shstk avx512_bf16 clzero irperf xsaveerptr rdpru wbnoinvd cppc arat npt lbrv svm_lock nrip_save tsc_scale vmcb_clean flushbyasid decodeassists pausefilter pfthreshold vgif x2avic v_spec_ctrl vnmi avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg avx512_vpopcntdq rdpid overflow_recov succor smca fsrm flush_l1d amd_lbr_pmc_freeze
Virtualization: AMD-V
L1d cache: 256 KiB (8 instances)
L1i cache: 256 KiB (8 instances)
L2 cache: 8 MiB (8 instances)
L3 cache: 16 MiB (1 instance)
NUMA node(s): 1
NUMA node0 CPU(s): 0-15
Vulnerability Gather data sampling: Not affected
Vulnerability Ghostwrite: Not affected
Vulnerability Indirect target selection: Not affected
Vulnerability Itlb multihit: Not affected
Vulnerability L1tf: Not affected
Vulnerability Mds: Not affected
Vulnerability Meltdown: Not affected
Vulnerability Mmio stale data: Not affected
Vulnerability Old microcode: Not affected
Vulnerability Reg file data sampling: Not affected
Vulnerability Retbleed: Not affected
Vulnerability Spec rstack overflow: Mitigation; Safe RET
Vulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl
Vulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization
Vulnerability Spectre v2: Mitigation; Enhanced / Automatic IBRS; IBPB conditional; STIBP always-on; PBRSB-eIBRS Not affected; BHI Not affected
Vulnerability Srbds: Not affected
Vulnerability Tsa: Mitigation; Clear CPU buffers
Vulnerability Tsx async abort: Not affected
Vulnerability Vmscape: Mitigation; IBPB before exit to userspace
@@ -0,0 +1,3 @@
total used free shared buff/cache available
Mem: 14Gi 11Gi 1,3Gi 139Mi 2,3Gi 3,1Gi
Swap: 8,0Gi 8,0Gi 548Ki
@@ -0,0 +1 @@
Docker Compose version v5.7.0
@@ -0,0 +1,17 @@
{
"lab": 6,
"workload": "closed-loop POST messages; no automatic retries",
"requested_duration_seconds": 30.0,
"elapsed_seconds": 30.004,
"concurrency": 4,
"successful_requests": 37708,
"failed_requests": 0,
"error_ratio": 0.0,
"successful_rps": 1256.75,
"successful_latency_ms": {
"p50": 2.828,
"p95": 4.327,
"p99": 7.211
},
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
}
@@ -0,0 +1,17 @@
CONTAINER ID NAME CPU % MEM USAGE / LIMIT MEM % NET I/O BLOCK I/O PIDS
2ee09eab650d highload-lab6-worker-1 0.01% 10.86MiB / 512MiB 2.12% 274kB / 393kB 4.58MB / 0B 7
7fdc55311297 highload-lab6-dispatcher-1 0.03% 10.59MiB / 128MiB 8.27% 1.05MB / 1.41MB 1.33MB / 422kB 7
188cf5f1391d highload-lab6-api-2 2.56% 22.67MiB / 512MiB 4.43% 85.5MB / 161MB 8.96MB / 406kB 7
93535c5e7b67 highload-lab6-api-1 0.06% 21.67MiB / 512MiB 4.23% 86.4MB / 163MB 8.18MB / 389kB 7
ae279d3a717c highload-lab6-upload-cleanup-1 0.00% 4.543MiB / 128MiB 3.55% 35.6kB / 31.2kB 32.8kB / 532kB 12
db0e2da28fb1 highload-lab6-prometheus-1 0.23% 35.93MiB / 14.89GiB 0.24% 3.86MB / 1.91MB 426MB / 6.98MB 22
66b201cac5eb highload-lab6-grafana-1 0.68% 89.27MiB / 14.89GiB 0.59% 59.5MB / 116MB 1.32GB / 242MB 25
0d0412e6fe25 highload-lab6-tempo-1 0.02% 307.1MiB / 14.89GiB 2.01% 9.75MB / 202kB 58.6MB / 55.3MB 21
a8fc40b42323 highload-lab6-loki-1 0.92% 161.8MiB / 14.89GiB 1.06% 5.45MB / 32.4MB 534MB / 24.9MB 21
a2048ea9933c highload-lab6-otel-collector-1 0.19% 60.13MiB / 192MiB 31.32% 123MB / 28.3MB 489MB / 20.9MB 21
efeed09a36ec highload-lab6-frontend-1 0.27% 24.01MiB / 512MiB 4.69% 52.3kB / 7.76MB 562MB / 161MB 36
2ac44f5d15bb highload-lab6-haproxy-1 2.36% 10.5MiB / 128MiB 8.20% 104MB / 106MB 30.2MB / 14MB 17
3999d26f1e22 highload-lab6-s3-1 9.80% 354.4MiB / 512MiB 69.22% 1.07MB / 760kB 125MB / 10.2MB 21
8837cc38b561 highload-lab6-db-1 0.66% 80.51MiB / 512MiB 15.72% 160MB / 109MB 459MB / 5.37GB 18
3e7d2e507d3d highload-lab6-broker-1 23.55% 89.5MiB / 512MiB 17.48% 374kB / 333kB 267MB / 91.5MB 47
65ad13aee585 highload-lab6-redis-1 2.71% 4.18MiB / 128MiB 3.27% 13.8MB / 8.47MB 163MB / 7.52MB 6
@@ -0,0 +1 @@
Upload → producer → consumer → S3/PostgreSQL, trace: a1b887f518133aa40e303527d73f514e
@@ -0,0 +1,25 @@
Container highload-lab6-worker-1 Stopping
Container highload-lab6-worker-1 Stopped
Container highload-lab6-s3-1 Stopping
Container highload-lab6-s3-1 Stopped
Container highload-lab6-worker-1 Starting
Container highload-lab6-worker-1 Started
Container highload-lab6-s3-1 Starting
Container highload-lab6-s3-1 Started
Container highload-lab6-s3-1 Waiting
Container highload-lab6-s3-1 Healthy
Исходное вложение перешло в ready, обе попытки в trace: 762071e58a275355660808c10b605df8
Container highload-lab6-broker-1 Waiting
Container highload-lab6-db-1 Waiting
Container highload-lab6-s3-1 Waiting
Container highload-lab6-db-1 Healthy
Container highload-lab6-migrate-1 Starting
Container highload-lab6-broker-1 Healthy
Container highload-lab6-s3-1 Healthy
Container highload-lab6-s3-bootstrap-1 Starting
Container highload-lab6-migrate-1 Started
Container highload-lab6-migrate-1 Waiting
Container highload-lab6-s3-bootstrap-1 Started
Container highload-lab6-s3-bootstrap-1 Waiting
Container highload-lab6-migrate-1 Exited
Container highload-lab6-s3-bootstrap-1 Exited
@@ -0,0 +1,20 @@
Container highload-lab6-worker-1 Stopping
Container highload-lab6-worker-1 Stopped
MessengerQueueLag firing
Container highload-lab6-broker-1 Waiting
Container highload-lab6-db-1 Waiting
Container highload-lab6-s3-1 Waiting
Container highload-lab6-db-1 Healthy
Container highload-lab6-migrate-1 Starting
Container highload-lab6-s3-1 Healthy
Container highload-lab6-s3-bootstrap-1 Starting
Container highload-lab6-broker-1 Healthy
Container highload-lab6-migrate-1 Started
Container highload-lab6-migrate-1 Waiting
Container highload-lab6-s3-bootstrap-1 Started
Container highload-lab6-s3-bootstrap-1 Waiting
Container highload-lab6-migrate-1 Exited
Container highload-lab6-s3-bootstrap-1 Exited
Container highload-lab6-worker-1 Starting
Container highload-lab6-worker-1 Started
MessengerQueueLag resolved
@@ -0,0 +1,8 @@
Container highload-lab6-tempo-1 Stopping
Container highload-lab6-tempo-1 Stopped
CONTAINER ID NAME CPU % MEM USAGE / LIMIT MEM % NET I/O BLOCK I/O PIDS
a2048ea9933c highload-lab6-otel-collector-1 0.42% 56.43MiB / 192MiB 29.39% 69.6MB / 15.8MB 466MB / 20.8MB 21
API и worker работали при остановленном Tempo.
Container highload-lab6-tempo-1 Starting
Container highload-lab6-tempo-1 Started
Спанов отброшено Collector за период проверки: 54
@@ -0,0 +1,140 @@
{
"grade3-stand": [
{
"command": "docker compose config --quiet",
"exit_code": 0,
"seconds": 0.08,
"output": "grade3-stand-1.txt"
},
{
"command": "docker compose ps --all",
"exit_code": 0,
"seconds": 0.11,
"output": "grade3-stand-2.txt"
},
{
"command": "docker compose config --volumes",
"exit_code": 0,
"seconds": 0.08,
"output": "grade3-stand-3.txt"
},
{
"command": "docker compose run --rm --no-deps otel-collector validate --config=/etc/otelcol-contrib/config.yaml",
"exit_code": 0,
"seconds": 1.28,
"output": "grade3-stand-4.txt"
},
{
"command": "/usr/bin/python3 scripts/check_observability.py --base-url http://127.0.0.1:8080 --grafana-url http://127.0.0.1:3000 provisioning",
"exit_code": 0,
"seconds": 0.13,
"output": "grade3-stand-5.txt"
}
],
"grade3-smoke": [
{
"command": "make check",
"exit_code": 0,
"seconds": 0.05,
"output": "grade3-smoke-1.txt"
},
{
"command": "make test BASE_URL=http://127.0.0.1:8080",
"exit_code": 0,
"seconds": 3.51,
"output": "grade3-smoke-2.txt"
}
],
"grade3-signals": [
{
"command": "/usr/bin/python3 scripts/check_observability.py --base-url http://127.0.0.1:8080 --grafana-url http://127.0.0.1:3000 signals",
"exit_code": 0,
"seconds": 7.2,
"output": "grade3-signals-1.txt"
}
],
"grade4-pipeline": [
{
"command": "/usr/bin/python3 scripts/check_observability.py --base-url http://127.0.0.1:8080 --grafana-url http://127.0.0.1:3000 pipeline",
"exit_code": 0,
"seconds": 4.92,
"output": "grade4-pipeline-1.txt"
},
{
"command": "/usr/bin/python3 scripts/check_observability_incidents.py --base-url http://127.0.0.1:8080 --grafana-url http://127.0.0.1:3000 retry",
"exit_code": 0,
"seconds": 15.62,
"output": "grade4-pipeline-2.txt"
}
],
"grade4-dashboard": [
{
"command": "/usr/bin/python3 scripts/check_observability.py --base-url http://127.0.0.1:8080 --grafana-url http://127.0.0.1:3000 dashboard",
"exit_code": 0,
"seconds": 0.16,
"output": "grade4-dashboard-1.txt"
},
{
"command": "docker compose exec -T prometheus promtool check rules /etc/prometheus/alerts.yaml",
"exit_code": 0,
"seconds": 0.23,
"output": "grade4-dashboard-2.txt"
}
],
"grade4-queue": [
{
"command": "/usr/bin/python3 scripts/check_observability_incidents.py --base-url http://127.0.0.1:8080 --grafana-url http://127.0.0.1:3000 worker",
"exit_code": 0,
"seconds": 55.56,
"output": "grade4-queue-1.txt"
}
],
"grade4-database": [
{
"command": "/usr/bin/python3 scripts/check_observability_incidents.py --base-url http://127.0.0.1:8080 --grafana-url http://127.0.0.1:3000 database",
"exit_code": 0,
"seconds": 104.65,
"output": "grade4-database-1.txt"
}
],
"grade4-telemetry": [
{
"command": "/usr/bin/python3 scripts/check_observability_incidents.py --base-url http://127.0.0.1:8080 --grafana-url http://127.0.0.1:3000 telemetry",
"exit_code": 0,
"seconds": 46.39,
"output": "grade4-telemetry-1.txt"
}
],
"grade4-load": [
{
"command": "lscpu",
"exit_code": 0,
"seconds": 0.01,
"output": "grade4-load-1.txt"
},
{
"command": "free -h",
"exit_code": 0,
"seconds": 0.0,
"output": "grade4-load-2.txt"
},
{
"command": "docker compose version",
"exit_code": 0,
"seconds": 0.04,
"output": "grade4-load-3.txt"
},
{
"command": "/usr/bin/python3 tests/load.py --base-url http://127.0.0.1:8080 --duration 30 --concurrency 4",
"exit_code": 0,
"seconds": 31.48,
"output": "grade4-load-4.txt"
},
{
"command": "docker compose stats --no-stream",
"exit_code": 0,
"seconds": 2.17,
"output": "grade4-load-5.txt"
}
]
}
Binary file not shown.

After

Width:  |  Height:  |  Size: 403 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 423 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 418 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 427 KiB

+49
View File
@@ -0,0 +1,49 @@
global
# Operator access only inside this container; no host port is published.
stats socket ipv4@127.0.0.1:9999 level admin
log stdout format raw local0
maxconn 1024
defaults
log global
mode http
option httplog
timeout connect 1s
timeout client 35s
timeout server 35s
timeout check 1s
# Let the client decide which failed POST is safe to repeat.
retries 0
# Docker returns the current addresses of all containers of the api service.
resolvers docker
nameserver docker_dns 127.0.0.11:53
accepted_payload_size 8192
resolve_retries 3
timeout resolve 1s
timeout retry 1s
hold valid 1s
hold obsolete 1s
hold nx 1s
hold timeout 1s
hold refused 1s
hold other 1s
frontend api_entry
bind :8080
default_backend api_replicas
backend api_replicas
balance roundrobin
option httpchk GET /health/ready
http-check expect status 200
# Separate slots receive distinct Docker DNS addresses. No sticky sessions.
server-template api "${API_REPLICAS}" "api:${APP_PORT}" check inter 1s fall 2 rise 2 resolvers docker resolve-prefer ipv4 init-addr libc,none
# Local teaching aid: backend addresses, UP/DOWN state and request counters.
listen statistics
bind :8404
stats enable
stats show-legends
stats uri /stats
stats refresh 2s
+19
View File
@@ -0,0 +1,19 @@
FROM golang:1.24.2-bookworm AS builder
WORKDIR /source
RUN git init . \
&& git remote add origin https://github.com/minio/minio.git \
&& git fetch --depth 1 origin 0d7408fc9969caf07de6a8c3a84f9fbb10a6739e \
&& git checkout --detach FETCH_HEAD
RUN CGO_ENABLED=0 go build -trimpath \
-ldflags="-s -w -X github.com/minio/minio/cmd.Version=2025-04-22T22:12:26Z -X github.com/minio/minio/cmd.ReleaseTag=RELEASE.2025-04-22T22-12-26Z -X github.com/minio/minio/cmd.CommitID=0d7408fc9969caf07de6a8c3a84f9fbb10a6739e" \
-o /out/minio .
FROM debian:12.15-slim
RUN apt-get update && apt-get install -y --no-install-recommends ca-certificates curl \
&& rm -rf /var/lib/apt/lists/* \
&& groupadd --system storage && useradd --system --gid storage storage \
&& mkdir /data && chown storage:storage /data
COPY --from=builder /out/minio /usr/local/bin/minio
USER storage:storage
ENTRYPOINT ["minio"]
CMD ["server", "/data", "--console-address", ":9001"]
+20
View File
@@ -0,0 +1,20 @@
groups:
- name: messenger
interval: 5s
rules:
- alert: MessengerHighErrorRate
expr: (sum(rate(messenger_http_requests_total{http_status_class="5xx"}[1m])) / sum(rate(messenger_http_requests_total[1m]))) > 0.05
for: 15s
labels:
severity: warning
annotations:
summary: More than 5% of business requests return 5xx over one minute
runbook: Check API error spans and PostgreSQL operations in Tempo.
- alert: MessengerQueueLag
expr: max(messenger_queue_oldest_seconds) > 20
for: 15s
labels:
severity: warning
annotations:
summary: Oldest outstanding attachment job is over 20 seconds old
runbook: Check outstanding jobs, worker saturation and consumer traces.
+45
View File
@@ -0,0 +1,45 @@
receivers:
otlp:
protocols:
http:
endpoint: 0.0.0.0:4318
processors:
memory_limiter:
check_interval: 1s
limit_mib: 128
spike_limit_mib: 32
batch:
timeout: 1s
send_batch_size: 256
send_batch_max_size: 512
exporters:
prometheus:
endpoint: 0.0.0.0:8889
resource_to_telemetry_conversion:
enabled: true
otlphttp/loki:
endpoint: http://loki:3100/otlp
sending_queue:
queue_size: 64
retry_on_failure:
max_elapsed_time: 30s
otlphttp/tempo:
endpoint: http://tempo:4318
sending_queue:
queue_size: 64
retry_on_failure:
max_elapsed_time: 30s
service:
pipelines:
metrics:
receivers: [otlp]
processors: [memory_limiter, batch]
exporters: [prometheus]
logs:
receivers: [otlp]
processors: [memory_limiter, batch]
exporters: [otlphttp/loki]
traces:
receivers: [otlp]
processors: [memory_limiter, batch]
exporters: [otlphttp/tempo]
@@ -0,0 +1,513 @@
{
"uid": "messenger-lab6",
"title": "Мессенджер · API и worker",
"schemaVersion": 39,
"refresh": "5s",
"time": {
"from": "now-15m",
"to": "now"
},
"panels": [
{
"id": 1,
"title": "API · запросы в секунду",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "lab6-prometheus"
},
"targets": [
{
"refId": "A",
"datasource": {
"type": "prometheus",
"uid": "lab6-prometheus"
},
"expr": "sum(rate(messenger_http_requests_total[1m]))",
"legendFormat": ""
}
],
"gridPos": {
"x": 0,
"y": 0,
"w": 12,
"h": 8
},
"fieldConfig": {
"defaults": {
"unit": "reqps"
},
"overrides": []
}
},
{
"id": 2,
"title": "API · 5xx в секунду",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "lab6-prometheus"
},
"targets": [
{
"refId": "A",
"datasource": {
"type": "prometheus",
"uid": "lab6-prometheus"
},
"expr": "sum(rate(messenger_http_requests_total{http_status_class=\"5xx\"}[1m])) or 0 * sum(rate(messenger_http_requests_total[1m]))",
"legendFormat": ""
}
],
"gridPos": {
"x": 12,
"y": 0,
"w": 12,
"h": 8
},
"fieldConfig": {
"defaults": {
"unit": "reqps"
},
"overrides": []
}
},
{
"id": 3,
"title": "API · p95 длительности",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "lab6-prometheus"
},
"targets": [
{
"refId": "A",
"datasource": {
"type": "prometheus",
"uid": "lab6-prometheus"
},
"expr": "histogram_quantile(0.95, sum by (le) (rate(messenger_http_duration_seconds_bucket[1m])))",
"legendFormat": ""
}
],
"gridPos": {
"x": 0,
"y": 8,
"w": 12,
"h": 8
},
"fieldConfig": {
"defaults": {
"unit": "s"
},
"overrides": []
}
},
{
"id": 4,
"title": "Worker · задания в минуту",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "lab6-prometheus"
},
"targets": [
{
"refId": "A",
"datasource": {
"type": "prometheus",
"uid": "lab6-prometheus"
},
"expr": "60 * sum by (job_outcome) (rate(messenger_worker_jobs_total[1m]))",
"legendFormat": "{{job_outcome}}"
}
],
"gridPos": {
"x": 12,
"y": 8,
"w": 12,
"h": 8
},
"fieldConfig": {
"defaults": {
"unit": "short"
},
"overrides": []
}
},
{
"id": 5,
"title": "Worker · p95 длительности",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "lab6-prometheus"
},
"targets": [
{
"refId": "A",
"datasource": {
"type": "prometheus",
"uid": "lab6-prometheus"
},
"expr": "histogram_quantile(0.95, sum by (le,job_outcome) (rate(messenger_worker_duration_seconds_bucket[1m])))",
"legendFormat": "{{job_outcome}}"
}
],
"gridPos": {
"x": 0,
"y": 16,
"w": 12,
"h": 8
},
"fieldConfig": {
"defaults": {
"unit": "s"
},
"overrides": []
}
},
{
"id": 6,
"title": "API и worker · события",
"type": "logs",
"datasource": {
"type": "loki",
"uid": "lab6-loki"
},
"targets": [
{
"refId": "A",
"datasource": {
"type": "loki",
"uid": "lab6-loki"
},
"expr": "{service_name=~\"messenger-api|messenger-worker\"}",
"legendFormat": "",
"queryType": "range"
}
],
"gridPos": {
"x": 12,
"y": 16,
"w": 12,
"h": 8
},
"fieldConfig": {
"defaults": {
"unit": ""
},
"overrides": []
}
},
{
"id": 7,
"title": "API · p50 длительности",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "lab6-prometheus"
},
"gridPos": {
"x": 0,
"y": 24,
"w": 12,
"h": 8
},
"targets": [
{
"refId": "A",
"expr": "histogram_quantile(0.5, sum by (le) (rate(messenger_http_duration_seconds_bucket[1m])))",
"legendFormat": ""
}
],
"fieldConfig": {
"defaults": {
"unit": "s"
},
"overrides": []
}
},
{
"id": 8,
"title": "API · p99 длительности",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "lab6-prometheus"
},
"gridPos": {
"x": 12,
"y": 24,
"w": 12,
"h": 8
},
"targets": [
{
"refId": "A",
"expr": "histogram_quantile(0.99, sum by (le) (rate(messenger_http_duration_seconds_bucket[1m])))",
"legendFormat": ""
}
],
"fieldConfig": {
"defaults": {
"unit": "s"
},
"overrides": []
}
},
{
"id": 9,
"title": "API · доля 5xx",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "lab6-prometheus"
},
"gridPos": {
"x": 0,
"y": 32,
"w": 12,
"h": 8
},
"targets": [
{
"refId": "A",
"expr": "(sum(rate(messenger_http_requests_total{http_status_class=\"5xx\"}[1m])) or 0 * sum(rate(messenger_http_requests_total[1m]))) / sum(rate(messenger_http_requests_total[1m]))",
"legendFormat": ""
}
],
"fieldConfig": {
"defaults": {
"unit": "percentunit"
},
"overrides": []
}
},
{
"id": 10,
"title": "Очередь attachments.v1 · незавершённые задания",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "lab6-prometheus"
},
"gridPos": {
"x": 12,
"y": 32,
"w": 12,
"h": 8
},
"targets": [
{
"refId": "A",
"expr": "max(messenger_queue_depth)",
"legendFormat": ""
}
],
"fieldConfig": {
"defaults": {
"unit": "short"
},
"overrides": []
}
},
{
"id": 11,
"title": "Очередь attachments.v1 · возраст старейшего задания",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "lab6-prometheus"
},
"gridPos": {
"x": 0,
"y": 40,
"w": 12,
"h": 8
},
"targets": [
{
"refId": "A",
"expr": "max(messenger_queue_oldest_seconds)",
"legendFormat": ""
}
],
"fieldConfig": {
"defaults": {
"unit": "s"
},
"overrides": []
}
},
{
"id": 12,
"title": "PostgreSQL · занятые соединения",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "lab6-prometheus"
},
"gridPos": {
"x": 12,
"y": 40,
"w": 12,
"h": 8
},
"targets": [
{
"refId": "A",
"expr": "sum by (service_name) (messenger_db_pool_used)",
"legendFormat": "{{service_name}}"
}
],
"fieldConfig": {
"defaults": {
"unit": "short"
},
"overrides": []
}
},
{
"id": 13,
"title": "PostgreSQL · запросы к пулу, ожидавшие соединение, в секунду",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "lab6-prometheus"
},
"gridPos": {
"x": 0,
"y": 48,
"w": 12,
"h": 8
},
"targets": [
{
"refId": "A",
"expr": "sum by (service_name) (rate(messenger_db_pool_waited_total[1m]))",
"legendFormat": "{{service_name}}"
}
],
"fieldConfig": {
"defaults": {
"unit": "ops"
},
"overrides": []
}
},
{
"id": 14,
"title": "Worker · загрузка",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "lab6-prometheus"
},
"gridPos": {
"x": 12,
"y": 48,
"w": 12,
"h": 8
},
"targets": [
{
"refId": "A",
"expr": "sum(messenger_worker_busy) / sum(messenger_worker_capacity)",
"legendFormat": ""
}
],
"fieldConfig": {
"defaults": {
"unit": "percentunit"
},
"overrides": []
}
},
{
"id": 15,
"title": "Вложения · p95 до ready",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "lab6-prometheus"
},
"gridPos": {
"x": 0,
"y": 56,
"w": 12,
"h": 8
},
"targets": [
{
"refId": "A",
"expr": "histogram_quantile(0.95, sum by (le) (rate(messenger_attachment_ready_seconds_bucket[1m])))",
"legendFormat": ""
}
],
"fieldConfig": {
"defaults": {
"unit": "s"
},
"overrides": []
}
},
{
"id": 16,
"title": "Процессы · загрузка CPU",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "lab6-prometheus"
},
"gridPos": {
"x": 12,
"y": 56,
"w": 12,
"h": 8
},
"targets": [
{
"refId": "A",
"expr": "sum by (service_name) (rate(messenger_process_cpu_seconds_total[1m]))",
"legendFormat": "{{service_name}}"
}
],
"fieldConfig": {
"defaults": {
"unit": "cores"
},
"overrides": []
}
},
{
"id": 17,
"title": "Процессы · резидентная память",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "lab6-prometheus"
},
"gridPos": {
"x": 0,
"y": 64,
"w": 12,
"h": 8
},
"targets": [
{
"refId": "A",
"expr": "sum by (service_name) (messenger_process_rss_bytes)",
"legendFormat": "{{service_name}}"
}
],
"fieldConfig": {
"defaults": {
"unit": "bytes"
},
"overrides": []
}
}
]
}
@@ -0,0 +1,6 @@
apiVersion: 1
providers:
- name: Messenger
type: file
options:
path: /var/lib/grafana/dashboards
@@ -0,0 +1,32 @@
apiVersion: 1
datasources:
- name: Prometheus
uid: lab6-prometheus
type: prometheus
access: proxy
url: http://prometheus:9090
isDefault: true
- name: Loki
uid: lab6-loki
type: loki
access: proxy
url: http://loki:3100
jsonData:
derivedFields:
- name: Trace
matcherType: label
matcherRegex: trace_id
datasourceUid: lab6-tempo
url: '$${__value.raw}'
- name: Tempo
uid: lab6-tempo
type: tempo
access: proxy
url: http://tempo:3200
jsonData:
tracesToLogsV2:
datasourceUid: lab6-loki
spanStartTimeShift: '-5s'
spanEndTimeShift: '5s'
customQuery: true
query: '{service_name=~"messenger-api|messenger-worker"} | trace_id="$${__span.traceId}"'
+31
View File
@@ -0,0 +1,31 @@
auth_enabled: false
server:
http_listen_port: 3100
common:
path_prefix: /loki
replication_factor: 1
ring:
instance_addr: 127.0.0.1
kvstore:
store: inmemory
storage:
filesystem:
chunks_directory: /loki/chunks
rules_directory: /loki/rules
schema_config:
configs:
- from: 2024-01-01
store: tsdb
object_store: filesystem
schema: v13
index:
prefix: index_
period: 24h
limits_config:
allow_structured_metadata: true
otlp_config:
resource_attributes:
ignore_defaults: true
attributes_config:
- action: index_label
attributes: [service.name, deployment.environment.name]
+8
View File
@@ -0,0 +1,8 @@
global:
scrape_interval: 5s
rule_files:
- /etc/prometheus/alerts.yaml
scrape_configs:
- job_name: otel-collector
static_configs:
- targets: [otel-collector:8889]
+27
View File
@@ -0,0 +1,27 @@
# Диагностика API и обработки вложений
SLI доступности API — доля бизнес-запросов без 5xx. Цель — ≥99% за 30 минут при наличии трафика. Проверка:
```promql
1 - (sum(increase(messenger_http_requests_total{http_status_class="5xx"}[30m])) or 0 * sum(increase(messenger_http_requests_total[30m]))) / sum(increase(messenger_http_requests_total[30m]))
```
Health исключён из обоих счётчиков. `MessengerHighErrorRate` срабатывает при доле 5xx выше 5% за минуту в течение 15 секунд. В Explore → Tempo найти ошибочный серверный спан и дочерний `postgresql query`. Сравнить длительность дочернего спана с длительностью HTTP, затем сверить занятые соединения и rate запросов, ожидавших пул.
`MessengerQueueLag` срабатывает, когда старейшее незавершённое задание старше 20 секунд в течение 15 секунд. В dashboard проверить число незавершённых заданий и наличие worker. Если возраст растёт при отсутствии новых consumer spans, проверить процесс worker. После восстановления дождаться перехода заданий в ready и исчезновения активного alert.
Глубина очереди измеряется по устойчивым заданиям в PostgreSQL. В неё входят работа в RabbitMQ, leases и ожидание retry. Возраст измеряется от создания задания. Время до ready включает ожидание и повторные попытки. Гистограммы времени используют секунды. Загрузка worker — доля занятых слотов, по одному слоту на процесс. Ожидание пула отображается как частота успешных получений соединения после ожидания. Метрики CPU и RSS относятся к основным процессам Go.
Правила доступны через API Prometheus `/api/v1/alerts` или запрос `ALERTS{alertstate="firing"}` в Grafana. При недоступности Loki или Tempo приложение продолжает работу. Collector ограничивает память и очередь экспорта, повторяет отправку до 30 секунд. Сигналы сверх лимитов или после истечения retry могут быть потеряны.
Команды локальной проверки после `make app-up`:
```sh
python3 scripts/check_observability.py pipeline
python3 scripts/check_observability_incidents.py worker
python3 scripts/check_observability_incidents.py database
python3 scripts/check_observability_incidents.py retry
python3 scripts/check_observability_incidents.py telemetry
```
Сценарии выполняются последовательно на учебном стенде. DB-инцидент держит временную блокировку таблицы чатов. Проверка retry кратко останавливает S3 и проверяет номера попыток в одной трассе исходного upload. Проверка telemetry останавливает Tempo и одновременно отправляет сообщения и обрабатывает вложение.
+18
View File
@@ -0,0 +1,18 @@
server:
http_listen_port: 3200
distributor:
receivers:
otlp:
protocols:
http:
endpoint: 0.0.0.0:4318
storage:
trace:
backend: local
wal:
path: /var/tempo/wal
local:
path: /var/tempo/blocks
compactor:
compaction:
block_retention: 48h
+204
View File
@@ -0,0 +1,204 @@
#!/usr/bin/env python3
"""Проверка сохранённых сигналов через источники Grafana."""
import argparse
import base64
import json
from pathlib import Path
import re
import struct
import sys
import time
import urllib.parse
import urllib.request
import uuid
import zlib
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "tests"))
from client import Client, Failure, check
def oversized_image():
# 9 million pixels fit the API limit but exceed the default worker limit.
def chunk(kind, data):
return struct.pack(">I", len(data)) + kind + data + struct.pack(">I", zlib.crc32(kind + data))
header = struct.pack(">IIBBBBB", 3000, 3000, 8, 0, 0, 0, 0)
return (bytes.fromhex("89504e470d0a1a0a") + chunk(b"IHDR", header)
+ chunk(b"IDAT", zlib.compress(bytes(3001 * 3000))) + chunk(b"IEND", b""))
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--base-url", default="http://127.0.0.1:8080")
parser.add_argument("--grafana-url", default="http://127.0.0.1:3000")
parser.add_argument("criterion", choices=["provisioning", "signals", "pipeline", "dashboard"])
args = parser.parse_args()
def query(path, **params):
suffix = "?" + urllib.parse.urlencode(params) if params else ""
with urllib.request.urlopen(args.grafana_url.rstrip("/") + path + suffix, timeout=5) as response:
return json.load(response)
def source(uid, path, **params):
return query("/api/datasources/proxy/uid/lab6-" + uid + path, **params)
if args.criterion == "provisioning":
for uid in ("prometheus", "loki", "tempo"):
check(query("/api/datasources/uid/lab6-" + uid)["type"] == uid, "source type mismatch")
print("Источник Grafana:", uid)
dashboard = query("/api/dashboards/uid/messenger-lab6")["dashboard"]
check(len(dashboard["panels"]) >= 6, "dashboard panels missing")
print("Загружен dashboard messenger-lab6, панелей:", len(dashboard["panels"]))
return
if args.criterion == "dashboard":
dashboard = query("/api/dashboards/uid/messenger-lab6")["dashboard"]
check(len(dashboard["panels"]) >= 17, "grade 4 dashboard panels missing")
for panel in dashboard["panels"]:
if panel["datasource"]["type"] != "prometheus":
continue
for target in panel["targets"]:
result = source("prometheus", "/api/v1/query", query=target["expr"])
check(result["status"] == "success", "PromQL query failed")
print(panel["title"], "—", target["expr"], "— рядов:", len(result["data"]["result"]))
expression = ('1 - (sum(increase(messenger_http_requests_total{http_status_class="5xx"}[30m])) '
'or 0 * sum(increase(messenger_http_requests_total[30m]))) '
'/ sum(increase(messenger_http_requests_total[30m]))')
result = source("prometheus", "/api/v1/query", query=expression)["data"]["result"]
check(bool(result), "SLI series missing")
print("SLI за 30 минут:", result[0]["value"][1], "цель: >= 0.99")
return
client = Client(args.base_url)
start = time.time_ns()
alice, bob, _, chat = client.bootstrap()
if args.criterion == "pipeline":
upload = client.call("POST", "/api/v1/chats/" + chat["id"] + "/attachments?filename=sample.png",
202, actor=alice, raw=(ROOT / "tests/fixtures/sample.png").read_bytes(),
headers={"Content-Type": "image/png"}, schema="Attachment")
client.wait_ready(bob, upload["json"])
request_id = upload["headers"]["X-Request-Id"]
deadline = time.monotonic() + 45
while time.monotonic() < deadline:
logs = source("loki", "/loki/api/v1/query_range",
query='{service_name="messenger-api"} | request_id="' + request_id + '"',
start=start, end=time.time_ns())["data"]["result"]
if logs:
trace_id = logs[0]["stream"]["trace_id"]
try:
trace = source("tempo", "/api/traces/" + trace_id)
except urllib.error.HTTPError as error:
if error.code != 404:
raise
time.sleep(2)
continue
spans = [span for batch in trace["batches"] for scope in batch["scopeSpans"] for span in scope["spans"]]
by_id = {span["spanId"]: span for span in spans}
for consumer in (span for span in spans if span["kind"] == "SPAN_KIND_CONSUMER"):
producer = by_id.get(consumer.get("parentSpanId"), {})
api = by_id.get(producer.get("parentSpanId"), {})
children = {span["name"] for span in spans if span.get("parentSpanId") == consumer["spanId"]}
if (producer.get("kind") == "SPAN_KIND_PRODUCER" and api.get("kind") == "SPAN_KIND_SERVER"
and {"postgresql finalize attachment", "s3 read staging and write variants"} <= children):
print("Upload → producer → consumer → S3/PostgreSQL, trace:", trace_id)
return
time.sleep(2)
raise Failure("upload trace did not include the worker pipeline")
marker = "telemetry-secret-" + uuid.uuid4().hex
message = client.message(alice, chat, marker)
request_id = message["headers"]["X-Request-Id"]
attachment, original = client.attachment(alice, chat)
client.wait_ready(bob, attachment)
oversized = client.call("POST", "/api/v1/chats/" + chat["id"] + "/attachments?filename=oversized.png",
202, actor=alice, raw=oversized_image(), headers={"Content-Type": "image/png"},
schema="Attachment")["json"]
deadline = time.monotonic() + 45
while time.monotonic() < deadline:
oversized = client.call("GET", "/api/v1/attachments/" + oversized["id"], actor=alice, schema="Attachment")["json"]
if oversized["status"] in ("failed", "ready"):
break
time.sleep(0.5)
check(oversized["status"] == "failed", "oversized job did not fail with the default worker limit")
print("Worker: изображение 3000×3000 завершилось в состоянии failed.")
bad_auth = "Basic " + base64.b64encode(("unknown:" + marker).encode()).decode()
client.call("POST", "/api/v1/auth/sessions", 401, headers={"Authorization": bad_auth})
forbidden = [
marker, alice["token"], alice["registration"]["password"], chat["id"],
message["json"]["id"], "sample.png", base64.b64encode(original).decode(),
]
def metrics(expr):
return source("prometheus", "/api/v1/query", query=expr)["data"]["result"]
def correlate(service, event, predicate, span_kind):
deadline = time.monotonic() + 45
while time.monotonic() < deadline:
logs = source("loki", "/loki/api/v1/query_range",
query='{service_name="' + service + '"} |= "' + event + '"',
start=start, end=time.time_ns(), limit=100)["data"]["result"]
for stream in logs:
for entry in stream["values"]:
metadata = {**stream["stream"], **(entry[2] if len(entry) > 2 else {})}
if not predicate(metadata):
continue
trace_id, span_id = metadata.get("trace_id", ""), metadata.get("span_id", "")
check(bool(re.fullmatch("[a-f0-9]{32}", trace_id)), "missing trace ID")
check(bool(re.fullmatch("[a-f0-9]{16}", span_id)), "missing span ID")
try:
trace = source("tempo", "/api/traces/" + trace_id)
except urllib.error.HTTPError as exc:
if exc.code == 404:
continue
raise
spans = [span for batch in trace["batches"] for scope in batch["scopeSpans"] for span in scope["spans"]]
span_id_base64 = base64.b64encode(bytes.fromhex(span_id)).decode()
matched = [span for span in spans if span["spanId"] in (span_id, span_id_base64)]
if not matched:
continue
check(len(matched) == 1, "duplicate span ID in trace")
span = matched[0]
check(span["kind"] in (span_kind, "SPAN_KIND_SERVER" if span_kind == 2 else "SPAN_KIND_CONSUMER"), "wrong span kind")
check(int(span["endTimeUnixNano"]) > int(span["startTimeUnixNano"]), "span duration missing")
print(f"{service}: {event}, trace={trace_id}, span={span_id}")
return [logs, trace]
time.sleep(2)
raise Failure(service + ": correlated log and span not found")
# Loki may include each structured-metadata value in its own result stream.
api_data = correlate("messenger-api", "http.request.completed", lambda attrs: attrs.get("request_id") == request_id, 2)
worker_data = correlate("messenger-worker", "job.processing.completed", lambda attrs: attrs.get("job_outcome") == "success" and attrs.get("attachment_id") == attachment["id"], 5)
error_data = correlate("messenger-worker", "job.processing.completed",
lambda attrs: attrs.get("job_outcome") == "error" and attrs.get("attachment_id") == oversized["id"], 5)
expressions = [
"messenger_http_requests_total",
"messenger_http_duration_seconds_bucket",
'messenger_worker_jobs_total{job_outcome="success"}',
'messenger_worker_jobs_total{job_outcome="error"}',
"messenger_worker_duration_seconds_bucket",
]
series = []
for expression in expressions:
deadline = time.monotonic() + 30
samples = metrics(expression)
while not samples and time.monotonic() < deadline:
time.sleep(2)
samples = metrics(expression)
check(bool(samples), "missing metric: " + expression)
for sample in samples:
labels = sample["metric"]
check(not (labels.keys() & {"user_id", "chat_id", "message_id", "attachment_id", "request_id", "url", "filename", "text"}), "unbounded labels")
series.extend(samples)
print(f"Prometheus: {expression}, рядов={len(samples)}")
captured = json.dumps([api_data, worker_data, error_data, series], ensure_ascii=False)
check(all(value not in captured for value in forbidden), "sensitive test value in telemetry")
print("Проверенные сигналы не содержат контрольных credentials, текста, имени/байтов файла и бизнес-ID.")
if __name__ == "__main__":
try:
main()
except (Failure, OSError, ValueError, KeyError) as error:
print("FAIL:", error)
sys.exit(1)
+171
View File
@@ -0,0 +1,171 @@
#!/usr/bin/env python3
"""Локальные проверки отказов уровня 4. Результаты выводятся в терминал."""
import argparse
import json
from pathlib import Path
import re
import subprocess
import sys
import time
import urllib.parse
import urllib.request
import uuid
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "tests"))
from client import Client, Failure
def compose(*args):
subprocess.run(["docker", "compose", *args], cwd=ROOT, check=True, timeout=90)
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("incident", choices=["worker", "database", "telemetry", "retry"])
parser.add_argument("--base-url", default="http://127.0.0.1:8080")
parser.add_argument("--grafana-url", default="http://127.0.0.1:3000")
args = parser.parse_args()
client = Client(args.base_url)
alice, _, _, chat = client.bootstrap()
def trace_for(request_id, matches):
base = args.grafana_url.rstrip("/") + "/api/datasources/proxy/uid/lab6-"
query = urllib.parse.urlencode({"query": '{service_name="messenger-api"} | request_id="' + request_id + '"'})
deadline = time.monotonic() + 35
while time.monotonic() < deadline:
with urllib.request.urlopen(base + "loki/loki/api/v1/query_range?" + query, timeout=5) as response:
logs = json.load(response)["data"]["result"]
if logs:
trace_id = logs[0]["stream"]["trace_id"]
try:
with urllib.request.urlopen(base + "tempo/api/traces/" + trace_id, timeout=5) as response:
trace = json.load(response)
except urllib.error.HTTPError as error:
if error.code != 404:
raise
else:
spans = [span for batch in trace["batches"] for scope in batch["scopeSpans"] for span in scope["spans"]]
if matches(spans):
return trace_id
time.sleep(2)
raise Failure("expected trace was not found")
def alerts(name):
endpoint = args.grafana_url.rstrip("/") + "/api/datasources/proxy/uid/lab6-prometheus/api/v1/alerts"
with urllib.request.urlopen(endpoint, timeout=5) as response:
active = json.load(response)["data"]["alerts"]
return any(item["labels"]["alertname"] == name and item["state"] == "firing" for item in active)
def wait_alert(name, firing, exercise=lambda: None):
deadline = time.monotonic() + 100
while time.monotonic() < deadline:
exercise()
if alerts(name) == firing:
print(name, "firing" if firing else "resolved", flush=True)
return
time.sleep(2)
raise Failure(name + ": expected alert state not observed")
if args.incident == "worker":
compose("stop", "worker")
try:
attachment, _ = client.attachment(alice, chat)
wait_alert("MessengerQueueLag", True)
finally:
compose("start", "worker")
client.wait_ready(alice, attachment)
wait_alert("MessengerQueueLag", False)
elif args.incident == "database":
# Hold a read-blocking lock on chats. The health probe can still reach PostgreSQL.
marker = "lab6-fault-" + uuid.uuid4().hex
command = ["docker", "compose", "exec", "-T", "db", "sh", "-c",
'exec psql -X -v ON_ERROR_STOP=1 -U "$POSTGRES_USER" -d "$POSTGRES_DB" '
'-c "BEGIN; LOCK TABLE chats IN ACCESS EXCLUSIVE MODE; SELECT pg_sleep(75); ROLLBACK; /* $1 */"',
"sh", marker]
lock = subprocess.Popen(command, cwd=ROOT, stdout=subprocess.DEVNULL)
try:
time.sleep(2)
request_ids = []
def exercise():
response = client.call("GET", "/api/v1/chats/" + chat["id"], [500, 503], actor=alice)
request_ids.append(response["headers"]["X-Request-Id"])
wait_alert("MessengerHighErrorRate", True, exercise)
finally:
# Terminate only the synthetic lock session so the transaction rolls back immediately.
compose("exec", "-T", "db", "sh", "-c",
'exec psql -X -U "$POSTGRES_USER" -d "$POSTGRES_DB" -c "$1"', "sh",
"SELECT pg_terminate_backend(pid) FROM pg_stat_activity "
"WHERE pid<>pg_backend_pid() AND query LIKE '%/* " + marker + " */'")
lock.wait(timeout=10)
client.call("GET", "/api/v1/chats/" + chat["id"], actor=alice, schema="Chat")
trace_id = trace_for(request_ids[-1], lambda spans: any(
span["name"] == "postgresql query"
and int(span["endTimeUnixNano"]) - int(span["startTimeUnixNano"]) > 1_000_000_000 for span in spans))
print("Медленный запрос PostgreSQL подтверждён в trace:", trace_id, flush=True)
wait_alert("MessengerHighErrorRate", False)
elif args.incident == "retry":
compose("stop", "worker")
try:
upload = client.call("POST", "/api/v1/chats/" + chat["id"] + "/attachments?filename=sample.png",
202, actor=alice, raw=(ROOT / "tests/fixtures/sample.png").read_bytes(),
headers={"Content-Type": "image/png"}, schema="Attachment")
attachment = upload["json"]
compose("stop", "s3")
try:
compose("up", "-d", "--no-deps", "worker")
deadline = time.monotonic() + 25
while time.monotonic() < deadline:
sql = ("SELECT count(*) FROM attachment_job_attempts a JOIN attachment_jobs j ON j.id=a.job_id "
"WHERE a.next_attempt_at IS NOT NULL AND j.attachment_id='" + str(uuid.UUID(attachment["id"])) + "'")
output = subprocess.check_output(
["docker", "compose", "exec", "-T", "db", "sh", "-c",
'exec psql -X -At -U "$POSTGRES_USER" -d "$POSTGRES_DB" -c "$1"', "sh", sql],
cwd=ROOT, text=True, timeout=10)
if int(output.strip()) > 0:
break
time.sleep(1)
else:
raise Failure("durable retry was not observed")
finally:
compose("up", "-d", "--wait", "s3")
client.wait_ready(alice, attachment)
def retried(spans):
attempts = [a["value"]["intValue"] for span in spans if span["kind"] == "SPAN_KIND_CONSUMER"
for a in span.get("attributes", []) if a["key"] == "job.attempt"]
return len(set(attempts)) >= 2
trace_id = trace_for(upload["headers"]["X-Request-Id"], retried)
print("Исходное вложение перешло в ready, обе попытки в trace:", trace_id, flush=True)
finally:
compose("start", "worker")
else:
since = str(int(time.time()))
compose("stop", "tempo")
try:
for _ in range(8):
client.message(alice, chat, "telemetry-backend-off")
time.sleep(5)
attachment, _ = client.attachment(alice, chat)
client.wait_ready(alice, attachment)
compose("stats", "--no-stream", "otel-collector")
print("API и worker работали при остановленном Tempo.", flush=True)
finally:
compose("start", "tempo")
output = subprocess.check_output(
["docker", "compose", "logs", "--no-color", "--since", since, "otel-collector"],
cwd=ROOT, text=True, timeout=15)
dropped = sum(int(match) for line in output.splitlines() if '"otlphttp/tempo"' in line
for match in re.findall(r'"dropped_items":\s*(\d+)', line))
print("Спанов отброшено Collector за период проверки:", dropped, flush=True)
if __name__ == "__main__":
try:
main()
except (Failure, OSError, subprocess.SubprocessError, ValueError) as error:
print("FAIL:", error, file=sys.stderr)
sys.exit(1)
@@ -0,0 +1,6 @@
#!/usr/bin/env python3
"""Совместимая точка входа для итогового генератора."""
from generate_evidence import main
if __name__ == "__main__":
raise SystemExit(main())
+103
View File
@@ -0,0 +1,103 @@
#!/usr/bin/env python3
"""Сбор воспроизводимых доказательств уровней 3–4."""
import argparse
from datetime import datetime, timezone
import json
from pathlib import Path
import re
import shlex
import subprocess
import sys
import time
ROOT = Path(__file__).resolve().parents[1]
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--base-url", default="http://127.0.0.1:8080")
parser.add_argument("--grafana-url", default="http://127.0.0.1:3000")
args = parser.parse_args()
probe = [sys.executable, "scripts/check_observability.py", "--base-url", args.base_url,
"--grafana-url", args.grafana_url]
incident = [sys.executable, "scripts/check_observability_incidents.py", "--base-url", args.base_url,
"--grafana-url", args.grafana_url]
checks = [
("grade3-stand", "Стенд, тома и provisioning",
[["docker", "compose", "config", "--quiet"],
["docker", "compose", "ps", "--all"],
["docker", "compose", "config", "--volumes"],
["docker", "compose", "run", "--rm", "--no-deps", "otel-collector", "validate", "--config=/etc/otelcol-contrib/config.yaml"],
probe + ["provisioning"]],
"Prometheus хранит временные ряды, Loki — журналы, Tempo — трассы. Collector принимает и маршрутизирует OTLP. Grafana читает данные и загружает dashboard из JSON."),
("grade3-smoke", "Совместимость API при включённой телеметрии",
[["make", "check"], ["make", "test", "BASE_URL=" + args.base_url]],
"Smoke проверяет сессии, сообщения, ACL, идемпотентность и готовность вложения с оригиналом и thumbnail."),
("grade3-signals", "Метрики, структурные логи и трассы API и worker",
[probe + ["signals"]],
"Контрольный POST сопоставляется с логом по X-Request-Id. Оба сервиса сопоставляются с Tempo по trace_id и span_id. Prometheus возвращает счётчики и histogram buckets. Проверка ищет утечки синтетических значений в полученных сигналах."),
("grade4-pipeline", "Сквозная трасса и повторная попытка",
[probe + ["pipeline"], incident + ["retry"]],
"Tempo подтверждает родительскую связь HTTP, producer и consumer, операции S3 и PostgreSQL и две попытки одной загрузки."),
("grade4-dashboard", "Метрики диагностики и SLO",
[probe + ["dashboard"], ["docker", "compose", "exec", "-T", "prometheus", "promtool", "check", "rules", "/etc/prometheus/alerts.yaml"]],
"Проверяются запросы панелей, SLI за 30 минут и синтаксис двух правил."),
("grade4-queue", "Остановка и восстановление worker",
[incident + ["worker"]],
"Возраст незавершённой работы вызывает firing. Вложение достигает ready после восстановления worker, алерт переходит в resolved."),
("grade4-database", "Задержка PostgreSQL и восстановление",
[incident + ["database"]],
"При блокировке таблицы растёт доля 5xx и алерт срабатывает. Задержку показывает дочерний спан PostgreSQL. После снятия блокировки алерт переходит в resolved."),
("grade4-telemetry", "Недоступность Tempo",
[incident + ["telemetry"]],
"Проверяются сообщения и готовность вложения при остановленном Tempo, снимок памяти Collector и число отброшенных спанов по его журналу."),
("grade4-load", "Короткая нагрузка на API",
[["lscpu"], ["free", "-h"], ["docker", "compose", "version"],
[sys.executable, "tests/load.py", "--base-url", args.base_url, "--duration", "30", "--concurrency", "4"],
["docker", "compose", "stats", "--no-stream"]],
"Один closed-loop прогон POST сообщений с четырьмя клиентами, отдельно от инцидентов. Снимок ресурсов сделан сразу после прогона."),
]
output_root = ROOT / "evidence/criteria"
stamp = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%S.%fZ")
run = output_root / "runs" / stamp
run.mkdir(parents=True)
summary = {}
failed = False
for name, title, commands, explanation in checks:
lines = [f"# {title}", "", "Предпосылки — локальный синтетический стенд после make app-up и доступ к Docker.",
"", explanation, ""]
results = []
summary[name] = results
for index, command in enumerate(commands, 1):
started = time.monotonic()
try:
result = subprocess.run(command, cwd=ROOT, stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
text=True, timeout=240)
code, output = result.returncode, result.stdout
except subprocess.TimeoutExpired:
code, output = 124, "Истёк лимит времени."
except OSError:
code, output = 127, "Команда недоступна."
failed |= code != 0
output = re.sub(r"(https?://[^\s?]+)\?[^\s]+", r"\1?[redacted]", output).strip()
output = "\n".join(line.rstrip() for line in output.splitlines())
raw = f"{name}-{index}.txt"
(run / raw).write_text(output + "\n", encoding="utf-8")
seconds = round(time.monotonic() - started, 2)
results.append(dict(command=shlex.join(command), exit_code=code, seconds=seconds, output=raw))
lines.extend([f"Команда `{shlex.join(command)}`. Код выхода **{code}**. Время {seconds} с.",
"", f"[Вывод](runs/{stamp}/{raw})", "", "````text", output or "(пусто)", "````", ""])
path = output_root / (name + ".md")
path.write_text("\n".join(lines), encoding="utf-8")
print(path.relative_to(ROOT), flush=True)
index = ["# Доказательства критериев", "", f"[Сводка прогона {stamp}](runs/{stamp}/summary.json)", ""]
for name, results in summary.items():
state = "PASS" if all(result["exit_code"] == 0 for result in results) else "FAIL"
index.append(f"- [{name}]({name}.md) — {state}")
(output_root / "README.md").write_text("\n".join(index) + "\n", encoding="utf-8")
(run / "summary.json").write_text(json.dumps(summary, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
return int(failed)
if __name__ == "__main__":
sys.exit(main())
+122
View File
@@ -0,0 +1,122 @@
import os
import secrets
from pathlib import Path
from urllib.parse import quote
ROOT = Path(__file__).resolve().parents[1]
def read_values(path):
values = {}
if path.exists():
for line in path.read_text(encoding="utf-8").splitlines():
line = line.strip()
if line and not line.startswith("#") and "=" in line:
key, value = line.split("=", 1)
values[key.strip()] = value.strip().strip("\"'")
return values
def main():
path = ROOT / ".env"
existing = read_values(path)
defaults = read_values(ROOT / ".env.example")
defaults.update(
POSTGRES_DB="messenger",
POSTGRES_USER="messenger",
POSTGRES_PASSWORD=secrets.token_hex(24),
)
defaults.update(API_HTTP_PORT="8080", UPLOAD_CLEANUP_GRACE_SECONDS="600", UPLOAD_TMP_DIR="/tmp/uploads",
S3_PUBLIC_ENDPOINT="http://127.0.0.1:9000",
S3_CORS_ORIGINS="http://127.0.0.1:5173,http://localhost:5173")
values = defaults | existing
for name, lower, upper in (
("APP_PORT", 1, 65535),
("API_REPLICAS", 2, 32),
("SESSION_TTL_SECONDS", 1, 31536000),
("IDEMPOTENCY_TTL_SECONDS", 1, 31536000),
):
try:
valid = lower <= int(values[name]) <= upper
except (ValueError, KeyError):
valid = False
if not valid:
raise SystemExit(f"{name} must be between {lower} and {upper}")
for name in (
"POSTGRES_DB",
"POSTGRES_USER",
"POSTGRES_PASSWORD",
"APP_HOST",
"APP_ENV",
):
if not values[name]:
raise SystemExit(f"{name} must not be empty")
# Existing values, including credentials, are never replaced.
missing = {key: value for key, value in values.items() if key not in existing}
if missing:
prefix = "\n" if path.exists() and path.stat().st_size else ""
with path.open("a", encoding="utf-8") as file:
file.write(
prefix + "".join(f"{key}={value}\n" for key, value in missing.items())
)
os.chmod(path, 0o600)
secret_dir = ROOT / "secrets"
secret_dir.mkdir(mode=0o700, exist_ok=True)
database_url = "postgres://{}:{}@db:5432/{}?sslmode=disable".format(
quote(values["POSTGRES_USER"], safe=""),
quote(values["POSTGRES_PASSWORD"], safe=""),
quote(values["POSTGRES_DB"], safe=""),
)
for filename,content in (("broker_user","messenger"),("broker_password",secrets.token_hex(32))):
target=secret_dir/filename
if not target.exists():
target.write_text(content+"\n",encoding="utf-8")
os.chmod(target,0o644)
broker_url="amqp://{}:{}@broker:5672/%2F".format(
quote((secret_dir/"broker_user").read_text().strip(),safe=""),
quote((secret_dir/"broker_password").read_text().strip(),safe=""))
# Migrate only the generated legacy Redis URL; preserve custom AMQP settings.
broker_file=secret_dir/"broker_url"
if broker_file.exists() and broker_file.read_text().strip()=="redis://broker:6379/0":
broker_file.write_text(broker_url+"\n",encoding="utf-8")
for name, content in (
("DATABASE_URL_FILE", database_url),
("CACHE_URL_FILE", "redis://redis:6379/0"),
("BROKER_URL_FILE", broker_url),
("S3_ACCESS_KEY_ID_FILE", secrets.token_hex(12)),
("S3_SECRET_ACCESS_KEY_FILE", secrets.token_hex(32)),
):
container_path = Path(values[name])
if container_path.parent != Path("/run/secrets") or not container_path.name:
raise SystemExit(f"{name} must name a file directly inside /run/secrets")
target = (
secret_dir
/ {
"DATABASE_URL_FILE": "database_url",
"CACHE_URL_FILE": "cache_url",
"BROKER_URL_FILE": "broker_url",
"S3_ACCESS_KEY_ID_FILE": "s3_access_key_id",
"S3_SECRET_ACCESS_KEY_FILE": "s3_secret_access_key",
}[name]
)
if not target.exists():
# Containers run as an unprivileged user; the private host directory
# protects the files, while read permission lets the API consume them.
with target.open("x", encoding="utf-8") as file:
file.write(content + "\n")
os.chmod(target, 0o644)
# Existing S3 files remain the root credentials; API gets a new, separate identity.
for filename, content in (
("s3_api_access_key_id", secrets.token_hex(12)),
("s3_api_secret_access_key", secrets.token_hex(32)),
):
target = secret_dir / filename
if not target.exists():
with target.open("x", encoding="utf-8") as file:
file.write(content + "\n")
os.chmod(target, 0o644)
print("Local .env and connection files ready; existing values and files preserved.")
if __name__ == "__main__":
main()
+7
View File
@@ -0,0 +1,7 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
docker compose --project-name highload-lab6 --env-file .env -f compose.yml restart db redis s3 broker >/dev/null
docker compose --project-name highload-lab6 --env-file .env -f compose.yml up -d --wait db redis s3 broker >/dev/null
docker compose --project-name highload-lab6 --env-file .env -f compose.yml restart api upload-cleanup worker dispatcher >/dev/null
docker compose --project-name highload-lab6 --env-file .env -f compose.yml up -d --wait >/dev/null
+16
View File
@@ -0,0 +1,16 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
: "${TARGET_INSTANCE_ID:?Set TARGET_INSTANCE_ID to the X-Instance-Id of an API replica}"
for container in $(docker compose --project-name highload-lab6 --env-file .env -f compose.yml ps -q api); do
hostname=$(docker inspect --format '{{.Config.Hostname}}' "$container")
if [ "$hostname" = "$TARGET_INSTANCE_ID" ]; then
# Only containers from this Compose project's api service are eligible.
docker stop --timeout 8 "$container" >/dev/null
printf 'Stopped API instance %s\n' "$hostname"
exit 0
fi
done
printf 'No running API replica matches TARGET_INSTANCE_ID\n' >&2
exit 1
+26
View File
@@ -0,0 +1,26 @@
.git
.gitignore
.github
.env
.env.*
!.env.example
secrets
certs
volumes
data
backups
*.key
*.pem
*.p12
*.pfx
*.jks
.venv
__pycache__
node_modules
evidence
.DS_Store
README.md
REPORT.md
COURSE.md
CONTRIBUTING.md
MAINTAINERS.md
+42
View File
@@ -0,0 +1,42 @@
FROM golang:1.22.2-bookworm AS builder
RUN apt-get update && apt-get install -y --no-install-recommends webp && rm -rf /var/lib/apt/lists/*
WORKDIR /src
COPY go.mod go.sum ./
RUN go mod download
COPY backend ./backend
RUN CGO_ENABLED=0 \
GOOS=linux \
go build \
-trimpath \
-ldflags="-s -w" \
-o /out/messenger \
./backend
FROM debian:12.15-slim
RUN apt-get update && apt-get install -y --no-install-recommends webp && rm -rf /var/lib/apt/lists/* \
&& groupadd --system app \
&& useradd --system --gid app --no-create-home app
WORKDIR /app
COPY --from=builder /out/messenger ./messenger
ENV APP_HOST=0.0.0.0
ENV APP_PORT=8080
EXPOSE 8080
HEALTHCHECK --interval=10s --timeout=3s --start-period=5s --retries=3 \
CMD ["./messenger", "healthcheck"]
USER app:app
CMD ["./messenger"]
+150
View File
@@ -0,0 +1,150 @@
package app
import (
"context"
"encoding/json"
"errors"
"fmt"
"log"
"net"
"net/http"
"os"
"os/signal"
"runtime"
"syscall"
"time"
"messenger/src/backend/handlers"
"messenger/src/backend/repositories"
"messenger/src/backend/services"
"messenger/src/backend/telemetry"
)
func serverAddress() string {
host := os.Getenv("APP_HOST")
if host == "" {
host = "0.0.0.0"
}
port := os.Getenv("APP_PORT")
if port == "" {
port = "8080"
}
return net.JoinHostPort(host, port)
}
func RunHTTP() error {
if _, err := repositories.UploadCleanupGrace(); err != nil {
return err
}
ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
db, err := repositories.OpenDatabase(ctx)
cancel()
if err != nil {
return fmt.Errorf("database initialization failed: %w", err)
}
defer db.Close()
// Opt-in experiment telemetry goes to container logs, never a public port.
if os.Getenv("BENCH_POOL_STATS") == "1" {
go func() {
for range time.Tick(time.Second) {
s := db.Stat()
data, _ := json.Marshal(map[string]any{
"time": time.Now().UTC(), "acquired": s.AcquiredConns(),
"total": s.TotalConns(), "max": s.MaxConns(),
"gomaxprocs": runtime.GOMAXPROCS(0), "goroutines": runtime.NumGoroutine(),
"acquire_count": s.AcquireCount(), "acquire_duration_ns": s.AcquireDuration().Nanoseconds(),
"empty_acquire_count": s.EmptyAcquireCount(), "canceled_acquire_count": s.CanceledAcquireCount(),
})
log.Printf("benchmark_pool %s", data)
}
}()
}
redisClient, err := repositories.RedisClientFromEnvironment()
if err != nil {
return err
}
defer redisClient.Close()
sessionStore := repositories.NewSessionStore(redisClient, "messenger:session:")
userStore := repositories.NewUserStore(db)
chatStore := repositories.NewChatStore(db)
idempotencyTTL, err := repositories.IdempotencyTTL()
if err != nil {
return err
}
messageStore := repositories.NewMessageStoreWithTTL(db, idempotencyTTL)
ttl, err := services.SessionTTL()
if err != nil {
return err
}
userService := services.NewUserService(userStore)
sessionService := services.NewSessionService(userService, sessionStore, ttl)
chatService := services.NewChatService(chatStore)
messageService := services.NewMessageService(messageStore)
objects, err := repositories.NewObjectStoreFromEnvironment()
if err != nil {
return err
}
attachmentService := services.NewAttachmentService(repositories.NewAttachmentStore(db), objects)
mux := handlers.NewRouter(services.NewHealthService(repositories.NewHealthStore(db), sessionStore, objects), userService, sessionService, chatService, messageService, attachmentService)
signals, stopTelemetry, err := telemetry.Start("messenger-api")
if err != nil {
return fmt.Errorf("telemetry initialization failed: %w", err)
}
defer stopTelemetry()
if err := signals.ObservePool(db); err != nil {
return err
}
server := &http.Server{
Addr: serverAddress(),
Handler: signals.HTTP(mux),
ReadHeaderTimeout: 5 * time.Second,
ReadTimeout: 30 * time.Second,
WriteTimeout: 30 * time.Second,
IdleTimeout: 60 * time.Second,
}
shutdownContext, stop := signal.NotifyContext(
context.Background(),
os.Interrupt,
syscall.SIGTERM,
)
defer stop()
serverErrors := make(chan error, 1)
go func() {
log.Printf("server is listening on %s", server.Addr)
if err := server.ListenAndServe(); err != nil &&
!errors.Is(err, http.ErrServerClosed) {
serverErrors <- err
}
}()
select {
case err := <-serverErrors:
log.Printf("server error: %v", err)
case <-shutdownContext.Done():
log.Printf("shutdown signal received")
handlers.BeginShutdown()
shutdownDeadline, cancel := context.WithTimeout(
context.Background(),
10*time.Second,
)
defer cancel()
if err := server.Shutdown(shutdownDeadline); err != nil {
log.Printf("graceful shutdown failed: %v", err)
return nil
}
log.Printf("server stopped gracefully")
}
return nil
}
+64
View File
@@ -0,0 +1,64 @@
package commands
import (
"context"
"encoding/json"
"flag"
"log"
"os"
"os/signal"
"syscall"
"time"
"messenger/src/backend/repositories"
)
func runUploadCleanup(args []string) error {
flags := flag.NewFlagSet("attachments-cleanup", flag.ContinueOnError)
loop := flags.Bool("loop", false, "run every 60 seconds")
dryRun := flags.Bool("dry-run", false, "report without deleting anything")
if err := flags.Parse(args); err != nil {
return err
}
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
defer stop()
init, cancel := context.WithTimeout(ctx, 20*time.Second)
db, err := repositories.OpenDatabase(init)
cancel()
if err != nil {
return err
}
defer db.Close()
objects, err := repositories.NewObjectStoreFromEnvironment()
if err != nil {
return err
}
store := repositories.NewAttachmentStore(db)
for {
pass, cancel := context.WithTimeout(ctx, 30*time.Second)
result, err := store.CleanupUploads(pass, *dryRun, objects.Delete)
direct, directErr := store.CleanupDirectUploads(pass, *dryRun, objects.Delete)
jobs, jobsErr := store.CleanupJobs(pass, 24*time.Hour, *dryRun, objects.Delete)
cancel()
if encodeErr := json.NewEncoder(os.Stdout).Encode(map[string]any{"originals": result, "direct": direct, "job_staging": jobs}); encodeErr != nil {
return encodeErr
}
if !*loop {
if err == nil {
err = directErr
if err == nil {
err = jobsErr
}
}
return err
}
if err != nil || directErr != nil || jobsErr != nil {
log.Print("upload cleanup pass failed; journal retained for retry")
}
select {
case <-ctx.Done():
return nil
case <-time.After(60 * time.Second):
}
}
}
+76
View File
@@ -0,0 +1,76 @@
package commands
import (
"context"
"encoding/json"
"errors"
"os"
"strconv"
"time"
"github.com/google/uuid"
"messenger/src/backend/repositories"
)
func runJobAdmin(args []string) error {
if len(args) < 1 {
return errors.New("usage: jobs list-failed | replay JOB_ID REASON | cleanup-staging JOB_ID | upgrade ATTACHMENT_ID 2 REASON")
}
ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
defer cancel()
db, err := repositories.OpenDatabase(ctx)
if err != nil {
return errors.New("database unavailable")
}
defer db.Close()
store := repositories.NewAttachmentStore(db)
if args[0] == "list-failed" && len(args) == 1 {
jobs, e := store.FailedJobs(ctx)
if e != nil {
return errors.New("cannot read failed registry")
}
return json.NewEncoder(os.Stdout).Encode(jobs)
}
if len(args) < 2 {
return errors.New("job id required")
}
if _, err = uuid.Parse(args[1]); err != nil {
return errors.New("invalid job id")
}
objects, err := repositories.NewObjectStoreFromEnvironment()
if err != nil {
return err
}
var changed bool
switch {
case args[0] == "upgrade" && len(args) == 4:
version, e := strconv.Atoi(args[2])
if e != nil {
return errors.New("invalid pipeline version")
}
changed, err = store.UpgradePipeline(ctx, args[1], version, args[3], func(ctx context.Context, key string) error {
r, e := objects.Get(ctx, key)
if e == nil {
r.Close()
}
return e
})
case args[0] == "replay" && len(args) == 3:
changed, err = store.ReplayJob(ctx, args[1], args[2], func(ctx context.Context, key string) error {
r, e := objects.Get(ctx, key)
if e == nil {
r.Close()
}
return e
})
case args[0] == "cleanup-staging" && len(args) == 2:
changed, err = store.CleanupJobStaging(ctx, args[1], 24*time.Hour, false, objects.Delete)
default:
return errors.New("usage: jobs list-failed | replay JOB_ID REASON | cleanup-staging JOB_ID | upgrade ATTACHMENT_ID 2 REASON")
}
if err != nil {
return err
}
return json.NewEncoder(os.Stdout).Encode(map[string]bool{"changed": changed})
}
+79
View File
@@ -0,0 +1,79 @@
package commands
import (
"context"
"errors"
"log"
"net"
"net/http"
"os"
"time"
"messenger/src/backend/app"
"messenger/src/backend/repositories"
"messenger/src/backend/workers"
)
// Run dispatches the existing executable modes; the binary and CLI stay the same.
func Run(args []string) error {
if len(args) == 0 {
if environment := os.Getenv("APP_ENV"); environment != "" {
log.SetPrefix("[" + environment + "] ")
}
return app.RunHTTP()
}
switch args[0] {
case "image-process":
return workers.RunImageChild(args[1:])
case "jobs":
return runJobAdmin(args[1:])
case "worker", "dispatcher", "broker-publish":
return workers.RunPipeline(args[0], args[1:])
}
if environment := os.Getenv("APP_ENV"); environment != "" {
log.SetPrefix("[" + environment + "] ")
}
switch {
case args[0] == "s3-bootstrap" && len(args) == 1:
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
if err := repositories.BootstrapS3(ctx); err != nil {
return err
}
log.Print("private S3 bucket ready")
return nil
case args[0] == "attachments-cleanup":
return runUploadCleanup(args[1:])
case args[0] == "healthcheck" && len(args) == 1:
return healthcheck()
case args[0] == "migrate" && len(args) == 2 && (args[1] == "up" || args[1] == "down"):
url, err := repositories.DatabaseURL()
if err != nil {
return err
}
if err = repositories.RunMigrations(url, args[1]); err != nil {
return err
}
log.Printf("migration %s complete", args[1])
return nil
default:
return errors.New("usage: messenger migrate up|down")
}
}
func healthcheck() error {
client := http.Client{Timeout: 2 * time.Second}
port := os.Getenv("APP_PORT")
if port == "" {
port = "8080"
}
response, err := client.Get("http://" + net.JoinHostPort("127.0.0.1", port) + "/health/ready")
if err != nil {
return err
}
response.Body.Close()
if response.StatusCode != http.StatusOK {
return errors.New("API is not ready")
}
return nil
}
+165
View File
@@ -0,0 +1,165 @@
package handlers
import (
"context"
"errors"
"github.com/google/uuid"
"io"
"log"
"messenger/src/backend/repositories"
"messenger/src/backend/services"
"net/http"
"net/url"
"strconv"
"strings"
"time"
)
func uploadAttachmentHandler(chats *services.ChatService, attachments *services.AttachmentService) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
requestID := newRequestID()
w.Header().Set("X-Request-Id", requestID)
if r.Method != http.MethodPost {
writeError(w, 405, "method_not_allowed", "method not allowed", requestID)
return
}
userID, ok := requestUserID(w, r, requestID)
if !ok {
return
}
raw := strings.TrimSuffix(strings.TrimPrefix(r.URL.Path, "/api/v1/chats/"), "/attachments")
chatID, err := uuid.Parse(raw)
if err != nil {
writeError(w, 400, "invalid_request", "invalid chat id", requestID)
return
}
if _, err = chats.Get(r.Context(), userID, chatID.String()); err != nil {
writeServiceError(w, err, requestID)
return
}
query, err := urlQuery(r)
if err != nil {
writeServiceError(w, err, requestID)
return
}
filenames := query["filename"]
if len(filenames) != 1 {
writeError(w, 400, "invalid_request", "exactly one filename is required", requestID)
return
}
if err = services.ValidateFilename(filenames[0]); err != nil {
writeServiceError(w, err, requestID)
return
}
if r.ContentLength > services.MaxUploadBytes {
writeServiceError(w, services.ErrPayloadTooLarge, requestID)
return
}
r.Body = http.MaxBytesReader(w, r.Body, services.MaxUploadBytes)
ctx, cancel := context.WithTimeout(r.Context(), 25*time.Second)
defer cancel()
attachment, err := attachments.Upload(ctx, userID, chatID.String(), filenames[0], r.Header.Get("Content-Type"), r.Body)
var tooLarge *http.MaxBytesError
if errors.As(err, &tooLarge) {
err = services.ErrPayloadTooLarge
}
if err != nil {
writeServiceError(w, err, requestID)
return
}
writeJSON(w, 202, attachment)
}
}
func attachmentHandler(attachments *services.AttachmentService) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
requestID := newRequestID()
w.Header().Set("X-Request-Id", requestID)
if r.Method != http.MethodGet {
writeError(w, 405, "method_not_allowed", "method not allowed", requestID)
return
}
userID, ok := requestUserID(w, r, requestID)
if !ok {
return
}
path := strings.TrimPrefix(r.URL.Path, "/api/v1/attachments/")
parts := strings.Split(path, "/")
if len(parts) > 2 || (len(parts) == 2 && parts[1] != "content") {
writeServiceError(w, services.ErrAttachmentNotFound, requestID)
return
}
id, err := uuid.Parse(parts[0])
if err != nil {
writeError(w, 400, "invalid_request", "invalid attachment id", requestID)
return
}
a, err := attachments.Get(r.Context(), id.String(), userID)
if err != nil {
writeServiceError(w, err, requestID)
return
}
if len(parts) == 1 {
writeJSON(w, 200, a)
return
}
query, err := urlQuery(r)
if err != nil {
writeServiceError(w, err, requestID)
return
}
if a.Status != "ready" {
writeServiceError(w, services.ErrConflict, requestID)
return
}
if variants, exists := query["variant"]; exists {
if len(variants) != 1 {
writeError(w, 400, "invalid_request", "invalid variant", requestID)
return
}
if variants[0] == "thumbnail" || variants[0] == "crop" || variants[0] == "optimized" {
found := false
for _, raw := range a.Variants {
if v, ok := raw.(map[string]any); ok && v["name"] == variants[0] {
a.ObjectKey = repositories.VariantObjectKey(a.ID, a.PipelineVersion, variants[0])
a.ContentType = v["content_type"].(string)
a.SizeBytes = int64(v["size_bytes"].(float64))
found = true
break
}
}
if !found {
writeServiceError(w, services.ErrAttachmentNotFound, requestID)
return
}
} else if variants[0] != "original" {
writeServiceError(w, services.ErrAttachmentNotFound, requestID)
return
}
}
ctx, cancel := context.WithTimeout(r.Context(), 25*time.Second)
defer cancel()
content, err := attachments.Content(ctx, a)
if err != nil {
writeServiceError(w, err, requestID)
return
}
defer content.Close()
w.Header().Set("Content-Type", a.ContentType)
w.Header().Set("Content-Length", strconv.FormatInt(a.SizeBytes, 10))
w.Header().Set("X-Content-Type-Options", "nosniff")
w.Header().Set("Cache-Control", "private, no-store")
w.WriteHeader(200)
if _, err = io.Copy(w, content); err != nil {
log.Printf("attachment download interrupted, request %s", requestID)
}
}
}
func urlQuery(r *http.Request) (url.Values, error) {
q, err := url.ParseQuery(r.URL.RawQuery)
if err != nil {
return nil, services.InvalidInput{Message: "invalid query"}
}
return q, nil
}
+137
View File
@@ -0,0 +1,137 @@
package handlers
import (
"net/http"
"strings"
"messenger/src/backend/services"
"github.com/google/uuid"
)
type createChatRequest struct {
Title string `json:"title"`
MemberIDs []string `json:"member_ids"`
}
func createChatHandler(chats *services.ChatService) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
requestID := newRequestID()
w.Header().Set("X-Request-Id", requestID)
if r.Method != http.MethodPost {
writeError(w, 405, "method_not_allowed", "method not allowed", requestID)
return
}
userID, ok := requestUserID(w, r, requestID)
if !ok {
return
}
if !strings.HasPrefix(r.Header.Get("Content-Type"), "application/json") {
writeError(w, 415, "unsupported_media_type", "content type must be application/json", requestID)
return
}
var request createChatRequest
if err := decodeJSONBody(r, &request); err != nil {
writeError(w, 400, "invalid_request", "invalid JSON body", requestID)
return
}
chat, err := chats.Create(r.Context(), userID, request.Title, request.MemberIDs)
if err != nil {
writeServiceError(w, err, requestID)
return
}
writeJSON(w, 201, chat)
}
}
func getChatHandler(chats *services.ChatService) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
requestID := newRequestID()
w.Header().Set("X-Request-Id", requestID)
if r.Method != http.MethodGet {
writeError(w, 405, "method_not_allowed", "method not allowed", requestID)
return
}
userID, ok := requestUserID(w, r, requestID)
if !ok {
return
}
chatID := strings.TrimPrefix(r.URL.Path, "/api/v1/chats/")
if chatID == "" || strings.Contains(chatID, "/") {
writeError(w, 400, "invalid_request", "invalid chat id", requestID)
return
}
if _, err := uuid.Parse(chatID); err != nil {
writeError(w, 400, "invalid_request", "invalid chat id", requestID)
return
}
chat, err := chats.Get(r.Context(), userID, chatID)
if err != nil {
writeServiceError(w, err, requestID)
return
}
writeJSON(w, 200, chat)
}
}
func listChatsHandler(chats *services.ChatService) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
requestID := newRequestID()
w.Header().Set("X-Request-Id", requestID)
if r.Method != http.MethodGet {
writeError(w, 405, "method_not_allowed", "method not allowed", requestID)
return
}
userID, ok := requestUserID(w, r, requestID)
if !ok {
return
}
limit, cursor, ok := pageParams(w, r, requestID)
if !ok {
return
}
page, err := chats.List(r.Context(), userID, limit, cursor)
if err != nil {
writeServiceError(w, err, requestID)
return
}
writeJSON(w, 200, page)
}
}
func chatCollectionHandler(chats *services.ChatService) http.HandlerFunc {
create := createChatHandler(chats)
list := listChatsHandler(chats)
return func(w http.ResponseWriter, r *http.Request) {
if r.Method == http.MethodGet {
list(w, r)
} else {
create(w, r)
}
}
}
func chatSubresourceHandler(chats *services.ChatService, messages *services.MessageService, attachments *services.AttachmentService) http.HandlerFunc {
get := getChatHandler(chats)
createMessage := createMessageHandler(chats, messages)
listMessages := listMessagesHandler(chats, messages)
return func(w http.ResponseWriter, r *http.Request) {
if attachments != nil && strings.Contains(r.URL.Path, "/attachment-uploads") {
directUploadHandler(chats, attachments)(w, r)
return
}
if attachments != nil && strings.HasSuffix(r.URL.Path, "/attachments") {
uploadAttachmentHandler(chats, attachments)(w, r)
return
}
if strings.HasSuffix(r.URL.Path, "/messages") {
if r.Method == http.MethodGet {
listMessages(w, r)
} else {
createMessage(w, r)
}
return
}
get(w, r)
}
}
@@ -0,0 +1,81 @@
package handlers
import (
"context"
"encoding/json"
"errors"
"io"
"net/http"
"strings"
"time"
"messenger/src/backend/services"
"github.com/google/uuid"
)
func directUploadHandler(chats *services.ChatService, attachments *services.AttachmentService) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
requestID := newRequestID()
w.Header().Set("X-Request-Id", requestID)
if r.Method != http.MethodPost {
writeError(w, 405, "method_not_allowed", "method not allowed", requestID)
return
}
user, ok := requestUserID(w, r, requestID)
if !ok {
return
}
parts := strings.Split(strings.TrimPrefix(r.URL.Path, "/api/v1/chats/"), "/")
if (len(parts) != 2 && (len(parts) != 4 || parts[3] != "complete")) || parts[1] != "attachment-uploads" {
writeError(w, 404, "not_found", "upload endpoint not found", requestID)
return
}
chat, err := uuid.Parse(parts[0])
if err != nil {
writeError(w, 400, "invalid_request", "invalid chat id", requestID)
return
}
if _, err = chats.Get(r.Context(), user, chat.String()); err != nil {
writeServiceError(w, err, requestID)
return
}
ctx, cancel := context.WithTimeout(r.Context(), 25*time.Second)
defer cancel()
if len(parts) == 4 {
id, err := uuid.Parse(parts[2])
if err != nil {
writeError(w, 400, "invalid_request", "invalid upload id", requestID)
return
}
a, err := attachments.CompleteDirect(ctx, user, chat.String(), id.String())
if err != nil {
writeServiceError(w, err, requestID)
return
}
writeJSON(w, 202, a)
return
}
r.Body = http.MaxBytesReader(w, r.Body, 4096)
var input services.DirectUploadInput
decoder := json.NewDecoder(r.Body)
decoder.DisallowUnknownFields()
err = decoder.Decode(&input)
if err == nil {
var extra any
if next := decoder.Decode(&extra); !errors.Is(next, io.EOF) {
err = errors.New("expected one JSON object")
}
}
if err != nil {
writeError(w, 400, "invalid_request", "invalid upload JSON (maximum 4 KiB)", requestID)
return
}
result, err := attachments.InitDirect(ctx, user, chat.String(), input)
if err != nil {
writeServiceError(w, err, requestID)
return
}
writeJSON(w, 201, result)
}
}
+163
View File
@@ -0,0 +1,163 @@
package handlers
import (
"context"
"crypto/rand"
"encoding/hex"
"encoding/json"
"errors"
"log"
"net/http"
"strconv"
"time"
"messenger/src/backend/repositories"
"messenger/src/backend/services"
)
type healthResponse struct {
Status string `json:"status"`
}
type apiError struct {
Code string `json:"code"`
Message string `json:"message"`
RequestID string `json:"request_id"`
}
type errorResponse struct {
Error apiError `json:"error"`
}
func newRequestID() string {
bytes := make([]byte, 16)
if _, err := rand.Read(bytes); err != nil {
return "request-id-generation-failed"
}
return hex.EncodeToString(bytes)
}
func writeError(w http.ResponseWriter, status int, code, message, requestID string) {
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(status)
response := errorResponse{Error: apiError{Code: code, Message: message, RequestID: requestID}}
if err := json.NewEncoder(w).Encode(response); err != nil {
log.Printf("failed to encode error response: %v", err)
}
}
func writeServiceError(w http.ResponseWriter, err error, requestID string) {
var upload repositories.UploadError
if errors.As(err, &upload) {
writeError(w, upload.Status, upload.Code, upload.Message, requestID)
return
}
var invalid services.InvalidInput
switch {
case errors.As(err, &invalid):
writeError(w, 400, "invalid_request", invalid.Message, requestID)
case errors.Is(err, services.ErrPayloadTooLarge):
writeError(w, 413, "payload_too_large", "image exceeds upload limits", requestID)
case errors.Is(err, services.ErrUnsupportedMedia):
writeError(w, 415, "unsupported_media_type", "a valid PNG or JPEG matching Content-Type is required", requestID)
case errors.Is(err, services.ErrAttachmentNotFound):
writeError(w, 404, "not_found", "attachment not found", requestID)
case errors.Is(err, services.ErrConflict):
writeError(w, 409, "conflict", "request conflicts with an existing resource", requestID)
case errors.Is(err, services.ErrUnknownUser):
writeError(w, 401, "unauthorized", "unknown user", requestID)
case errors.Is(err, services.ErrChatNotFound):
writeError(w, 404, "not_found", "chat not found", requestID)
default:
writeError(w, 503, "unavailable", "dependency unavailable", requestID)
}
}
func writeJSON(w http.ResponseWriter, status int, value any) {
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(status)
if err := json.NewEncoder(w).Encode(value); err != nil {
log.Printf("failed to encode response: %v", err)
}
}
func requestUserID(w http.ResponseWriter, r *http.Request, requestID string) (string, bool) {
id := currentUserID(r)
if id == "" {
writeError(w, 401, "unauthorized", "valid Bearer session is required", requestID)
return "", false
}
return id, true
}
func pageParams(w http.ResponseWriter, r *http.Request, requestID string) (int, string, bool) {
limit := 50
if values, exists := r.URL.Query()["limit"]; exists {
if len(values) != 1 || values[0] == "" {
writeError(w, 400, "invalid_request", "invalid limit", requestID)
return 0, "", false
}
parsed, err := strconv.Atoi(values[0])
if err != nil {
writeError(w, 400, "invalid_request", "limit must be between 1 and 100", requestID)
return 0, "", false
}
limit = parsed
}
var cursor string
if values, exists := r.URL.Query()["cursor"]; exists {
if len(values) != 1 || values[0] == "" {
writeError(w, 400, "invalid_request", "invalid cursor", requestID)
return 0, "", false
}
cursor = values[0]
}
return limit, cursor, true
}
func healthHandler(w http.ResponseWriter, r *http.Request) {
requestID := newRequestID()
w.Header().Set("X-Request-Id", requestID)
if r.Method != http.MethodGet {
writeError(w, 405, "method_not_allowed", "method not allowed", requestID)
return
}
writeJSON(w, 200, healthResponse{"ok"})
}
func readyHandler(health *services.HealthService) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
requestID := newRequestID()
w.Header().Set("X-Request-Id", requestID)
if r.Method != http.MethodGet {
writeError(w, 405, "method_not_allowed", "method not allowed", requestID)
return
}
// Readiness must finish before HAProxy's one-second check read timeout.
ctx, cancel := context.WithTimeout(r.Context(), 500*time.Millisecond)
defer cancel()
if err := health.Ready(ctx); err != nil {
writeError(w, 503, "unavailable", "dependency unavailable", requestID)
return
}
writeJSON(w, 200, healthResponse{"ok"})
}
}
func NewRouter(health *services.HealthService, users *services.UserService, sessions *services.SessionService, chats *services.ChatService, messages *services.MessageService, attachmentServices ...*services.AttachmentService) http.Handler {
var attachments *services.AttachmentService
if len(attachmentServices) > 0 {
attachments = attachmentServices[0]
}
mux := http.NewServeMux()
mux.HandleFunc("/health/live", healthHandler)
mux.HandleFunc("/health/ready", readyHandler(health))
mux.Handle("/api/v1/users", createUserHandler(users))
mux.Handle("/api/v1/auth/sessions", sessionHandler(sessions))
mux.Handle("/api/v1/auth/sessions/current", authRequired(sessions, logoutHandler(sessions)))
mux.Handle("/api/v1/users/me", authRequired(sessions, currentUserHandler(users)))
mux.Handle("/api/v1/chats", authRequired(sessions, chatCollectionHandler(chats)))
mux.Handle("/api/v1/chats/", authRequired(sessions, chatSubresourceHandler(chats, messages, attachments)))
if attachments != nil {
mux.Handle("/api/v1/attachments/", authRequired(sessions, attachmentHandler(attachments)))
}
return withInstance(mux)
}
+34
View File
@@ -0,0 +1,34 @@
package handlers
import (
"net/http"
"os"
"sync/atomic"
"github.com/google/uuid"
)
// Docker's hostname lets the operator map an API response to its container.
var instanceID = func() string {
id, err := os.Hostname()
if err != nil || id == "" {
return uuid.NewString()
}
return id
}()
var draining atomic.Bool
func BeginShutdown() { draining.Store(true) }
func withInstance(next http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("X-Instance-Id", instanceID)
w.Header().Set("X-Request-Id", newRequestID())
if draining.Load() && r.URL.Path != "/health/live" {
writeError(w, 503, "unavailable", "instance is shutting down", w.Header().Get("X-Request-Id"))
return
}
next.ServeHTTP(w, r)
})
}
+127
View File
@@ -0,0 +1,127 @@
package handlers
import (
"encoding/json"
"net/http"
"strings"
"github.com/google/uuid"
"messenger/src/backend/services"
)
type createMessageRequest struct {
Text *string `json:"text"`
AttachmentIDs json.RawMessage `json:"attachment_ids"`
}
func messageChatID(path string) (string, bool) {
const prefix = "/api/v1/chats/"
const suffix = "/messages"
if !strings.HasPrefix(path, prefix) || !strings.HasSuffix(path, suffix) {
return "", false
}
chatID := strings.TrimSuffix(strings.TrimPrefix(path, prefix), suffix)
if chatID == "" || strings.Contains(chatID, "/") {
return "", false
}
return chatID, true
}
func createMessageHandler(chats *services.ChatService, messages *services.MessageService) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
requestID := newRequestID()
w.Header().Set("X-Request-Id", requestID)
if r.Method != http.MethodPost {
writeError(w, 405, "method_not_allowed", "method not allowed", requestID)
return
}
userID, ok := requestUserID(w, r, requestID)
if !ok {
return
}
chatID, ok := messageChatID(r.URL.Path)
if !ok {
writeError(w, 400, "invalid_request", "invalid chat id", requestID)
return
}
if _, err := uuid.Parse(chatID); err != nil {
writeError(w, 400, "invalid_request", "invalid chat id", requestID)
return
}
chat, err := chats.Get(r.Context(), userID, chatID)
if err != nil {
writeServiceError(w, err, requestID)
return
}
if !strings.HasPrefix(r.Header.Get("Content-Type"), "application/json") {
writeError(w, 415, "unsupported_media_type", "content type must be application/json", requestID)
return
}
var request createMessageRequest
if err := decodeJSONBody(r, &request); err != nil {
writeError(w, 400, "invalid_request", "invalid JSON body", requestID)
return
}
ids := []string{}
if request.Text == nil {
writeError(w, 400, "invalid_request", "text must be a string", requestID)
return
}
if len(request.AttachmentIDs) > 0 {
if err := json.Unmarshal(request.AttachmentIDs, &ids); err != nil || ids == nil {
writeError(w, 400, "invalid_request", "attachment_ids must be an array of UUIDs", requestID)
return
}
}
keys := r.Header.Values("Idempotency-Key")
if len(keys) != 1 {
writeError(w, 400, "invalid_request", "exactly one Idempotency-Key is required", requestID)
return
}
message, err := messages.CreateIdempotent(r.Context(), userID, chat, *request.Text, keys[0], ids)
if err != nil {
writeServiceError(w, err, requestID)
return
}
writeJSON(w, 201, message)
}
}
func listMessagesHandler(chats *services.ChatService, messages *services.MessageService) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
requestID := newRequestID()
w.Header().Set("X-Request-Id", requestID)
if r.Method != http.MethodGet {
writeError(w, 405, "method_not_allowed", "method not allowed", requestID)
return
}
userID, ok := requestUserID(w, r, requestID)
if !ok {
return
}
chatID, ok := messageChatID(r.URL.Path)
if !ok {
writeError(w, 400, "invalid_request", "invalid chat id", requestID)
return
}
if _, err := uuid.Parse(chatID); err != nil {
writeError(w, 400, "invalid_request", "invalid chat id", requestID)
return
}
chat, err := chats.Get(r.Context(), userID, chatID)
if err != nil {
writeServiceError(w, err, requestID)
return
}
limit, cursor, ok := pageParams(w, r, requestID)
if !ok {
return
}
page, err := messages.List(r.Context(), userID, chat, limit, cursor)
if err != nil {
writeServiceError(w, err, requestID)
return
}
writeJSON(w, 200, page)
}
}
+98
View File
@@ -0,0 +1,98 @@
package handlers
import (
"context"
"errors"
"net/http"
"strings"
"time"
"messenger/src/backend/services"
)
type authenticatedUserKey struct{}
func currentUserID(r *http.Request) string {
id, _ := r.Context().Value(authenticatedUserKey{}).(string)
return id
}
func authRequired(sessions *services.SessionService, next http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
authorization := r.Header.Get("Authorization")
if !strings.HasPrefix(authorization, "Bearer ") || strings.TrimSpace(strings.TrimPrefix(authorization, "Bearer ")) == "" {
unauthorized(w, "Bearer", newRequestID())
return
}
user, err := sessions.User(r.Context(), strings.TrimPrefix(authorization, "Bearer "))
if errors.Is(err, services.ErrInvalidSession) {
unauthorized(w, "Bearer", newRequestID())
return
}
if err != nil {
requestID := newRequestID()
w.Header().Set("X-Request-Id", requestID)
writeError(w, 503, "unavailable", "dependency unavailable", requestID)
return
}
next.ServeHTTP(w, r.WithContext(context.WithValue(r.Context(), authenticatedUserKey{}, user.ID)))
})
}
func unauthorized(w http.ResponseWriter, scheme, requestID string) {
w.Header().Set("X-Request-Id", requestID)
if scheme == "Basic" {
w.Header().Set("WWW-Authenticate", `Basic realm="messenger", charset="UTF-8"`)
} else {
w.Header().Set("WWW-Authenticate", `Bearer realm="messenger"`)
}
writeError(w, 401, "unauthorized", "invalid credentials", requestID)
}
func sessionHandler(sessions *services.SessionService) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
requestID := newRequestID()
w.Header().Set("X-Request-Id", requestID)
if r.Method != http.MethodPost {
writeError(w, 405, "method_not_allowed", "method not allowed", requestID)
return
}
username, password, ok := r.BasicAuth()
if !ok || r.ContentLength > 0 {
unauthorized(w, "Basic", requestID)
return
}
user, token, expires, valid, err := sessions.Login(r.Context(), username, password)
if err != nil {
writeError(w, 503, "unavailable", "dependency unavailable", requestID)
return
}
if !valid {
unauthorized(w, "Basic", requestID)
return
}
writeJSON(w, 201, struct {
Token string `json:"token"`
TokenType string `json:"token_type"`
ExpiresAt time.Time `json:"expires_at"`
User any `json:"user"`
}{token, "Bearer", expires, user})
}
}
func logoutHandler(sessions *services.SessionService) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
requestID := newRequestID()
w.Header().Set("X-Request-Id", requestID)
if r.Method != http.MethodDelete {
writeError(w, 405, "method_not_allowed", "method not allowed", requestID)
return
}
token := strings.TrimPrefix(r.Header.Get("Authorization"), "Bearer ")
if err := sessions.Revoke(r.Context(), token); err != nil {
writeError(w, 503, "unavailable", "dependency unavailable", requestID)
return
}
w.WriteHeader(204)
}
}
+103
View File
@@ -0,0 +1,103 @@
package handlers
import (
"encoding/json"
"fmt"
"io"
"net/http"
"reflect"
"strings"
"messenger/src/backend/services"
)
type createUserRequest struct {
Username string `json:"username"`
DisplayName string `json:"display_name"`
Password string `json:"password"`
}
func decodeJSONBody(r *http.Request, destination any) error {
decoder := json.NewDecoder(r.Body)
var body json.RawMessage
if err := decoder.Decode(&body); err != nil {
return err
}
var extra any
if err := decoder.Decode(&extra); err != io.EOF {
if err != nil {
return err
}
return fmt.Errorf("more than one JSON value")
}
var fields map[string]json.RawMessage
if err := json.Unmarshal(body, &fields); err != nil {
return err
}
if fields == nil {
return fmt.Errorf("JSON body must be an object")
}
// encoding/json matches struct fields case-insensitively, even with
// DisallowUnknownFields. Check exact contract names before decoding values.
typeOfRequest := reflect.TypeOf(destination).Elem()
allowed := make(map[string]bool, typeOfRequest.NumField())
for i := 0; i < typeOfRequest.NumField(); i++ {
name := strings.Split(typeOfRequest.Field(i).Tag.Get("json"), ",")[0]
if name != "" && name != "-" {
allowed[name] = true
}
}
for name := range fields {
if !allowed[name] {
return fmt.Errorf("unknown JSON field %q", name)
}
}
return json.Unmarshal(body, destination)
}
func createUserHandler(users *services.UserService) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
requestID := newRequestID()
w.Header().Set("X-Request-Id", requestID)
if r.Method != http.MethodPost {
writeError(w, 405, "method_not_allowed", "method not allowed", requestID)
return
}
if !strings.HasPrefix(r.Header.Get("Content-Type"), "application/json") {
writeError(w, 415, "unsupported_media_type", "content type must be application/json", requestID)
return
}
var request createUserRequest
if err := decodeJSONBody(r, &request); err != nil {
writeError(w, 400, "invalid_request", "invalid JSON body", requestID)
return
}
user, err := users.Create(r.Context(), request.Username, request.DisplayName, request.Password)
if err != nil {
writeServiceError(w, err, requestID)
return
}
writeJSON(w, 201, user)
}
}
func currentUserHandler(users *services.UserService) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
requestID := newRequestID()
w.Header().Set("X-Request-Id", requestID)
if r.Method != http.MethodGet {
writeError(w, 405, "method_not_allowed", "method not allowed", requestID)
return
}
userID, ok := requestUserID(w, r, requestID)
if !ok {
return
}
user, err := users.Get(r.Context(), userID)
if err != nil {
writeServiceError(w, err, requestID)
return
}
writeJSON(w, 200, user)
}
}
+64
View File
@@ -0,0 +1,64 @@
package faults
import (
"context"
"log"
"os"
"path/filepath"
"time"
"github.com/google/uuid"
)
// Only an operator-mounted directory enables faults; clients cannot set them.
// Rename makes each injected crash one-shot across automatic container restarts.
func Take(point, id string) bool {
directory := os.Getenv("PIPELINE_FAULT_DIR")
if directory == "" {
return false
}
if _, err := uuid.Parse(id); err != nil {
return false
}
name := filepath.Join(directory, point+"."+id)
return os.Rename(name, name+".hit") == nil
}
func Crash(point, id string) {
if Take(point, id) {
log.Printf("injected pipeline crash: %s, attachment %s", point, id)
os.Exit(86)
}
}
// Observe records a test-only branch without exposing an API or a lease token.
func Observe(point, id string) {
directory := os.Getenv("PIPELINE_FAULT_DIR")
if directory == "" {
return
}
if _, err := uuid.Parse(id); err != nil {
return
}
_ = os.WriteFile(filepath.Join(directory, point+"."+id+".observed"), []byte("observed\n"), 0644)
}
// Pause is operator-only and ends on release or context expiry.
func Pause(ctx context.Context, point, id string) {
if !Take(point, id) {
return
}
release := filepath.Join(os.Getenv("PIPELINE_FAULT_DIR"), point+"."+id+".release")
ticker := time.NewTicker(50 * time.Millisecond)
defer ticker.Stop()
for {
select {
case <-ctx.Done():
return
case <-ticker.C:
if _, err := os.Stat(release); err == nil {
_ = os.WriteFile(release+".observed", []byte("released\n"), 0644)
return
}
}
}
}
+17
View File
@@ -0,0 +1,17 @@
package main
import (
"log"
"os"
"messenger/src/backend/commands"
)
func main() {
if err := commands.Run(os.Args[1:]); err != nil {
if len(os.Args) > 1 && (os.Args[1] == "image-process" || os.Args[1] == "healthcheck") {
os.Exit(1)
}
log.Fatal(err)
}
}
+48
View File
@@ -0,0 +1,48 @@
package repositories
import (
"context"
"encoding/json"
"github.com/jackc/pgx/v5/pgxpool"
"time"
)
type Attachment struct {
ID string `json:"id"`
ChatID string `json:"chat_id"`
OwnerID string `json:"owner_id"`
Filename string `json:"filename"`
ContentType string `json:"content_type"`
SizeBytes int64 `json:"size_bytes"`
SHA256 string `json:"sha256"`
Status string `json:"status"`
Variants []any `json:"variants"`
ErrorCode *string `json:"error_code"`
CreatedAt time.Time `json:"created_at"`
UpdatedAt time.Time `json:"updated_at"`
ObjectKey string `json:"-"`
PipelineVersion int `json:"-"`
}
type AttachmentStore struct{ db *pgxpool.Pool }
func NewAttachmentStore(db *pgxpool.Pool) *AttachmentStore { return &AttachmentStore{db: db} }
func (s *AttachmentStore) Create(ctx context.Context, a Attachment) error {
ctx, cancel := boundedQuery(ctx)
defer cancel()
_, err := s.db.Exec(ctx, `INSERT INTO attachments(id,chat_id,owner_id,filename,object_key,content_type,size_bytes,sha256,created_at,updated_at) VALUES($1,$2,$3,$4,$5,$6,$7,$8,$9,$10)`, a.ID, a.ChatID, a.OwnerID, a.Filename, a.ObjectKey, a.ContentType, a.SizeBytes, a.SHA256, a.CreatedAt, a.UpdatedAt)
return err
}
func (s *AttachmentStore) GetForMember(ctx context.Context, id, userID string) (Attachment, error) {
ctx, cancel := boundedQuery(ctx)
defer cancel()
a := Attachment{Variants: []any{}}
var variants []byte
err := s.db.QueryRow(ctx, `SELECT a.id,a.chat_id,a.owner_id,a.filename,a.object_key,a.content_type,a.size_bytes,a.sha256,a.created_at,a.updated_at,a.status,a.variants,a.error_code,a.published_pipeline_version FROM attachments a JOIN chat_members cm ON cm.chat_id=a.chat_id WHERE a.id=$1 AND cm.user_id=$2`, id, userID).Scan(&a.ID, &a.ChatID, &a.OwnerID, &a.Filename, &a.ObjectKey, &a.ContentType, &a.SizeBytes, &a.SHA256, &a.CreatedAt, &a.UpdatedAt, &a.Status, &variants, &a.ErrorCode, &a.PipelineVersion)
if err == nil {
err = json.Unmarshal(variants, &a.Variants)
}
a.CreatedAt = a.CreatedAt.UTC()
a.UpdatedAt = a.UpdatedAt.UTC()
return a, err
}
+83
View File
@@ -0,0 +1,83 @@
package repositories
import (
"context"
"errors"
"messenger/src/backend/telemetry"
"net"
"time"
amqp "github.com/rabbitmq/amqp091-go"
)
const jobQueue = "attachments.v1"
type JobBroker struct {
connection *amqp.Connection
channel *amqp.Channel
returned chan amqp.Return
}
func NewJobBroker(ctx context.Context, address string) (*JobBroker, error) {
connection, err := amqp.DialConfig(address, amqp.Config{
Heartbeat: 10 * time.Second,
Dial: func(network, address string) (net.Conn, error) {
return (&net.Dialer{Timeout: 5 * time.Second}).DialContext(ctx, network, address)
},
})
if err != nil {
return nil, errors.New("RabbitMQ connection unavailable")
}
channel, err := connection.Channel()
if err != nil {
connection.Close()
return nil, errors.New("RabbitMQ channel unavailable")
}
b := &JobBroker{connection: connection, channel: channel, returned: make(chan amqp.Return, 1)}
// Queue metadata and message bodies both survive a broker restart.
_, err = channel.QueueDeclare(jobQueue, true, false, false, false, nil)
if err == nil {
err = channel.Confirm(false)
}
if err == nil {
err = channel.Qos(1, 0, false)
}
if err != nil {
b.Close()
return nil, errors.New("RabbitMQ queue initialization failed")
}
channel.NotifyReturn(b.returned)
return b, nil
}
func (b *JobBroker) Close() { b.channel.Close(); b.connection.Close() }
func (b *JobBroker) Publish(ctx context.Context, body []byte) error {
// mandatory catches an unroutable message; confirms alone do not prove routing.
headers := amqp.Table{}
for key, value := range telemetry.TraceHeaders(ctx) {
headers[key] = value
}
confirmation, err := b.channel.PublishWithDeferredConfirmWithContext(ctx, "", jobQueue, true, false, amqp.Publishing{
DeliveryMode: amqp.Persistent, ContentType: "application/json", Body: body, Headers: headers,
})
if err != nil {
return errors.New("RabbitMQ publication failed")
}
confirmed, err := confirmation.WaitContext(ctx)
if err != nil || !confirmed {
return errors.New("RabbitMQ publication not confirmed")
}
// basic.return precedes basic.ack on the channel; NotifyReturn is buffered.
select {
case <-b.returned:
return errors.New("RabbitMQ message was not routed")
default:
return nil
}
}
func (b *JobBroker) Consume() (<-chan amqp.Delivery, error) {
return b.channel.Consume(jobQueue, "", false, false, false, false, nil)
}
+129
View File
@@ -0,0 +1,129 @@
package repositories
import (
"context"
"errors"
"time"
"github.com/google/uuid"
"github.com/jackc/pgx/v5/pgxpool"
)
type Chat struct {
ID string `json:"id"`
Title string `json:"title"`
MemberIDs []string `json:"member_ids"`
CreatedAt time.Time `json:"created_at"`
}
type ChatStore struct {
db *pgxpool.Pool
}
func NewChatStore(db *pgxpool.Pool) *ChatStore {
return &ChatStore{db: db}
}
var ErrUnknownMember = errors.New("unknown member")
func (s *ChatStore) Create(ctx context.Context, title string, memberIDs []string) (Chat, error) {
ctx, cancel := boundedQuery(ctx)
defer cancel()
membersCopy := append([]string(nil), memberIDs...)
chat := Chat{
ID: uuid.New().String(),
Title: title,
MemberIDs: membersCopy,
CreatedAt: time.Now().UTC().Truncate(time.Microsecond),
}
tx, err := s.db.Begin(ctx)
if err != nil {
return Chat{}, err
}
defer tx.Rollback(ctx)
for _, id := range membersCopy {
var exists bool
if err = tx.QueryRow(ctx, "SELECT EXISTS(SELECT 1 FROM users WHERE id=$1)", id).Scan(&exists); err != nil {
return Chat{}, err
}
if !exists {
return Chat{}, ErrUnknownMember
}
}
if _, err = tx.Exec(ctx, "INSERT INTO chats(id,title,created_at) VALUES($1,$2,$3)", chat.ID, title, chat.CreatedAt); err != nil {
return Chat{}, err
}
for _, id := range membersCopy {
if _, err = tx.Exec(ctx, "INSERT INTO chat_members(chat_id,user_id) VALUES($1,$2)", chat.ID, id); err != nil {
return Chat{}, err
}
}
if err = tx.Commit(ctx); err != nil {
return Chat{}, err
}
return chat, nil
}
func (s *ChatStore) GetByID(ctx context.Context, id string) (Chat, error) {
ctx, cancel := boundedQuery(ctx)
defer cancel()
var chat Chat
err := s.db.QueryRow(ctx, "SELECT id,title,created_at FROM chats WHERE id=$1", id).Scan(&chat.ID, &chat.Title, &chat.CreatedAt)
if err != nil {
return Chat{}, err
}
rows, err := s.db.Query(ctx, "SELECT user_id FROM chat_members WHERE chat_id=$1 ORDER BY user_id", id)
if err != nil {
return Chat{}, err
}
defer rows.Close()
chat.MemberIDs = []string{}
for rows.Next() {
var member string
if err = rows.Scan(&member); err != nil {
return Chat{}, err
}
chat.MemberIDs = append(chat.MemberIDs, member)
}
return chat, rows.Err()
}
func (s *ChatStore) ListPageForUser(ctx context.Context, userID string, cursor *ChatCursor, limit int) ([]Chat, bool, error) {
ctx, cancel := boundedQuery(ctx)
defer cancel()
afterTime := time.Time{}
afterID := "00000000-0000-0000-0000-000000000000"
if cursor != nil {
afterTime = cursor.CreatedAt
afterID = cursor.ChatID
}
rows, err := s.db.Query(ctx, `SELECT c.id,c.title,c.created_at FROM chats c JOIN chat_members m ON m.chat_id=c.id WHERE m.user_id=$1 AND (c.created_at,c.id)>($2,$3::uuid) ORDER BY c.created_at,c.id LIMIT $4`, userID, afterTime, afterID, limit+1)
if err != nil {
return nil, false, err
}
defer rows.Close()
chats := []Chat{}
for rows.Next() {
var c Chat
if err = rows.Scan(&c.ID, &c.Title, &c.CreatedAt); err != nil {
return nil, false, err
}
chats = append(chats, c)
}
if err = rows.Err(); err != nil {
return nil, false, err
}
more := len(chats) > limit
if more {
chats = chats[:limit]
}
for i := range chats {
chat, err := s.GetByID(ctx, chats[i].ID)
if err != nil {
return nil, false, err
}
chats[i] = chat
}
return chats, more, nil
}
@@ -0,0 +1,20 @@
package repositories
import (
"errors"
"os"
"strconv"
"time"
)
func UploadCleanupGrace() (time.Duration, error) {
value := os.Getenv("UPLOAD_CLEANUP_GRACE_SECONDS")
if value == "" {
value = "600"
}
seconds, err := strconv.Atoi(value)
if err != nil || seconds < 120 || seconds > 86400 {
return 0, errors.New("UPLOAD_CLEANUP_GRACE_SECONDS must be between 120 and 86400")
}
return time.Duration(seconds) * time.Second, nil
}
+47
View File
@@ -0,0 +1,47 @@
package repositories
import (
"errors"
"os"
"strconv"
"strings"
"time"
"github.com/redis/go-redis/v9"
)
func RedisClientFromEnvironment() (*redis.Client, error) {
if path := os.Getenv("CACHE_URL_FILE"); path != "" {
data, err := os.ReadFile(path)
if err != nil {
return nil, err
}
options, err := redis.ParseURL(strings.TrimSpace(string(data)))
if err != nil {
return nil, errors.New("invalid Redis URL in CACHE_URL_FILE")
}
options.PoolSize = 10
options.MaxRetries = -1
options.DialTimeout = time.Second
options.ReadTimeout = time.Second
options.WriteTimeout = time.Second
return redis.NewClient(options), nil
}
address := os.Getenv("REDIS_ADDR")
if address == "" {
address = "redis:6379"
}
return NewRedisClient(address), nil
}
func IdempotencyTTL() (time.Duration, error) {
value := os.Getenv("IDEMPOTENCY_TTL_SECONDS")
if value == "" {
value = "86400"
}
seconds, err := strconv.ParseInt(value, 10, 64)
if err != nil || seconds < 1 || seconds > 31536000 {
return 0, errors.New("IDEMPOTENCY_TTL_SECONDS must be between 1 and 31536000")
}
return time.Duration(seconds) * time.Second, nil
}
+15
View File
@@ -0,0 +1,15 @@
package repositories
import "time"
type ChatCursor struct {
UserID string `json:"user_id"`
CreatedAt time.Time `json:"created_at"`
ChatID string `json:"chat_id"`
}
type MessageCursor struct {
ChatID string `json:"chat_id"`
CreatedAt time.Time `json:"created_at"`
MessageID string `json:"message_id"`
}
+100
View File
@@ -0,0 +1,100 @@
package repositories
import (
"context"
"database/sql"
"embed"
"errors"
"os"
"strconv"
"strings"
"time"
"messenger/src/backend/telemetry"
"github.com/golang-migrate/migrate/v4"
pgxmigrate "github.com/golang-migrate/migrate/v4/database/pgx/v5"
"github.com/golang-migrate/migrate/v4/source/iofs"
"github.com/jackc/pgx/v5/pgxpool"
_ "github.com/jackc/pgx/v5/stdlib"
)
//go:embed migrations/*.sql
var migrationFiles embed.FS
func DatabaseURL() (string, error) {
if path := os.Getenv("DATABASE_URL_FILE"); path != "" {
data, err := os.ReadFile(path)
if err != nil {
return "", err
}
return strings.TrimSpace(string(data)), nil
}
if url := os.Getenv("DATABASE_URL"); url != "" {
return url, nil
}
return "", errors.New("DATABASE_URL_FILE or DATABASE_URL is required")
}
func OpenDatabase(ctx context.Context) (*pgxpool.Pool, error) {
url, err := DatabaseURL()
if err != nil {
return nil, err
}
cfg, err := pgxpool.ParseConfig(url)
if err != nil {
return nil, err
}
cfg.ConnConfig.Tracer = telemetry.PostgresTracer{}
cfg.MaxConns = 10
if value := os.Getenv("DATABASE_MAX_CONNS"); value != "" {
maxConns, parseErr := strconv.Atoi(value)
if parseErr != nil || maxConns < 1 || maxConns > 100 {
return nil, errors.New("DATABASE_MAX_CONNS must be between 1 and 100")
}
cfg.MaxConns = int32(maxConns)
}
cfg.ConnConfig.ConnectTimeout = 5 * time.Second
db, err := pgxpool.NewWithConfig(ctx, cfg)
if err != nil {
return nil, err
}
if err = db.Ping(ctx); err != nil {
db.Close()
return nil, err
}
return db, nil
}
// RunMigrations uses versioned up/down SQL embedded in the API binary.
// "down" rolls back exactly one version and is an explicit operator command.
func RunMigrations(url, direction string) error {
source, err := iofs.New(migrationFiles, "migrations")
if err != nil {
return err
}
sqlDB, err := sql.Open("pgx", url)
if err != nil {
return err
}
driver, err := pgxmigrate.WithInstance(sqlDB, &pgxmigrate.Config{})
if err != nil {
sqlDB.Close()
return err
}
m, err := migrate.NewWithInstance("iofs", source, "pgx5", driver)
if err != nil {
sqlDB.Close()
return err
}
defer m.Close()
if direction == "down" {
err = m.Steps(-1)
} else {
err = m.Up()
}
if errors.Is(err, migrate.ErrNoChange) {
return nil
}
return err
}
+106
View File
@@ -0,0 +1,106 @@
package repositories
import (
"context"
"errors"
"fmt"
"time"
"github.com/google/uuid"
"github.com/jackc/pgx/v5"
)
func (s *AttachmentStore) CleanupDirectUploads(ctx context.Context, dryRun bool, remove func(context.Context, string) error) (CleanupResult, error) {
result := CleanupResult{DryRun: dryRun}
rows, err := s.db.Query(ctx, `SELECT id::text FROM direct_uploads WHERE cleanup_after<=clock_timestamp() ORDER BY cleanup_after,id LIMIT 100`)
if err != nil {
return result, err
}
ids := []string{}
for rows.Next() {
var id string
if err = rows.Scan(&id); err != nil {
rows.Close()
return result, err
}
ids = append(ids, id)
}
err = rows.Err()
rows.Close()
if err != nil {
return result, err
}
result.Candidates = len(ids)
for _, id := range ids {
outcome, err := s.cleanupDirectUpload(ctx, id, dryRun, remove)
if err != nil {
result.Failed++
continue
}
switch outcome {
case "deleted":
result.Deleted++
case "eligible":
result.Eligible++
default:
result.Skipped++
}
}
if result.Failed > 0 {
return result, fmt.Errorf("%d direct upload cleanup operations failed", result.Failed)
}
return result, nil
}
func (s *AttachmentStore) cleanupDirectUpload(ctx context.Context, id string, dryRun bool, remove func(context.Context, string) error) (string, error) {
ctx, cancel := context.WithTimeout(ctx, 10*time.Second)
defer cancel()
tx, err := s.db.Begin(ctx)
if err != nil {
return "", err
}
defer tx.Rollback(context.Background())
var key, state string
var barrier, complete, purge, now time.Time
err = tx.QueryRow(ctx, `SELECT staging_key,state,complete_before+interval '600 seconds',complete_before,purge_after,clock_timestamp() FROM direct_uploads WHERE id=$1 AND cleanup_after<=clock_timestamp() FOR UPDATE SKIP LOCKED`, id).Scan(&key, &state, &barrier, &complete, &purge, &now)
if errors.Is(err, pgx.ErrNoRows) {
return "skipped", nil
}
if err != nil {
return "", err
}
parsed, err := uuid.Parse(id)
if err != nil || key != "staging/direct/"+parsed.String() {
return "", errors.New("invalid direct staging key")
}
if state == "pending" && now.Before(barrier) {
return "skipped", nil
}
if dryRun {
return "eligible", nil
}
if err = remove(ctx, key); err != nil {
return "", err
}
if now.After(purge) && !now.Before(barrier) {
_, err = tx.Exec(ctx, `DELETE FROM direct_uploads WHERE id=$1`, id)
} else {
if state == "pending" && !now.Before(complete) {
state = "expired"
}
// An early cleanup is not final: a still valid POST can recreate staging.
// Keep a second cleanup after the safety barrier, then retain the receipt 24h.
next := barrier
if !now.Before(barrier) {
next = purge
}
_, err = tx.Exec(ctx, `UPDATE direct_uploads SET state=$2,cleanup_after=$3 WHERE id=$1`, id, state, next)
}
if err != nil {
return "", err
}
if err = tx.Commit(ctx); err != nil {
return "", err
}
return "deleted", nil
}
+150
View File
@@ -0,0 +1,150 @@
package repositories
import (
"context"
"errors"
"time"
"github.com/jackc/pgx/v5"
)
// UploadError is also persisted for terminal validation failures, so retries agree.
type UploadError struct {
Status int
Code, Message string
}
func (e UploadError) Error() string { return e.Message }
func DirectError(code string) UploadError {
switch code {
case "upload_not_uploaded":
return UploadError{409, code, "file has not been uploaded"}
case "upload_expired":
return UploadError{409, code, "upload confirmation period has expired"}
case "upload_size_mismatch":
return UploadError{400, code, "uploaded size differs from declared size"}
case "upload_checksum_mismatch":
return UploadError{400, code, "uploaded SHA-256 differs from declared checksum"}
case "payload_too_large":
return UploadError{413, code, "image exceeds upload limits"}
default:
return UploadError{415, "unsupported_media_type", "a valid PNG or JPEG matching Content-Type is required"}
}
}
type DirectUpload struct {
ID string `json:"upload_id"`
AttachmentID string `json:"-"`
OwnerID string `json:"-"`
ChatID string `json:"-"`
Filename string `json:"-"`
ContentType string `json:"-"`
SizeBytes int64 `json:"-"`
SHA256 string `json:"-"`
StagingKey string `json:"-"`
FinalKey string `json:"-"`
State string `json:"-"`
ErrorCode *string `json:"-"`
ExpiresAt time.Time `json:"expires_at"`
CompleteBefore time.Time `json:"complete_before"`
}
const directColumns = `d.id,d.attachment_id,d.owner_id,d.chat_id,d.filename,d.content_type,d.size_bytes,d.sha256,d.staging_key,d.final_key,d.state,d.error_code,d.expires_at,d.complete_before`
func scanDirect(row pgx.Row) (DirectUpload, error) {
var u DirectUpload
err := row.Scan(&u.ID, &u.AttachmentID, &u.OwnerID, &u.ChatID, &u.Filename, &u.ContentType, &u.SizeBytes, &u.SHA256, &u.StagingKey, &u.FinalKey, &u.State, &u.ErrorCode, &u.ExpiresAt, &u.CompleteBefore)
return u, err
}
func (s *AttachmentStore) CreateDirect(ctx context.Context, u DirectUpload) (DirectUpload, error) {
ctx, cancel := boundedQuery(ctx)
defer cancel()
return scanDirect(s.db.QueryRow(ctx, `INSERT INTO direct_uploads AS d(id,attachment_id,owner_id,chat_id,filename,content_type,size_bytes,sha256,staging_key,final_key) VALUES($1,$2,$3,$4,$5,$6,$7,$8,$9,$10) RETURNING `+directColumns, u.ID, u.AttachmentID, u.OwnerID, u.ChatID, u.Filename, u.ContentType, u.SizeBytes, u.SHA256, u.StagingKey, u.FinalKey))
}
func (s *AttachmentStore) GetDirect(ctx context.Context, id, chat, user string) (DirectUpload, error) {
ctx, cancel := boundedQuery(ctx)
defer cancel()
return scanDirect(s.db.QueryRow(ctx, `SELECT `+directColumns+` FROM direct_uploads d JOIN chat_members cm ON cm.chat_id=d.chat_id AND cm.user_id=$3 WHERE d.id=$1 AND d.chat_id=$2 AND d.owner_id=$3`, id, chat, user))
}
// The session lock serializes complete across replicas. A separately committed
// reservation survives rollback/crash, while its row lock protects the final key.
func (s *AttachmentStore) CompleteDirect(ctx context.Context, id, chat, user string, validate func(context.Context, DirectUpload) error) (Attachment, error) {
var a Attachment
tx, err := s.db.Begin(ctx)
if err != nil {
return a, err
}
defer tx.Rollback(context.Background())
u, err := scanDirect(tx.QueryRow(ctx, `SELECT `+directColumns+` FROM direct_uploads d JOIN chat_members cm ON cm.chat_id=d.chat_id AND cm.user_id=$3 WHERE d.id=$1 AND d.chat_id=$2 AND d.owner_id=$3 FOR UPDATE OF d`, id, chat, user))
if err != nil {
return a, err
}
if u.StagingKey != "staging/direct/"+u.ID || u.FinalKey != "originals/"+u.AttachmentID {
return a, errors.New("invalid direct upload object keys")
}
if u.State == "ready" {
return s.GetForMember(ctx, u.AttachmentID, user)
}
if u.State == "rejected" && u.ErrorCode != nil {
return a, DirectError(*u.ErrorCode)
}
var expired bool
if err = tx.QueryRow(ctx, `SELECT clock_timestamp()>=$1`, u.CompleteBefore).Scan(&expired); err != nil {
return a, err
}
if expired || u.State == "expired" {
return a, DirectError("upload_expired")
}
grace, err := UploadCleanupGrace()
if err != nil {
return a, err
}
// ON CONFLICT handles retry after an interrupted COPY without losing tracking.
_, err = s.db.Exec(ctx, `INSERT INTO upload_attempts(id,object_key,cleanup_after) VALUES($1,$2,clock_timestamp()+$3*interval '1 second') ON CONFLICT(id) DO UPDATE SET cleanup_after=GREATEST(upload_attempts.cleanup_after,EXCLUDED.cleanup_after) WHERE upload_attempts.object_key=EXCLUDED.object_key`, u.AttachmentID, u.FinalKey, grace.Seconds())
if err != nil {
return a, err
}
var key string
if err = tx.QueryRow(ctx, `SELECT object_key FROM upload_attempts WHERE id=$1 AND cleanup_after>clock_timestamp() FOR UPDATE`, u.AttachmentID).Scan(&key); err != nil {
return a, err
}
if key != u.FinalKey {
return a, errors.New("direct upload reservation mismatch")
}
if err = validate(ctx, u); err != nil {
var invalid UploadError
if errors.As(err, &invalid) && (invalid.Status == 400 || invalid.Status == 413 || invalid.Status == 415) {
_, saveErr := tx.Exec(ctx, `UPDATE direct_uploads SET state='rejected',error_code=$2,cleanup_after=clock_timestamp(),purge_after=clock_timestamp()+interval '24 hours' WHERE id=$1`, id, invalid.Code)
if saveErr != nil {
return a, saveErr
}
if saveErr = tx.Commit(ctx); saveErr != nil {
return a, saveErr
}
}
return a, err
}
now := time.Now().UTC().Truncate(time.Microsecond)
a = Attachment{ID: u.AttachmentID, ChatID: u.ChatID, OwnerID: u.OwnerID, Filename: u.Filename, ContentType: u.ContentType, SizeBytes: u.SizeBytes, SHA256: u.SHA256, ObjectKey: u.FinalKey, Status: "queued", Variants: []any{}, CreatedAt: now, UpdatedAt: now}
_, err = tx.Exec(ctx, `INSERT INTO attachments(id,chat_id,owner_id,filename,object_key,content_type,size_bytes,sha256,created_at,updated_at) VALUES($1,$2,$3,$4,$5,$6,$7,$8,$9,$10)`, a.ID, a.ChatID, a.OwnerID, a.Filename, a.ObjectKey, a.ContentType, a.SizeBytes, a.SHA256, a.CreatedAt, a.UpdatedAt)
if err != nil {
return Attachment{}, err
}
if err = enqueueJob(ctx, tx, a, a.ObjectKey); err != nil {
return Attachment{}, err
}
if _, err = tx.Exec(ctx, `UPDATE direct_uploads SET state='ready',cleanup_after=clock_timestamp(),purge_after=clock_timestamp()+interval '24 hours' WHERE id=$1`, id); err != nil {
return Attachment{}, err
}
if _, err = tx.Exec(ctx, `DELETE FROM upload_attempts WHERE id=$1`, a.ID); err != nil {
return Attachment{}, err
}
if err = tx.Commit(ctx); err != nil {
return Attachment{}, err
}
return a, nil
}
+21
View File
@@ -0,0 +1,21 @@
package repositories
import (
"errors"
"github.com/jackc/pgx/v5"
"github.com/jackc/pgx/v5/pgconn"
)
func NotFound(err error) bool {
return errors.Is(err, pgx.ErrNoRows)
}
func CheckViolation(err error) bool {
var pgerr *pgconn.PgError
return errors.As(err, &pgerr) && pgerr.Code == "23514"
}
func UniqueViolation(err error) bool {
var pgerr *pgconn.PgError
return errors.As(err, &pgerr) && pgerr.Code == "23505"
}
+21
View File
@@ -0,0 +1,21 @@
package repositories
import (
"context"
"github.com/jackc/pgx/v5/pgxpool"
)
type HealthStore struct {
db *pgxpool.Pool
}
func NewHealthStore(db *pgxpool.Pool) *HealthStore {
return &HealthStore{db: db}
}
func (s *HealthStore) Ping(ctx context.Context) error {
ctx, cancel := boundedQuery(ctx)
defer cancel()
return s.db.Ping(ctx)
}
+95
View File
@@ -0,0 +1,95 @@
package repositories
import (
"context"
"errors"
"reflect"
"time"
"github.com/google/uuid"
"github.com/jackc/pgx/v5"
)
var ErrAttachmentNotFound = errors.New("attachment not found")
var ErrAttachmentFailed = errors.New("attachment failed")
var ErrIdempotencyConflict = errors.New("idempotency key belongs to a different message")
// CreateIdempotent commits the message and its replay record together.
func (s *MessageStore) CreateIdempotent(ctx context.Context, chatID, senderID, text, key string, attachments ...[]string) (Message, error) {
ids := []string{}
if len(attachments) > 0 && attachments[0] != nil {
ids = attachments[0]
}
ctx, cancel := boundedQuery(ctx)
defer cancel()
tx, err := s.db.Begin(ctx)
if err != nil {
return Message{}, err
}
defer tx.Rollback(context.Background())
// The transaction-scoped lock is shared by all API replicas. Hash collisions
// only serialize unrelated requests; the full scope is checked by the query.
scope := senderID + ":POST:/api/v1/chats/" + chatID + "/messages:" + key
if _, err = tx.Exec(ctx, "SELECT pg_advisory_xact_lock(hashtextextended($1, 0))", scope); err != nil {
return Message{}, err
}
// A shared row lock prevents failure publication racing with a new link.
hasFailed := false
for _, id := range ids {
var status string
err = tx.QueryRow(ctx, "SELECT status FROM attachments WHERE id=$1 AND chat_id=$2 FOR SHARE", id, chatID).Scan(&status)
if errors.Is(err, pgx.ErrNoRows) {
return Message{}, ErrAttachmentNotFound
}
if err != nil {
return Message{}, err
}
hasFailed = hasFailed || status == "failed"
}
var message Message
err = tx.QueryRow(ctx, `
SELECT m.id, m.chat_id, m.sender_id, m.text, m.created_at, ARRAY(SELECT attachment_id::text FROM message_attachments ma WHERE ma.message_id=m.id ORDER BY position)
FROM message_idempotency i JOIN messages m ON m.id = i.message_id
WHERE i.sender_id=$1 AND i.chat_id=$2 AND i.key=$3
AND i.expires_at > clock_timestamp()`, senderID, chatID, key).
Scan(&message.ID, &message.ChatID, &message.SenderID, &message.Text, &message.CreatedAt, &message.AttachmentIDs)
if err == nil {
message.CreatedAt = message.CreatedAt.UTC()
if message.Text != text || !reflect.DeepEqual(message.AttachmentIDs, ids) {
return Message{}, ErrIdempotencyConflict
}
return message, tx.Commit(ctx)
}
if !errors.Is(err, pgx.ErrNoRows) {
return Message{}, err
}
if hasFailed {
return Message{}, ErrAttachmentFailed
}
// Expiry permits key reuse; it never deletes the original message.
if _, err = tx.Exec(ctx, "DELETE FROM message_idempotency WHERE sender_id=$1 AND chat_id=$2 AND key=$3", senderID, chatID, key); err != nil {
return Message{}, err
}
message = Message{
AttachmentIDs: ids, ID: uuid.NewString(), ChatID: chatID, SenderID: senderID, Text: text,
CreatedAt: time.Now().UTC().Truncate(time.Microsecond),
}
if _, err = tx.Exec(ctx, "INSERT INTO messages(id,chat_id,sender_id,text,created_at) VALUES($1,$2,$3,$4,$5)", message.ID, chatID, senderID, text, message.CreatedAt); err != nil {
return Message{}, err
}
for position, id := range ids {
if _, err = tx.Exec(ctx, "INSERT INTO message_attachments(message_id,attachment_id,position) VALUES($1,$2,$3)", message.ID, id, position); err != nil {
return Message{}, err
}
}
if _, err = tx.Exec(ctx, `INSERT INTO message_idempotency(sender_id,chat_id,key,message_id,expires_at)
VALUES($1,$2,$3,$4,clock_timestamp() + $5 * interval '1 second')`, senderID, chatID, key, message.ID, s.idempotencyTTL.Seconds()); err != nil {
return Message{}, err
}
return message, tx.Commit(ctx)
}
+142
View File
@@ -0,0 +1,142 @@
package repositories
import (
"context"
"errors"
"github.com/jackc/pgx/v5"
"strings"
"time"
"unicode/utf8"
)
type FailedJob struct {
ID string `json:"job_id"`
AttachmentID string `json:"attachment_id"`
Code string `json:"error_code"`
Attempts int `json:"attempts"`
FailedAt time.Time `json:"failed_at"`
}
func (s *AttachmentStore) FailedJobs(ctx context.Context) ([]FailedJob, error) {
rows, err := s.db.Query(ctx, `SELECT j.id,j.attachment_id,f.error_code,f.attempts,f.failed_at FROM attachment_job_failures f JOIN attachment_jobs j ON j.id=f.job_id JOIN attachments a ON a.id=j.attachment_id WHERE a.status='failed' AND j.pipeline_version=a.pipeline_version AND j.superseded_at IS NULL ORDER BY f.failed_at DESC LIMIT 100`)
if err != nil {
return nil, err
}
defer rows.Close()
result := []FailedJob{}
for rows.Next() {
var j FailedJob
if err = rows.Scan(&j.ID, &j.AttachmentID, &j.Code, &j.Attempts, &j.FailedAt); err != nil {
return nil, err
}
result = append(result, j)
}
return result, rows.Err()
}
func (s *AttachmentStore) ReplayJob(ctx context.Context, id, reason string, exists func(context.Context, string) error) (bool, error) {
reason = strings.TrimSpace(reason)
if reason == "" || utf8.RuneCountInString(reason) > 200 {
return false, errors.New("replay reason must contain 1-200 characters")
}
tx, err := s.db.Begin(ctx)
if err != nil {
return false, err
}
defer tx.Rollback(context.Background())
var status, key, attachment string
var deleted, obsolete bool
err = tx.QueryRow(ctx, `SELECT a.status,j.staging_key,a.id,j.staging_deleted_at IS NOT NULL,j.pipeline_version<>a.pipeline_version OR j.superseded_at IS NOT NULL FROM attachment_jobs j JOIN attachments a ON a.id=j.attachment_id WHERE j.id=$1 FOR UPDATE OF j,a`, id).Scan(&status, &key, &attachment, &deleted, &obsolete)
if err != nil {
return false, err
}
if obsolete || status != "failed" {
return false, nil
}
if deleted {
return false, errors.New("staging has expired; replay unavailable")
}
if key != "staging/jobs/"+attachment && key != "originals/"+attachment {
return false, errors.New("invalid staging key")
}
if err = exists(ctx, key); err != nil {
return false, errors.New("staging unavailable; replay not scheduled")
}
if _, err = tx.Exec(ctx, `INSERT INTO attachment_job_replays(job_id,reason) VALUES($1,$2)`, id, reason); err != nil {
return false, err
}
if _, err = tx.Exec(ctx, `UPDATE attachment_jobs SET attempts=0,generation=generation+1,next_attempt_at=now(),lease_until=NULL,lease_token=NULL,published_at=NULL WHERE id=$1`, id); err != nil {
return false, err
}
if _, err = tx.Exec(ctx, `UPDATE attachments SET status='queued',error_code=NULL,updated_at=now() WHERE id=$1`, attachment); err != nil {
return false, err
}
return true, tx.Commit(ctx)
}
// Replay and cleanup lock the same rows. A ready source is removed only after
// final publication, failed sources expire after retention; queued work is safe.
func (s *AttachmentStore) CleanupJobStaging(ctx context.Context, id string, retention time.Duration, dryRun bool, remove func(context.Context, string) error) (bool, error) {
tx, err := s.db.Begin(ctx)
if err != nil {
return false, err
}
defer tx.Rollback(context.Background())
var key, attachment, status string
var eligible bool
err = tx.QueryRow(ctx, `SELECT j.staging_key,a.id,a.status,j.staging_deleted_at IS NULL AND (a.status='ready' AND jsonb_array_length(a.variants)>=3 OR a.status='failed' AND j.pipeline_version=a.pipeline_version AND COALESCE(f.failed_at,a.updated_at)<now()-$2*interval '1 second') FROM attachment_jobs j JOIN attachments a ON a.id=j.attachment_id LEFT JOIN attachment_job_failures f ON f.job_id=j.id WHERE j.id=$1 FOR UPDATE OF j,a SKIP LOCKED`, id, retention.Seconds()).Scan(&key, &attachment, &status, &eligible)
if errors.Is(err, pgx.ErrNoRows) {
return false, nil
}
if err != nil {
return false, err
}
if !eligible || key == "originals/"+attachment {
return false, nil
}
if key != "staging/jobs/"+attachment {
return false, errors.New("invalid staging key")
}
if dryRun {
return true, nil
}
if err = remove(ctx, key); err != nil {
return false, err
}
if _, err = tx.Exec(ctx, `UPDATE attachment_jobs SET staging_deleted_at=now() WHERE id=$1`, id); err != nil {
return false, err
}
return true, tx.Commit(ctx)
}
func (s *AttachmentStore) CleanupJobs(ctx context.Context, retention time.Duration, dryRun bool, remove func(context.Context, string) error) (int, error) {
rows, err := s.db.Query(ctx, `SELECT j.id FROM attachment_jobs j JOIN attachments a ON a.id=j.attachment_id LEFT JOIN attachment_job_failures f ON f.job_id=j.id WHERE j.staging_deleted_at IS NULL AND j.staging_key='staging/jobs/'||j.attachment_id::text AND (a.status='ready' AND jsonb_array_length(a.variants)>=3 OR a.status='failed' AND j.pipeline_version=a.pipeline_version AND COALESCE(f.failed_at,a.updated_at)<now()-$1*interval '1 second') ORDER BY j.created_at LIMIT 100`, retention.Seconds())
if err != nil {
return 0, err
}
ids := []string{}
for rows.Next() {
var id string
if err = rows.Scan(&id); err != nil {
rows.Close()
return 0, err
}
ids = append(ids, id)
}
err = rows.Err()
rows.Close()
if err != nil {
return 0, err
}
count := 0
for _, id := range ids {
done, e := s.CleanupJobStaging(ctx, id, retention, dryRun, remove)
if e != nil {
return count, e
}
if done {
count++
}
}
return count, nil
}
+232
View File
@@ -0,0 +1,232 @@
package repositories
import (
"context"
"crypto/rand"
"encoding/json"
"errors"
"math/big"
"time"
"messenger/src/backend/telemetry"
"github.com/google/uuid"
"github.com/jackc/pgx/v5"
"go.opentelemetry.io/otel/trace"
)
type Variant struct {
Name string `json:"name"`
ContentType string `json:"content_type"`
SizeBytes int64 `json:"size_bytes"`
Width int `json:"width"`
Height int `json:"height"`
SHA256 string `json:"sha256"`
}
type Job struct {
SchemaVersion int `json:"schema_version"`
ID string `json:"job_id"`
Type string `json:"type"`
AttachmentID string `json:"attachment_id"`
PipelineVersion int `json:"pipeline_version"`
Attempt int `json:"attempt"`
CreatedAt time.Time `json:"created_at"`
Traceparent string `json:"-"`
Tracestate string `json:"-"`
}
// QueuedJobs is also the recovery source when publish/confirm is uncertain.
// Re-publication is intentional: a durable intent is never removed by publish.
func (s *AttachmentStore) QueuedJobs(ctx context.Context) ([]Job, error) {
rows, err := s.db.Query(ctx, `SELECT j.id,j.attachment_id,j.pipeline_version,j.attempts+1,j.created_at,j.traceparent,j.tracestate FROM attachment_jobs j JOIN attachments a ON a.id=j.attachment_id WHERE j.superseded_at IS NULL AND j.pipeline_version=a.pipeline_version AND a.status IN ('queued','processing') AND j.next_attempt_at<=now() AND (j.lease_until IS NULL OR j.lease_until<now()) AND (j.published_at IS NULL OR j.published_at<now()-interval '30 seconds') ORDER BY j.created_at LIMIT 100`)
if err != nil {
return nil, err
}
defer rows.Close()
jobs := []Job{}
for rows.Next() {
j := Job{SchemaVersion: 1, Type: "attachment.process.v1"}
if err = rows.Scan(&j.ID, &j.AttachmentID, &j.PipelineVersion, &j.Attempt, &j.CreatedAt, &j.Traceparent, &j.Tracestate); err != nil {
return nil, err
}
j.CreatedAt = j.CreatedAt.UTC()
jobs = append(jobs, j)
}
return jobs, rows.Err()
}
func (s *AttachmentStore) MarkPublished(ctx context.Context, id string) error {
_, err := s.db.Exec(ctx, `UPDATE attachment_jobs SET published_at=now() WHERE id=$1`, id)
return err
}
// StartAttempt commits the lease and attempt before decoding. A token fences a
// consumer that was paused between claim and processing past its lease deadline.
func (s *AttachmentStore) StartAttempt(ctx context.Context, j Job, timeout time.Duration, maxAttempts int) (string, error) {
tx, err := s.db.Begin(ctx)
if err != nil {
return "", err
}
defer tx.Rollback(context.Background())
var status string
var attempts, generation int
var available bool
err = tx.QueryRow(ctx, `SELECT a.status,j.attempts,j.generation,j.next_attempt_at<=now() AND (j.lease_until IS NULL OR j.lease_until<now()) FROM attachment_jobs j JOIN attachments a ON a.id=j.attachment_id WHERE j.id=$1 AND j.attachment_id=$2 AND j.pipeline_version=$3 AND j.superseded_at IS NULL AND a.pipeline_version=j.pipeline_version FOR UPDATE OF j,a SKIP LOCKED`, j.ID, j.AttachmentID, j.PipelineVersion).Scan(&status, &attempts, &generation, &available)
if errors.Is(err, pgx.ErrNoRows) {
return "", nil
}
if err != nil {
return "", err
}
if status == "ready" || status == "failed" || !available {
return "", nil
}
// A crashed attempt remains visible in the audit trail.
if _, err = tx.Exec(ctx, `UPDATE attachment_job_attempts SET finished_at=clock_timestamp(),error_code='processing_timeout' WHERE job_id=$1 AND finished_at IS NULL`, j.ID); err != nil {
return "", err
}
if attempts >= maxAttempts {
if err = failJob(ctx, tx, j, "retry_exhausted", attempts); err != nil {
return "", err
}
return "", tx.Commit(ctx)
}
token := uuid.NewString()
_, err = tx.Exec(ctx, `UPDATE attachment_jobs SET attempts=attempts+1,lease_token=$2,lease_until=now()+$3*interval '1 second' WHERE id=$1`, j.ID, token, timeout.Seconds()+5)
if err != nil {
return "", err
}
if _, err = tx.Exec(ctx, `UPDATE attachments SET status='processing',updated_at=now() WHERE id=$1`, j.AttachmentID); err != nil {
return "", err
}
if _, err = tx.Exec(ctx, `INSERT INTO attachment_job_attempts(job_id,generation,attempt) VALUES($1,$2,$3)`, j.ID, generation, attempts+1); err != nil {
return "", err
}
return token, tx.Commit(ctx)
}
var ErrStaleJob = errors.New("stale job lease or version")
var ErrProcessingTimeout = errors.New("processing timeout")
// ProcessJob holds the row lock across deterministic object writes. The lock and
// token check prevent overlapping/stale attempts from publishing metadata.
func (s *AttachmentStore) ProcessJob(ctx context.Context, j Job, token string, maxAttempts int, process func(Attachment, string) ([]Variant, string, error)) (result error) {
ctx, finish := telemetry.Operation(ctx, "postgresql finalize attachment", trace.SpanKindClient)
defer func() { finish(result) }()
tx, err := s.db.Begin(ctx)
if err != nil {
return err
}
defer tx.Rollback(context.Background())
var staging string
var attempts, generation int
var a Attachment
err = tx.QueryRow(ctx, `SELECT a.id,a.object_key,a.content_type,a.size_bytes,a.sha256,a.status,j.staging_key,j.attempts,j.generation FROM attachment_jobs j JOIN attachments a ON a.id=j.attachment_id WHERE j.id=$1 AND j.attachment_id=$2 AND j.pipeline_version=$3 AND j.lease_token=$4 AND j.lease_until>now() AND j.superseded_at IS NULL AND a.pipeline_version=j.pipeline_version FOR UPDATE OF j,a`, j.ID, j.AttachmentID, j.PipelineVersion, token).Scan(&a.ID, &a.ObjectKey, &a.ContentType, &a.SizeBytes, &a.SHA256, &a.Status, &staging, &attempts, &generation)
if errors.Is(err, pgx.ErrNoRows) {
return ErrStaleJob
}
if err != nil {
return err
}
if a.Status != "processing" {
return errors.New("job is not processing")
}
a.PipelineVersion = j.PipelineVersion
variants, code, workErr := process(a, staging)
var next *time.Time
var auditCode *string
if workErr == nil {
data, e := json.Marshal(variants)
if e != nil {
return e
}
_, err = tx.Exec(ctx, `UPDATE attachments SET status='ready',variants=$2,published_pipeline_version=$3,error_code=NULL,updated_at=now() WHERE id=$1 AND pipeline_version=$3`, a.ID, data, j.PipelineVersion)
if err == nil {
_, err = tx.Exec(ctx, `INSERT INTO attachment_publications(attachment_id,job_id,pipeline_version,variants) VALUES($1,$2,$3,$4)`, a.ID, j.ID, j.PipelineVersion, data)
}
} else {
reason := "temporary_failure"
if errors.Is(workErr, ErrProcessingTimeout) {
reason = "processing_timeout"
}
if code != "" {
reason = code
}
auditCode = &reason
if code != "" || attempts >= maxAttempts {
if code == "" {
code = "retry_exhausted"
}
err = failJob(ctx, tx, j, code, attempts)
} else {
due := time.Now().UTC().Add(RetryDelay(attempts))
next = &due
_, err = tx.Exec(ctx, `UPDATE attachments SET status='queued',error_code=NULL,updated_at=now() WHERE id=$1`, a.ID)
}
}
if err != nil {
return err
}
_, err = tx.Exec(ctx, `UPDATE attachment_job_attempts SET finished_at=clock_timestamp(),error_code=$4,next_attempt_at=$5 WHERE job_id=$1 AND generation=$2 AND attempt=$3`, j.ID, generation, attempts, auditCode, next)
if err != nil {
return err
}
_, err = tx.Exec(ctx, `UPDATE attachment_jobs SET lease_until=NULL,lease_token=NULL,published_at=NULL,next_attempt_at=COALESCE($2,now()) WHERE id=$1`, j.ID, next)
if err != nil {
return err
}
return tx.Commit(ctx)
}
func failJob(ctx context.Context, tx pgx.Tx, j Job, code string, attempts int) error {
if _, err := tx.Exec(ctx, `UPDATE attachments SET status='failed',error_code=$2,updated_at=now() WHERE id=$1`, j.AttachmentID, code); err != nil {
return err
}
if _, err := tx.Exec(ctx, `UPDATE attachment_jobs SET lease_token=NULL,lease_until=NULL WHERE id=$1`, j.ID); err != nil {
return err
}
_, err := tx.Exec(ctx, `INSERT INTO attachment_job_failures(job_id,error_code,attempts) VALUES($1,$2,$3) ON CONFLICT(job_id) DO UPDATE SET error_code=EXCLUDED.error_code,attempts=EXCLUDED.attempts,failed_at=clock_timestamp()`, j.ID, code, attempts)
return err
}
// Equal jitter: exponential ceiling 2,4,8,...,30 seconds; delay in [ceiling/2,ceiling).
func RetryDelay(attempt int) time.Duration {
ceiling := 2 * time.Second
for i := 1; i < attempt && ceiling < 30*time.Second; i++ {
ceiling *= 2
}
if ceiling > 30*time.Second {
ceiling = 30 * time.Second
}
n, err := rand.Int(rand.Reader, big.NewInt(int64(ceiling/2)))
if err != nil {
return ceiling
}
return ceiling/2 + time.Duration(n.Int64())
}
func enqueueJob(ctx context.Context, tx pgx.Tx, a Attachment, staging string) error {
_, err := tx.Exec(ctx, `UPDATE attachments SET status='queued' WHERE id=$1`, a.ID)
if err != nil {
return err
}
headers := telemetry.TraceHeaders(ctx)
_, err = tx.Exec(ctx, `INSERT INTO attachment_jobs(id,attachment_id,staging_key,traceparent,tracestate) VALUES($1,$2,$3,$4,$5)`, uuid.NewString(), a.ID, staging, headers["traceparent"], headers["tracestate"])
return err
}
func (s *AttachmentStore) JobTerminal(ctx context.Context, id string) (bool, error) {
var done bool
err := s.db.QueryRow(ctx, `SELECT j.superseded_at IS NOT NULL OR j.pipeline_version<a.pipeline_version OR a.status IN ('ready','failed') FROM attachment_jobs j JOIN attachments a ON a.id=j.attachment_id WHERE j.id=$1`, id).Scan(&done)
return done, err
}
// PendingJobs counts durable outstanding work, including leases and scheduled retries.
func (s *AttachmentStore) PendingJobs(ctx context.Context) (int64, float64, error) {
var count int64
var age float64
err := s.db.QueryRow(ctx, `SELECT count(*), COALESCE(EXTRACT(EPOCH FROM now()-min(j.created_at)),0)::double precision
FROM attachment_jobs j JOIN attachments a ON a.id=j.attachment_id
WHERE j.superseded_at IS NULL AND j.pipeline_version=a.pipeline_version AND a.status IN ('queued','processing')`).Scan(&count, &age)
return count, age, err
}
+89
View File
@@ -0,0 +1,89 @@
package repositories
import (
"context"
"time"
"github.com/google/uuid"
"github.com/jackc/pgx/v5/pgxpool"
)
type Message struct {
ID string `json:"id"`
ChatID string `json:"chat_id"`
SenderID string `json:"sender_id"`
Text string `json:"text"`
CreatedAt time.Time `json:"created_at"`
AttachmentIDs []string `json:"attachment_ids"`
}
type MessageStore struct {
db *pgxpool.Pool
idempotencyTTL time.Duration
}
func NewMessageStore(db *pgxpool.Pool) *MessageStore {
return NewMessageStoreWithTTL(db, 24*time.Hour)
}
func NewMessageStoreWithTTL(db *pgxpool.Pool, ttl time.Duration) *MessageStore {
return &MessageStore{db: db, idempotencyTTL: ttl}
}
func (s *MessageStore) Create(
ctx context.Context,
chatID string,
senderID string,
text string,
) (Message, error) {
ctx, cancel := boundedQuery(ctx)
defer cancel()
message := Message{
AttachmentIDs: []string{},
ID: uuid.New().String(),
ChatID: chatID,
SenderID: senderID,
Text: text,
CreatedAt: time.Now().UTC().Truncate(time.Microsecond),
}
_, err := s.db.Exec(ctx, "INSERT INTO messages(id,chat_id,sender_id,text,created_at) VALUES($1,$2,$3,$4,$5)", message.ID, message.ChatID, message.SenderID, message.Text, message.CreatedAt)
return message, err
}
func (s *MessageStore) ListPageByChat(
ctx context.Context,
chatID string,
cursor *MessageCursor,
limit int,
) ([]Message, bool, error) {
ctx, cancel := boundedQuery(ctx)
defer cancel()
afterTime := time.Time{}
afterID := "00000000-0000-0000-0000-000000000000"
if cursor != nil {
afterTime = cursor.CreatedAt
afterID = cursor.MessageID
}
rows, err := s.db.Query(ctx, `SELECT id,chat_id,sender_id,text,created_at,ARRAY(SELECT attachment_id::text FROM message_attachments ma WHERE ma.message_id=messages.id ORDER BY position) FROM messages WHERE chat_id=$1 AND (created_at,id)>($2,$3::uuid) ORDER BY created_at,id LIMIT $4`, chatID, afterTime, afterID, limit+1)
if err != nil {
return nil, false, err
}
defer rows.Close()
items := []Message{}
for rows.Next() {
var m Message
if err = rows.Scan(&m.ID, &m.ChatID, &m.SenderID, &m.Text, &m.CreatedAt, &m.AttachmentIDs); err != nil {
return nil, false, err
}
items = append(items, m)
}
if err = rows.Err(); err != nil {
return nil, false, err
}
more := len(items) > limit
if more {
items = items[:limit]
}
return items, more, nil
}
@@ -0,0 +1,5 @@
DROP TABLE IF EXISTS sessions;
DROP TABLE IF EXISTS messages;
DROP TABLE IF EXISTS chat_members;
DROP TABLE IF EXISTS chats;
DROP TABLE IF EXISTS users;
@@ -0,0 +1,32 @@
CREATE TABLE IF NOT EXISTS users (
id uuid PRIMARY KEY,
username varchar(32) NOT NULL UNIQUE CHECK (username ~ '^[a-z0-9_]{3,32}$'),
display_name text NOT NULL CHECK (char_length(display_name) BETWEEN 1 AND 100),
password_hash bytea NOT NULL,
created_at timestamptz NOT NULL
);
CREATE TABLE IF NOT EXISTS chats (
id uuid PRIMARY KEY,
title text NOT NULL CHECK (char_length(title) BETWEEN 1 AND 100),
created_at timestamptz NOT NULL
);
CREATE TABLE IF NOT EXISTS chat_members (
chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE,
user_id uuid NOT NULL REFERENCES users(id),
PRIMARY KEY (chat_id,user_id)
);
CREATE INDEX IF NOT EXISTS chat_members_user_idx ON chat_members(user_id,chat_id);
CREATE TABLE IF NOT EXISTS messages (
id uuid PRIMARY KEY,
chat_id uuid NOT NULL REFERENCES chats(id),
sender_id uuid NOT NULL REFERENCES users(id),
text text NOT NULL CHECK (char_length(text) BETWEEN 1 AND 4000),
created_at timestamptz NOT NULL
);
CREATE INDEX IF NOT EXISTS messages_history_idx ON messages(chat_id,created_at,id);
CREATE TABLE IF NOT EXISTS sessions (
token_hash bytea PRIMARY KEY,
user_id uuid NOT NULL REFERENCES users(id),
expires_at timestamptz NOT NULL
);
CREATE INDEX IF NOT EXISTS sessions_expiry_idx ON sessions(expires_at);
@@ -0,0 +1,6 @@
CREATE TABLE sessions (
token_hash bytea PRIMARY KEY,
user_id uuid NOT NULL REFERENCES users(id),
expires_at timestamptz NOT NULL
);
CREATE INDEX sessions_expiry_idx ON sessions(expires_at);
@@ -0,0 +1 @@
DROP TABLE sessions;
@@ -0,0 +1 @@
DROP TABLE message_idempotency;
@@ -0,0 +1,8 @@
CREATE TABLE message_idempotency (
sender_id uuid NOT NULL REFERENCES users(id),
chat_id uuid NOT NULL REFERENCES chats(id),
key varchar(128) NOT NULL CHECK (key ~ '^[A-Za-z0-9._:-]{1,128}$'),
message_id uuid NOT NULL REFERENCES messages(id),
expires_at timestamptz NOT NULL,
PRIMARY KEY (sender_id, chat_id, key)
);
@@ -0,0 +1,5 @@
-- An operator must first remove attachment-only messages before this rollback.
ALTER TABLE messages DROP CONSTRAINT messages_text_check;
ALTER TABLE messages ADD CONSTRAINT messages_text_check CHECK (char_length(text) BETWEEN 1 AND 4000);
DROP TABLE message_attachments;
DROP TABLE attachments;
@@ -0,0 +1,22 @@
CREATE TABLE attachments (
id uuid PRIMARY KEY,
chat_id uuid NOT NULL REFERENCES chats(id),
owner_id uuid NOT NULL REFERENCES users(id),
filename text NOT NULL,
object_key text NOT NULL UNIQUE,
content_type text NOT NULL CHECK (content_type IN ('image/png','image/jpeg')),
size_bytes bigint NOT NULL CHECK (size_bytes BETWEEN 1 AND 10485760),
sha256 text NOT NULL CHECK (sha256 ~ '^[a-f0-9]{64}$'),
created_at timestamptz NOT NULL,
updated_at timestamptz NOT NULL
);
CREATE INDEX attachments_chat_idx ON attachments(chat_id);
CREATE TABLE message_attachments (
message_id uuid NOT NULL REFERENCES messages(id) ON DELETE CASCADE,
attachment_id uuid NOT NULL REFERENCES attachments(id),
position smallint NOT NULL CHECK (position BETWEEN 0 AND 9),
PRIMARY KEY (message_id, attachment_id),
UNIQUE(message_id, position)
);
ALTER TABLE messages DROP CONSTRAINT messages_text_check;
ALTER TABLE messages ADD CONSTRAINT messages_text_check CHECK (char_length(text) <= 4000);
@@ -0,0 +1 @@
DROP TABLE upload_attempts;
@@ -0,0 +1,7 @@
CREATE TABLE upload_attempts (
id uuid PRIMARY KEY,
object_key text NOT NULL UNIQUE,
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
cleanup_after timestamptz NOT NULL
);
CREATE INDEX upload_attempts_cleanup_idx ON upload_attempts(cleanup_after,id);
@@ -0,0 +1 @@
DROP TABLE direct_uploads;
@@ -0,0 +1,20 @@
CREATE TABLE direct_uploads (
id uuid PRIMARY KEY,
attachment_id uuid NOT NULL UNIQUE,
owner_id uuid NOT NULL REFERENCES users(id),
chat_id uuid NOT NULL REFERENCES chats(id),
filename text NOT NULL,
content_type text NOT NULL CHECK (content_type IN ('image/png','image/jpeg')),
size_bytes bigint NOT NULL CHECK (size_bytes BETWEEN 1 AND 10485760),
sha256 text NOT NULL CHECK (sha256 ~ '^[a-f0-9]{64}$'),
staging_key text NOT NULL UNIQUE,
final_key text NOT NULL UNIQUE,
state text NOT NULL DEFAULT 'pending' CHECK (state IN ('pending','ready','rejected','expired')),
error_code text,
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
expires_at timestamptz NOT NULL DEFAULT clock_timestamp()+interval '120 seconds',
complete_before timestamptz NOT NULL DEFAULT clock_timestamp()+interval '720 seconds',
cleanup_after timestamptz NOT NULL DEFAULT clock_timestamp()+interval '1320 seconds',
purge_after timestamptz NOT NULL DEFAULT clock_timestamp()+interval '24 hours'
);
CREATE INDEX direct_uploads_cleanup_idx ON direct_uploads(cleanup_after,id);

Some files were not shown because too many files have changed in this diff Show More