forked from mtusi2026/mtusi2026lab7
Compare commits
7
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
a779b7a19c | ||
|
|
eb6655c72f | ||
|
|
c69d735692 | ||
|
|
c5e3ac3aff | ||
|
|
8986ba6bef | ||
|
|
48e6a6ac0d | ||
|
|
e3d7df4e72 |
@@ -1,4 +1,5 @@
|
||||
# Local/private state
|
||||
/.memory/
|
||||
.env
|
||||
.env.*
|
||||
!.env.example
|
||||
|
||||
@@ -4,10 +4,26 @@ CA_FILE ?=
|
||||
ACTION_SCRIPT ?=
|
||||
TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",)
|
||||
|
||||
CLEANUP_ARGS ?= --dry-run
|
||||
SERVICES ?=
|
||||
JOB_ID ?=
|
||||
ATTACHMENT_ID ?=
|
||||
REASON ?=
|
||||
export JOB_ID ATTACHMENT_ID REASON
|
||||
|
||||
COMPOSE = docker compose --project-name highload-lab7 --env-file .env -f compose.yml
|
||||
|
||||
.PHONY: app-init app-build app-up app-start app-stop app-down app-restart app-logs app-status
|
||||
.PHONY: app-migrate app-s3-bootstrap worker-start worker-stop queue-status attachments-cleanup
|
||||
.PHONY: jobs-failed jobs-replay jobs-upgrade jobs-cleanup-staging
|
||||
.PHONY: test-backend test-pipeline app-provision
|
||||
|
||||
.PHONY: help check validate test load test-persistence test-failover test-security
|
||||
|
||||
help:
|
||||
@echo "check: offline structure; validate: full OpenAPI (requirements-dev.txt); test: running API; load: synthetic writes"
|
||||
@echo "app-provision: prepare dependencies and OpenBao before first startup"
|
||||
@echo "app-up: build and start the stand; app-status: inspect it; app-down: stop it"
|
||||
|
||||
check:
|
||||
$(PYTHON) scripts/check_contract.py
|
||||
@@ -31,3 +47,76 @@ test-failover:
|
||||
|
||||
test-security:
|
||||
$(PYTHON) tests/security.py --base-url "$(BASE_URL)" $(TLS_ARGS)
|
||||
|
||||
app-init:
|
||||
$(PYTHON) scripts/init_env.py
|
||||
./scripts/bootstrap_tls.sh
|
||||
|
||||
app-provision: app-build
|
||||
$(COMPOSE) up -d openbao
|
||||
./scripts/provision_dependencies.sh
|
||||
|
||||
app-build: app-init
|
||||
$(COMPOSE) build
|
||||
|
||||
app-up: app-init
|
||||
$(COMPOSE) up -d --build --wait --wait-timeout 120
|
||||
./scripts/wait_ingress.sh
|
||||
|
||||
app-start: app-init
|
||||
$(COMPOSE) up -d --no-build --wait --wait-timeout 120
|
||||
./scripts/wait_ingress.sh
|
||||
|
||||
app-stop:
|
||||
$(COMPOSE) stop
|
||||
|
||||
app-down:
|
||||
$(COMPOSE) down
|
||||
|
||||
app-restart: app-init
|
||||
./scripts/restart.sh
|
||||
|
||||
app-logs:
|
||||
$(COMPOSE) logs -f --tail=100 $(SERVICES)
|
||||
|
||||
app-status:
|
||||
$(COMPOSE) ps --all
|
||||
|
||||
app-migrate: app-init
|
||||
$(COMPOSE) run --rm migrate
|
||||
|
||||
app-s3-bootstrap: app-init
|
||||
$(COMPOSE) run --rm s3-bootstrap
|
||||
|
||||
worker-start:
|
||||
$(COMPOSE) start worker
|
||||
|
||||
worker-stop:
|
||||
$(COMPOSE) stop worker
|
||||
|
||||
queue-status:
|
||||
$(COMPOSE) exec -T broker rabbitmqctl -q list_queues name messages_ready messages_unacknowledged --formatter json
|
||||
|
||||
attachments-cleanup:
|
||||
$(COMPOSE) run --rm --no-deps upload-cleanup ./messenger attachments-cleanup $(CLEANUP_ARGS)
|
||||
|
||||
jobs-failed:
|
||||
$(COMPOSE) run --rm -T job-admin ./messenger jobs list-failed
|
||||
|
||||
jobs-replay:
|
||||
@test -n "$$JOB_ID" && test -n "$$REASON" || { echo "Set JOB_ID=... REASON='...'"; exit 2; }
|
||||
$(COMPOSE) run --rm -T job-admin ./messenger jobs replay "$$JOB_ID" "$$REASON"
|
||||
|
||||
jobs-upgrade:
|
||||
@test -n "$$ATTACHMENT_ID" && test -n "$$REASON" || { echo "Set ATTACHMENT_ID=... REASON='...'"; exit 2; }
|
||||
$(COMPOSE) run --rm -T job-admin ./messenger jobs upgrade "$$ATTACHMENT_ID" 2 "$$REASON"
|
||||
|
||||
jobs-cleanup-staging:
|
||||
@test -n "$$JOB_ID" || { echo "Set JOB_ID=..."; exit 2; }
|
||||
$(COMPOSE) run --rm -T job-admin ./messenger jobs cleanup-staging "$$JOB_ID"
|
||||
|
||||
test-backend: app-init
|
||||
$(COMPOSE) --profile test run --rm --build backend-tests
|
||||
|
||||
test-pipeline: app-init
|
||||
$(COMPOSE) --profile test run --rm --build backend-tests go test -race -count=1 -v ./backend/tests ./backend/workers -run 'TestPipeline|TestVersionedRendering|TestThumbnail|TestOptimized'
|
||||
|
||||
@@ -1,37 +1,68 @@
|
||||
# Отчёт — лабораторная 7
|
||||
# Лабораторная 7
|
||||
|
||||
- Автор, группа: TODO
|
||||
- Целевая оценка; выбранный трек 5, если нужен: TODO
|
||||
- Commit/PR: TODO после публикации
|
||||
- Использованные источники, библиотеки и помощь ИИ: TODO
|
||||
- Автор — Гришко Дмитрий Михайлович, БВТ2303.
|
||||
- Целевая оценка — 4.
|
||||
- PR: https://gitea.privcloud.tech/mtusi2026/mtusi2026lab7/pulls/16.
|
||||
- Источники — задание курса, документация [RabbitMQ](https://www.rabbitmq.com/docs/ssl) и [OpenBao](https://openbao.org/docs/auth/cert/). Библиотеки указаны в `src/go.mod`. Codex использовался при переносе, реализации и проверках.
|
||||
|
||||
## Запуск из чистого clone
|
||||
## Запуск из чистого клона
|
||||
|
||||
TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных.
|
||||
Нужны Linux, Git, Docker Compose, OpenSSL, Python 3.9+ и curl. Приложение собирается на Go 1.22.2. Версии образов указаны в [Dockerfile](src/Dockerfile) и [compose.yml](compose.yml).
|
||||
|
||||
```sh
|
||||
git clone --branch solution/Grishko-BVT2303 --single-branch https://gitea.privcloud.tech/TAskMAster339/mtusi2026lab7-fork.git
|
||||
cd mtusi2026lab7-fork
|
||||
make app-provision
|
||||
python3 scripts/bao_operator.py init
|
||||
python3 scripts/bao_operator.py unseal
|
||||
python3 scripts/configure_bao.py --operator-file secrets/bao-operator.json
|
||||
make app-up
|
||||
make test test-security CA_FILE=secrets/tls/lab7-ingress-ca.crt
|
||||
```
|
||||
|
||||
`app-provision` создаёт `.env` и локальные ключи и пароли, поднимает зависимости, применяет миграции и настраивает права. `init` выполняют один раз для нового тома OpenBao. После перезапуска OpenBao достаточно повторить `unseal` и запустить приложение. Файл оператора хранится на хосте с правами 0600. `make app-up` ждёт ответа HTTPS на `/health/ready`.
|
||||
|
||||
`make app-stop` останавливает контейнеры с сохранением данных. Для намеренного удаления учебных томов используется `docker compose down -v`.
|
||||
|
||||
## Архитектура и решения
|
||||
|
||||
TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab6, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации.
|
||||
Из ЛР 6 перенесены API мессенджера, обработка вложений и сбор телеметрии. В ЛР 7 добавлены TLS между сервисами, отдельные сервисные роли и выдача учётных данных через OpenBao.
|
||||
|
||||
**Вход в API.** Клиент подключается к HAProxy на `localhost:8443`. Балансировщик проверяет имя в сертификате каждой реплики API и проверяет готовность реплик тоже по mTLS. Реплики принимают клиентский сертификат с URI SAN балансировщика. Порты реплик доступны внутри Docker. Доступ к сообщениям и вложениям API проверяет по Bearer-сессии и членству в чате. Обе реплики используют Redis для сессий и общего счётчика попыток входа по IP.
|
||||
|
||||
**Обработка вложений.** API сохраняет исходный файл в закрытом staging-префиксе S3. Метаданные вложения и запись о задании появляются в одной транзакции PostgreSQL. Диспетчер читает запись из БД и публикует задание в RabbitMQ. Воркер забирает файл из staging, сохраняет оригинал и производные изображения в S3 и фиксирует результат в PostgreSQL. Подтверждение RabbitMQ отправляется после записи результата или решения о повторе. После обрыва соединения неподтверждённое задание может поступить повторно. Воркер сверяет версию задания и его состояние в БД перед обработкой.
|
||||
|
||||
**Идентичность и секреты.** В RabbitMQ сертификаты диспетчера и воркера сопоставляются с разными учётными записями через SASL EXTERNAL. Диспетчер может публиковать, воркер может читать очередь. Collector принимает только зарегистрированные сертификаты API, диспетчера и воркера. При старте каждый прикладной процесс получает в OpenBao токен на чтение своего пути KV. Пароли зависимостей записываются в `tmpfs` и используются до перезапуска процесса или отзыва в самой зависимости. Сертификаты и операторские ключи создаются локально.
|
||||
|
||||
`API_HTTPS_PORT`, `API_REPLICAS` и `SESSION_TTL_SECONDS` задают внешний порт, число реплик и срок сессии. Адреса TLS-сервисов заданы в Compose. Файлы подключения приложений заполняются из OpenBao при запуске.
|
||||
|
||||
[Матрица доверия](contracts/trust-matrix.md) · [Модель угроз](security/THREAT_MODEL.md)
|
||||
|
||||
## Доказательства критериев
|
||||
|
||||
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
|
||||
| --- | --- | --- | --- |
|
||||
| Публичный smoke | make test | TODO | TODO |
|
||||
| Остальные пункты уровня 3 | TODO | TODO | TODO |
|
||||
| Все пункты уровня 4, если заявлен | TODO | TODO | TODO |
|
||||
| Выбранный трек 5, если заявлен | TODO | TODO | TODO |
|
||||
|
||||
TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены.
|
||||
| Публичный smoke по HTTPS | `make test test-security CA_FILE=secrets/tls/lab7-ingress-ca.crt` | API прошёл smoke. Соединение без доверия к CA отклонено. | [HTTPS](evidence/grade3-https.md) |
|
||||
| Argon2id | Бенчмарк `BenchmarkPasswordVerification` при 0,5 CPU | 64 МиБ, 3 прохода, p=2. В трёх сериях по три операции среднее время проверки составило 177–196 мс. | [Замер](evidence/grade3-argon2.md) |
|
||||
| mTLS HAProxy → API | `python3 scripts/check_ingress_mtls.py` | Обе реплики приняли сертификат HAProxy. Отказ без сертификата, с чужим CA и с чужим SAN. | [mTLS](evidence/grade3-mtls.md) |
|
||||
| TLS зависимостей | `python3 scripts/check_grade3_tls.py`, `./scripts/check_dependency_roles.sh` | Хранилища, брокер и телеметрия приняли соединения с проверкой имени сервера и отклонили чужой CA. Проверены роли PostgreSQL, Redis и RabbitMQ. | [TLS и роли](evidence/grade3-dependencies.md) |
|
||||
| Доступ к секретам | `python3 scripts/check_bao_roles.py` | Каждая роль прочитала свой путь KV. Попытки прочитать чужой путь или выполнить запись отклонены. | [OpenBao](evidence/grade3-openbao.md) |
|
||||
| Изоляция и бюджет воркера | `python3 scripts/check_grade3_runtime.py` | Проверены локальные публикации портов, read-only root, сброс capabilities и tmpfs. Для воркера заданы 1 CPU, 512 МиБ и 60 с на обработку по умолчанию. | [Изоляция](evidence/grade4-hardening.md) |
|
||||
| Вложения, сессии и лимит входа | `check_grade3_http.py`, `check_session_expiry.py`, `check_login_rate.py`, smoke | API отклонил чужое вложение, подмену пользователя, неверный MIME и превышение размера. Токен после выхода или истечения отклонён. На 13-й попытке входа получен 429. | [ACL](evidence/grade3-http-security.md), [сессии](evidence/grade3-expiry.md), [лимит](evidence/grade3-login.md) |
|
||||
| mTLS брокера и Collector | `python3 scripts/collect_grade4.py --only grade4-mtls` | Сертификат чужой роли от того же CA отклонён. RabbitMQ запретил публикацию воркером и чтение диспетчером. Телеметрия поступила. | [Роли и mTLS](evidence/grade4-mtls.md) |
|
||||
| Обновление хеша и нагрузка на вход | `python3 scripts/check_grade4_login.py` | Хеш обновился в PostgreSQL после успешного входа. Из 24 параллельных запросов 8 получили 201 и 16 получили 429. Медиана 548 мс, максимум 1156 мс. | [Вход под нагрузкой](evidence/grade4-login.md) |
|
||||
| Ротация и запечатывание OpenBao | `python3 scripts/check_grade4_lifecycle.py` | Пароль БД и сертификат OpenBao заменены без пересборки. Старые данные и токен после истечения отклонены. При seal запущенный API оставался готов. | [Ротация](evidence/grade4-lifecycle.md) |
|
||||
| Модель угроз и аудит | `python3 scripts/collect_grade4.py --only grade4-audit` | Повторены проверки TLS и ACL. Синтетический секрет не найден в связанном логе и трейсе, метриках и логах контейнеров. | [Аудит](evidence/grade4-audit.md) |
|
||||
| Сканирование | `./scripts/scan_grade4.sh` | Trivy проверил зависимости и образы. Для MinIO использован текущий собранный образ, поскольку прежний ID Docker не смог экспортировать. | [Сканирование](evidence/grade4-scan.md), [применимость находок](evidence/grade4-scan-triage.md) |
|
||||
|
||||
## Эксперименты и отказы
|
||||
|
||||
TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения.
|
||||
|
||||
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
|
||||
| --- | --- | --- | --- |
|
||||
| TODO | TODO | TODO | TODO |
|
||||
| Перезапуск PostgreSQL, Redis, S3, RabbitMQ и двух API | Данные и сессия сохраняются | Повторный запрос с прежним токеном прошёл. Время старта контейнеров изменилось. | Подтверждённые данные сохранились. [Результат](evidence/grade3-resilience.md) |
|
||||
| Остановка первой реплики API | Вторая реплика продолжит принимать запросы | В двух прогонах прошли 124 и 122 записи. | Запись восстановилась за 1,454 и 1,580 с. Подтверждённые сообщения сохранились. [Результат](evidence/grade3-resilience.md) |
|
||||
| Остановка второй реплики API | Первая реплика обслужит бизнес-сценарий | HTTPS smoke прошёл. | Время восстановления отдельно не замерялось. [Результат](evidence/grade3-other-replica.md) |
|
||||
|
||||
## Ограничения и следующие шаги
|
||||
## Ограничения
|
||||
|
||||
TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза.
|
||||
Стенд работает на одном хосте. Эксперименты покрывают перезапуск хранилищ и остановку одной API-реплики. OpenBao разблокируется вручную. Истечение токена OpenBao прекращает чтение KV, а уже выданный статический пароль действует до смены в целевом сервисе.
|
||||
|
||||
+557
@@ -0,0 +1,557 @@
|
||||
x-bao-environment: &bao-environment
|
||||
BAO_ADDR: https://openbao:8200
|
||||
SSL_CERT_FILE: /run/tls/ca.crt
|
||||
x-secret-tmpfs: &secret-tmpfs /run/runtime-secrets:rw,noexec,nosuid,size=1m,mode=1777
|
||||
|
||||
name: highload-lab7
|
||||
|
||||
services:
|
||||
redis:
|
||||
cpus: 0.25
|
||||
mem_limit: 128m
|
||||
networks: [data]
|
||||
image: redis:7.4.11-bookworm
|
||||
command:
|
||||
[
|
||||
"redis-server",
|
||||
"--port", "0",
|
||||
"--tls-port", "6379",
|
||||
"--tls-cert-file", "/run/tls/redis.crt",
|
||||
"--tls-key-file", "/run/tls/redis.key",
|
||||
"--tls-ca-cert-file", "/run/tls/ca.crt",
|
||||
"--tls-auth-clients", "no",
|
||||
"--aclfile", "/run/secrets/redis_acl",
|
||||
"--appendonly",
|
||||
"yes",
|
||||
"--appendfsync",
|
||||
"everysec",
|
||||
"--dir",
|
||||
"/data",
|
||||
]
|
||||
volumes:
|
||||
- redis_data:/data
|
||||
- ./secrets/tls/lab7-ca.crt:/run/tls/ca.crt:ro
|
||||
- ./secrets/tls/redis.crt:/run/tls/redis.crt:ro
|
||||
- ./secrets/tls/redis.key:/run/tls/redis.key:ro
|
||||
- ./secrets/redis_acl:/run/secrets/redis_acl:ro
|
||||
- ./secrets/redis_password:/run/secrets/redis_password:ro
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "REDISCLI_AUTH=$$(cat /run/secrets/redis_password) redis-cli --tls --cacert /run/tls/ca.crt -h redis --user messenger ping | grep -qx PONG"]
|
||||
interval: 3s
|
||||
timeout: 3s
|
||||
retries: 20
|
||||
db:
|
||||
cpus: 1.0
|
||||
mem_limit: 512m
|
||||
networks: [data]
|
||||
image: postgres:16.10-bookworm
|
||||
environment:
|
||||
POSTGRES_DB: ${POSTGRES_DB}
|
||||
POSTGRES_USER: ${POSTGRES_USER}
|
||||
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD}
|
||||
volumes:
|
||||
- postgres_data:/var/lib/postgresql/data
|
||||
- ./secrets/tls/lab7-ca.crt:/run/tls/ca.crt:ro
|
||||
- ./secrets/tls/db.crt:/run/tls/db.crt:ro
|
||||
- ./secrets/tls/db.key:/run/tls/db.key:ro
|
||||
- ./secrets/database_api_password:/run/secrets/database_api_password:ro
|
||||
- ./secrets/database_worker_password:/run/secrets/database_worker_password:ro
|
||||
- ./secrets/database_dispatcher_password:/run/secrets/database_dispatcher_password:ro
|
||||
- ./secrets/database_cleanup_password:/run/secrets/database_cleanup_password:ro
|
||||
- ./infra/postgres-hba.conf:/run/tls/pg_hba.conf:ro
|
||||
entrypoint: ["/bin/sh", "-ec", "install -o postgres -g postgres -m 600 /run/tls/db.key /tmp/db.key; exec docker-entrypoint.sh postgres -c ssl=on -c ssl_cert_file=/run/tls/db.crt -c ssl_key_file=/tmp/db.key -c hba_file=/run/tls/pg_hba.conf"]
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "pg_isready -U $$POSTGRES_USER -d $$POSTGRES_DB"]
|
||||
interval: 3s
|
||||
timeout: 3s
|
||||
retries: 20
|
||||
openbao:
|
||||
image: openbao/openbao:2.4.4
|
||||
user: openbao
|
||||
networks: [bao, edge]
|
||||
ports: ["127.0.0.1:8200:8200"]
|
||||
cap_drop: [ALL]
|
||||
security_opt: ["no-new-privileges:true"]
|
||||
mem_limit: 256m
|
||||
cpus: 0.5
|
||||
entrypoint: ["bao"]
|
||||
command: ["server", "-config=/openbao/config/server.hcl"]
|
||||
volumes:
|
||||
- openbao_data:/openbao/file
|
||||
- ./infra/openbao/server.hcl:/openbao/config/server.hcl:ro
|
||||
- ./secrets/tls/openbao.crt:/run/tls/bao.crt:ro
|
||||
- ./secrets/tls/openbao.key:/run/tls/bao.key:ro
|
||||
migrate:
|
||||
cpus: 0.5
|
||||
mem_limit: 128m
|
||||
networks: [data]
|
||||
build: ./src
|
||||
environment:
|
||||
DATABASE_URL_FILE: ${DATABASE_URL_FILE:-/run/secrets/database_url}
|
||||
volumes:
|
||||
- ./secrets/tls/lab7-ca.crt:/run/tls/ca.crt:ro
|
||||
- ./secrets/database_tls_url:${DATABASE_URL_FILE:-/run/secrets/database_url}:ro
|
||||
command: ["./messenger", "migrate", "up"]
|
||||
depends_on:
|
||||
db:
|
||||
condition: service_healthy
|
||||
s3:
|
||||
cpus: 1.0
|
||||
mem_limit: 512m
|
||||
networks: [edge, data]
|
||||
ports:
|
||||
- "127.0.0.1:9000:9000"
|
||||
build:
|
||||
context: ./infra
|
||||
dockerfile: minio.Dockerfile
|
||||
environment:
|
||||
MINIO_ROOT_USER_FILE: /run/secrets/s3_access_key_id
|
||||
MINIO_ROOT_PASSWORD_FILE: /run/secrets/s3_secret_access_key
|
||||
MINIO_BROWSER: "off"
|
||||
MINIO_API_CORS_ALLOW_ORIGIN: ${S3_CORS_ORIGINS:-http://127.0.0.1:5173,http://localhost:5173}
|
||||
volumes:
|
||||
- s3_data:/data
|
||||
- ./secrets/tls/s3.crt:/run/minio-certs/public.crt:ro
|
||||
- ./secrets/tls/s3.key:/run/minio-certs/private.key:ro
|
||||
- ./secrets/tls/lab7-ca.crt:/run/tls/ca.crt:ro
|
||||
- ./secrets/s3_access_key_id:/run/secrets/s3_access_key_id:ro
|
||||
- ./secrets/s3_secret_access_key:/run/secrets/s3_secret_access_key:ro
|
||||
command: ["server", "/data", "--certs-dir", "/run/minio-certs", "--console-address", "127.0.0.1:9001"]
|
||||
healthcheck:
|
||||
test: ["CMD", "curl", "--cacert", "/run/tls/ca.crt", "-f", "https://s3:9000/minio/health/ready"]
|
||||
interval: 3s
|
||||
timeout: 2s
|
||||
retries: 30
|
||||
s3-bootstrap:
|
||||
cpus: 0.5
|
||||
mem_limit: 128m
|
||||
networks: [data]
|
||||
build: ./src
|
||||
environment: &s3-environment
|
||||
SSL_CERT_FILE: /run/tls/ca.crt
|
||||
S3_CLEANUP_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_cleanup_secret_access_key
|
||||
S3_CLEANUP_ACCESS_KEY_ID_FILE: /run/secrets/s3_cleanup_access_key_id
|
||||
S3_WORKER_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_worker_secret_access_key
|
||||
S3_WORKER_ACCESS_KEY_ID_FILE: /run/secrets/s3_worker_access_key_id
|
||||
S3_API_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_api_secret_access_key
|
||||
S3_API_ACCESS_KEY_ID_FILE: /run/secrets/s3_api_access_key_id
|
||||
S3_ROOT_ACCESS_KEY_ID_FILE: /run/secrets/s3_root_access_key_id
|
||||
S3_ROOT_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_root_secret_access_key
|
||||
S3_ENDPOINT: https://s3:9000
|
||||
S3_PUBLIC_ENDPOINT: https://localhost:9000
|
||||
S3_REGION: ${S3_REGION:-us-east-1}
|
||||
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
|
||||
S3_ACCESS_KEY_ID_FILE: ${S3_ACCESS_KEY_ID_FILE:-/run/secrets/s3_access_key_id}
|
||||
S3_SECRET_ACCESS_KEY_FILE: ${S3_SECRET_ACCESS_KEY_FILE:-/run/secrets/s3_secret_access_key}
|
||||
volumes:
|
||||
- ./secrets/s3_cleanup_secret_access_key:/run/secrets/s3_cleanup_secret_access_key:ro
|
||||
- ./secrets/s3_cleanup_access_key_id:/run/secrets/s3_cleanup_access_key_id:ro
|
||||
- ./secrets/s3_worker_secret_access_key:/run/secrets/s3_worker_secret_access_key:ro
|
||||
- ./secrets/s3_worker_access_key_id:/run/secrets/s3_worker_access_key_id:ro
|
||||
- ./secrets/s3_api_secret_access_key:/run/secrets/s3_api_secret_access_key:ro
|
||||
- ./secrets/s3_api_access_key_id:/run/secrets/s3_api_access_key_id:ro
|
||||
- ./secrets/tls/lab7-ca.crt:/run/tls/ca.crt:ro
|
||||
- ./secrets/s3_access_key_id:/run/secrets/s3_root_access_key_id:ro
|
||||
- ./secrets/s3_secret_access_key:/run/secrets/s3_root_secret_access_key:ro
|
||||
- ./secrets/s3_api_access_key_id:${S3_ACCESS_KEY_ID_FILE:-/run/secrets/s3_access_key_id}:ro
|
||||
- ./secrets/s3_api_secret_access_key:${S3_SECRET_ACCESS_KEY_FILE:-/run/secrets/s3_secret_access_key}:ro
|
||||
command: ["./messenger", "s3-bootstrap"]
|
||||
depends_on:
|
||||
s3:
|
||||
condition: service_healthy
|
||||
api:
|
||||
cap_drop: [ALL]
|
||||
security_opt: ["no-new-privileges:true"]
|
||||
read_only: true
|
||||
cpus: 0.5
|
||||
mem_limit: 512m
|
||||
networks: [proxy, data, ingest, bao]
|
||||
build: ./src
|
||||
deploy:
|
||||
replicas: ${API_REPLICAS:-2}
|
||||
stop_grace_period: 15s
|
||||
environment: &api-environment
|
||||
<<: [*s3-environment, *bao-environment]
|
||||
OTEL_EXPORTER_OTLP_ENDPOINT: https://otel-collector:4318
|
||||
OTEL_EXPORTER_OTLP_CLIENT_CERTIFICATE: /run/otel/client.crt
|
||||
OTEL_EXPORTER_OTLP_CLIENT_KEY: /run/otel/client.key
|
||||
S3_ROOT_ACCESS_KEY_ID_FILE: ""
|
||||
S3_ROOT_SECRET_ACCESS_KEY_FILE: ""
|
||||
JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5}
|
||||
JOB_TIMEOUT_SECONDS: ${JOB_TIMEOUT_SECONDS:-60}
|
||||
WORKER_MAX_PIXELS: ${WORKER_MAX_PIXELS:-8000000}
|
||||
UPLOAD_TMP_DIR: /tmp/uploads
|
||||
UPLOAD_CLEANUP_GRACE_SECONDS: ${UPLOAD_CLEANUP_GRACE_SECONDS:-600}
|
||||
# Go 1.22 sees host CPUs; keep parallelism suitable for this 0.5 CPU limit.
|
||||
GOMAXPROCS: ${API_GOMAXPROCS:-1}
|
||||
GOMEMLIMIT: 384MiB
|
||||
BAO_ROLE: api
|
||||
TLS_CA_FILE: /run/tls/ca.crt
|
||||
TLS_CERT_FILE: /run/tls/api.crt
|
||||
TLS_KEY_FILE: /run/tls/api.key
|
||||
APP_HOST: ${APP_HOST:-0.0.0.0}
|
||||
APP_PORT: ${APP_PORT:-8080}
|
||||
APP_ENV: ${APP_ENV:-lab}
|
||||
IDEMPOTENCY_TTL_SECONDS: ${IDEMPOTENCY_TTL_SECONDS:-86400}
|
||||
DATABASE_URL_FILE: ${DATABASE_URL_FILE:-/run/secrets/database_url}
|
||||
CACHE_URL_FILE: ${CACHE_URL_FILE:-/run/secrets/cache_url}
|
||||
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600}
|
||||
REDIS_ADDR: redis:6379
|
||||
tmpfs:
|
||||
- *secret-tmpfs
|
||||
- /tmp/uploads:rw,noexec,nosuid,size=16m,mode=1777
|
||||
volumes:
|
||||
- &ca ./secrets/tls/lab7-ca.crt:/run/tls/ca.crt:ro
|
||||
- ./secrets/tls/api.crt:/run/tls/api.crt:ro
|
||||
- ./secrets/tls/api.key:/run/tls/api.key:ro
|
||||
- ./secrets/tls/bao-api.crt:/run/bao/client.crt:ro
|
||||
- ./secrets/tls/bao-api.key:/run/bao/client.key:ro
|
||||
- ./secrets/tls/otel-api.crt:/run/otel/client.crt:ro
|
||||
- ./secrets/tls/otel-api.key:/run/otel/client.key:ro
|
||||
depends_on:
|
||||
s3-bootstrap:
|
||||
condition: service_completed_successfully
|
||||
migrate:
|
||||
condition: service_completed_successfully
|
||||
db:
|
||||
condition: service_healthy
|
||||
redis:
|
||||
condition: service_healthy
|
||||
healthcheck:
|
||||
disable: true
|
||||
broker:
|
||||
image: rabbitmq:4.2.0-management
|
||||
hostname: broker
|
||||
environment:
|
||||
RABBITMQ_SERVER_ADDITIONAL_ERL_ARGS: "+S 2:2"
|
||||
RABBITMQ_CTL_ERL_ARGS: "+S 1:1"
|
||||
cpus: 0.5
|
||||
mem_limit: 512m
|
||||
networks: [data]
|
||||
command:
|
||||
- sh
|
||||
- -ec
|
||||
- |
|
||||
export RABBITMQ_DEFAULT_USER="$$(cat /run/secrets/broker_user)"
|
||||
export RABBITMQ_DEFAULT_PASS="$$(cat /run/secrets/broker_password)"
|
||||
exec docker-entrypoint.sh rabbitmq-server
|
||||
volumes:
|
||||
- ./infra/rabbitmq/bootstrap.sh:/bootstrap.sh:ro
|
||||
- rabbitmq_data:/var/lib/rabbitmq
|
||||
- ./infra/rabbitmq/enabled_plugins:/etc/rabbitmq/enabled_plugins:ro
|
||||
- ./secrets/tls/broker-client-ca.crt:/run/tls/broker-client-ca.crt:ro
|
||||
- ./infra/rabbitmq/rabbitmq.conf:/etc/rabbitmq/rabbitmq.conf:ro
|
||||
- ./secrets/tls/lab7-ca.crt:/run/tls/ca.crt:ro
|
||||
- ./secrets/tls/broker.crt:/run/tls/broker.crt:ro
|
||||
- ./secrets/tls/broker.key:/run/tls/broker.key:ro
|
||||
- ./secrets/broker_user:/run/secrets/broker_user:ro
|
||||
- ./secrets/broker_password:/run/secrets/broker_password:ro
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "rabbitmq-diagnostics -q check_running"]
|
||||
interval: 5s
|
||||
timeout: 10s
|
||||
start_period: 20s
|
||||
retries: 30
|
||||
worker:
|
||||
cap_drop: [ALL]
|
||||
security_opt: ["no-new-privileges:true"]
|
||||
read_only: true
|
||||
tmpfs:
|
||||
- *secret-tmpfs
|
||||
- /tmp:rw,noexec,nosuid,size=64m,mode=1777
|
||||
build: ./src
|
||||
restart: unless-stopped
|
||||
cpus: 1.0
|
||||
mem_limit: 512m
|
||||
networks: [data, ingest, bao]
|
||||
environment:
|
||||
<<: *api-environment
|
||||
BAO_ROLE: worker
|
||||
BROKER_MTLS: "1"
|
||||
CACHE_URL_FILE: ""
|
||||
volumes:
|
||||
- *ca
|
||||
- ./secrets/tls/bao-worker.crt:/run/bao/client.crt:ro
|
||||
- ./secrets/tls/bao-worker.key:/run/bao/client.key:ro
|
||||
- ./secrets/tls/messenger_worker.crt:/run/broker/client.crt:ro
|
||||
- ./secrets/tls/messenger_worker.key:/run/broker/client.key:ro
|
||||
- ./secrets/tls/otel-worker.crt:/run/otel/client.crt:ro
|
||||
- ./secrets/tls/otel-worker.key:/run/otel/client.key:ro
|
||||
command: ["./messenger", "worker"]
|
||||
healthcheck:
|
||||
disable: true
|
||||
depends_on:
|
||||
migrate:
|
||||
condition: service_completed_successfully
|
||||
s3-bootstrap:
|
||||
condition: service_completed_successfully
|
||||
broker:
|
||||
condition: service_healthy
|
||||
dispatcher:
|
||||
cap_drop: [ALL]
|
||||
security_opt: ["no-new-privileges:true"]
|
||||
read_only: true
|
||||
tmpfs: [*secret-tmpfs]
|
||||
build: ./src
|
||||
restart: unless-stopped
|
||||
cpus: 0.25
|
||||
mem_limit: 128m
|
||||
networks: [data, ingest, bao]
|
||||
environment:
|
||||
<<: *api-environment
|
||||
BAO_ROLE: dispatcher
|
||||
BROKER_MTLS: "1"
|
||||
CACHE_URL_FILE: ""
|
||||
S3_ACCESS_KEY_ID_FILE: ""
|
||||
S3_SECRET_ACCESS_KEY_FILE: ""
|
||||
volumes:
|
||||
- *ca
|
||||
- ./secrets/tls/bao-dispatcher.crt:/run/bao/client.crt:ro
|
||||
- ./secrets/tls/bao-dispatcher.key:/run/bao/client.key:ro
|
||||
- ./secrets/tls/messenger_dispatcher.crt:/run/broker/client.crt:ro
|
||||
- ./secrets/tls/messenger_dispatcher.key:/run/broker/client.key:ro
|
||||
- ./secrets/tls/otel-dispatcher.crt:/run/otel/client.crt:ro
|
||||
- ./secrets/tls/otel-dispatcher.key:/run/otel/client.key:ro
|
||||
command: ["./messenger", "dispatcher"]
|
||||
healthcheck:
|
||||
disable: true
|
||||
depends_on:
|
||||
migrate:
|
||||
condition: service_completed_successfully
|
||||
broker:
|
||||
condition: service_healthy
|
||||
upload-cleanup:
|
||||
cap_drop: [ALL]
|
||||
security_opt: ["no-new-privileges:true"]
|
||||
read_only: true
|
||||
tmpfs: [*secret-tmpfs]
|
||||
restart: unless-stopped
|
||||
cpus: 0.25
|
||||
mem_limit: 128m
|
||||
networks: [data, bao]
|
||||
build: ./src
|
||||
command: ["./messenger", "attachments-cleanup", "--loop"]
|
||||
environment:
|
||||
<<: [*s3-environment, *bao-environment]
|
||||
BAO_ROLE: cleanup
|
||||
S3_ROOT_ACCESS_KEY_ID_FILE: ""
|
||||
S3_ROOT_SECRET_ACCESS_KEY_FILE: ""
|
||||
DATABASE_URL_FILE: ${DATABASE_URL_FILE:-/run/secrets/database_url}
|
||||
volumes:
|
||||
- *ca
|
||||
- ./secrets/tls/bao-cleanup.crt:/run/bao/client.crt:ro
|
||||
- ./secrets/tls/bao-cleanup.key:/run/bao/client.key:ro
|
||||
healthcheck:
|
||||
disable: true
|
||||
depends_on:
|
||||
migrate:
|
||||
condition: service_completed_successfully
|
||||
s3-bootstrap:
|
||||
condition: service_completed_successfully
|
||||
haproxy:
|
||||
cap_drop: [ALL]
|
||||
security_opt: ["no-new-privileges:true"]
|
||||
read_only: true
|
||||
tmpfs: ["/tmp:rw,noexec,nosuid,size=16m,mode=1777"]
|
||||
environment:
|
||||
APP_PORT: ${APP_PORT:-8080}
|
||||
API_REPLICAS: ${API_REPLICAS:-2}
|
||||
cpus: 0.5
|
||||
mem_limit: 128m
|
||||
networks: [edge, proxy]
|
||||
image: haproxy:3.2.25-alpine
|
||||
ports:
|
||||
- "127.0.0.1:${API_HTTPS_PORT:-8443}:8443"
|
||||
volumes:
|
||||
- ./infra/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro
|
||||
- ./secrets/tls/lab7-ca.crt:/run/tls/ca.crt:ro
|
||||
- ./secrets/tls/ingress-browser.pem:/run/tls/ingress.pem:ro
|
||||
- ./secrets/tls/lb.pem:/run/tls/lb.pem:ro
|
||||
depends_on:
|
||||
api:
|
||||
condition: service_started
|
||||
job-admin:
|
||||
profiles: ["operator"]
|
||||
build: ./src
|
||||
networks: [data]
|
||||
cpus: 0.25
|
||||
mem_limit: 128m
|
||||
cap_drop: [ALL]
|
||||
security_opt: ["no-new-privileges:true"]
|
||||
environment:
|
||||
<<: *s3-environment
|
||||
DATABASE_URL_FILE: /run/secrets/database_url
|
||||
S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id
|
||||
S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key
|
||||
volumes:
|
||||
- *ca
|
||||
- ./secrets/database_tls_url:/run/secrets/database_url:ro
|
||||
- ./secrets/s3_access_key_id:/run/secrets/s3_access_key_id:ro
|
||||
- ./secrets/s3_secret_access_key:/run/secrets/s3_secret_access_key:ro
|
||||
backend-tests:
|
||||
cpus: 2.0
|
||||
mem_limit: 1g
|
||||
networks: [data]
|
||||
profiles: ["test"]
|
||||
build:
|
||||
context: ./src
|
||||
target: builder
|
||||
environment:
|
||||
<<: *s3-environment
|
||||
S3_PUBLIC_ENDPOINT: https://s3:9000
|
||||
TEST_DATABASE_URL: postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@db:5432/${POSTGRES_DB}?sslmode=verify-full&sslrootcert=/run/tls/ca.crt
|
||||
TEST_REDIS_ADDR: redis:6379
|
||||
CACHE_URL_FILE: /run/secrets/cache_url
|
||||
volumes:
|
||||
- *ca
|
||||
- ./secrets/cache_tls_url:/run/secrets/cache_url:ro
|
||||
- ./secrets/s3_api_access_key_id:${S3_ACCESS_KEY_ID_FILE:-/run/secrets/s3_access_key_id}:ro
|
||||
- ./secrets/s3_api_secret_access_key:${S3_SECRET_ACCESS_KEY_FILE:-/run/secrets/s3_secret_access_key}:ro
|
||||
- ./secrets/s3_access_key_id:/run/secrets/s3_root_access_key_id:ro
|
||||
- ./secrets/s3_secret_access_key:/run/secrets/s3_root_secret_access_key:ro
|
||||
depends_on:
|
||||
s3-bootstrap:
|
||||
condition: service_completed_successfully
|
||||
db:
|
||||
condition: service_healthy
|
||||
redis:
|
||||
condition: service_healthy
|
||||
command: ["go", "test", "-race", "-count=1", "-v", "./backend/..."]
|
||||
otel-collector:
|
||||
cap_drop: [ALL]
|
||||
security_opt: ["no-new-privileges:true"]
|
||||
read_only: true
|
||||
tmpfs: ["/tmp:rw,noexec,nosuid,size=16m,mode=1777"]
|
||||
cpus: 0.5
|
||||
image: otel/opentelemetry-collector-contrib:0.120.0
|
||||
networks: [telemetry, ingest]
|
||||
mem_limit: 192m
|
||||
command: ["--config=/etc/otelcol-contrib/config.yaml"]
|
||||
volumes:
|
||||
- ./secrets/tls/telemetry-export-ca.crt:/run/otel/export-ca.crt:ro
|
||||
- ./secrets/tls/collector-exporter.crt:/run/otel/client.crt:ro
|
||||
- ./secrets/tls/collector-exporter.key:/run/otel/client.key:ro
|
||||
- *ca
|
||||
- ./secrets/tls/otel-collector.crt:/run/otel/server.crt:ro
|
||||
- ./secrets/tls/otel-collector.key:/run/otel/server.key:ro
|
||||
- ./secrets/tls/telemetry-ingest-clients.crt:/run/otel/ingest-ca.crt:ro
|
||||
- ./infra/otel/collector.yaml:/etc/otelcol-contrib/config.yaml:ro
|
||||
|
||||
prometheus:
|
||||
cap_drop: [ALL]
|
||||
security_opt: ["no-new-privileges:true"]
|
||||
read_only: true
|
||||
tmpfs: ["/tmp:rw,noexec,nosuid,size=16m,mode=1777"]
|
||||
cpus: 0.5
|
||||
mem_limit: 384m
|
||||
image: prom/prometheus:v3.2.1
|
||||
networks: [telemetry]
|
||||
command: ["--config.file=/etc/prometheus/prometheus.yml", "--storage.tsdb.path=/prometheus", "--storage.tsdb.retention.time=7d", "--web.config.file=/etc/prometheus/web.yaml"]
|
||||
volumes:
|
||||
- ./secrets/tls/telemetry-read-ca.crt:/run/otel/read-ca.crt:ro
|
||||
- ./secrets/tls/prometheus-scraper.crt:/run/otel/client.crt:ro
|
||||
- ./secrets/tls/prometheus-scraper.key:/run/otel/client.key:ro
|
||||
- *ca
|
||||
- ./secrets/tls/prometheus.crt:/run/otel/server.crt:ro
|
||||
- ./secrets/tls/prometheus.key:/run/otel/server.key:ro
|
||||
- ./infra/otel/prometheus-web.yaml:/etc/prometheus/web.yaml:ro
|
||||
- ./infra/otel/prometheus.yaml:/etc/prometheus/prometheus.yml:ro
|
||||
- ./infra/otel/alerts.yaml:/etc/prometheus/alerts.yaml:ro
|
||||
- prometheus_data:/prometheus
|
||||
|
||||
loki:
|
||||
cap_drop: [ALL]
|
||||
security_opt: ["no-new-privileges:true"]
|
||||
read_only: true
|
||||
tmpfs: ["/tmp:rw,noexec,nosuid,size=16m,mode=1777"]
|
||||
cpus: 0.5
|
||||
mem_limit: 384m
|
||||
image: grafana/loki:3.4.2
|
||||
networks: [telemetry]
|
||||
command: ["-config.file=/etc/loki/lab.yaml"]
|
||||
volumes:
|
||||
- ./secrets/tls/loki-client-ca.crt:/run/otel/loki-client-ca.crt:ro
|
||||
- *ca
|
||||
- ./secrets/tls/loki.crt:/run/otel/server.crt:ro
|
||||
- ./secrets/tls/loki.key:/run/otel/server.key:ro
|
||||
- ./infra/otel/loki.yaml:/etc/loki/lab.yaml:ro
|
||||
- loki_data:/loki
|
||||
|
||||
tempo:
|
||||
cap_drop: [ALL]
|
||||
security_opt: ["no-new-privileges:true"]
|
||||
read_only: true
|
||||
tmpfs: ["/tmp:rw,noexec,nosuid,size=16m,mode=1777"]
|
||||
cpus: 0.5
|
||||
mem_limit: 384m
|
||||
image: grafana/tempo:2.7.1
|
||||
networks: [telemetry]
|
||||
command: ["-config.file=/etc/tempo/lab.yaml"]
|
||||
volumes:
|
||||
- ./secrets/tls/telemetry-read-ca.crt:/run/otel/read-ca.crt:ro
|
||||
- ./secrets/tls/telemetry-export-ca.crt:/run/otel/export-ca.crt:ro
|
||||
- *ca
|
||||
- ./secrets/tls/tempo.crt:/run/otel/server.crt:ro
|
||||
- ./secrets/tls/tempo.key:/run/otel/server.key:ro
|
||||
- ./infra/otel/tempo.yaml:/etc/tempo/lab.yaml:ro
|
||||
- tempo_data:/var/tempo
|
||||
|
||||
grafana:
|
||||
cap_drop: [ALL]
|
||||
security_opt: ["no-new-privileges:true"]
|
||||
read_only: true
|
||||
tmpfs: ["/tmp:rw,noexec,nosuid,size=16m,mode=1777"]
|
||||
entrypoint:
|
||||
- /bin/sh
|
||||
- -ec
|
||||
- |
|
||||
export GRAFANA_READER_CERT="$$(cat /run/otel/client.crt)"
|
||||
export GRAFANA_READER_KEY="$$(cat /run/otel/client.key)"
|
||||
exec /run.sh
|
||||
cpus: 0.5
|
||||
mem_limit: 384m
|
||||
image: grafana/grafana:11.5.2
|
||||
networks: [telemetry, edge]
|
||||
environment:
|
||||
SSL_CERT_FILE: /run/tls/ca.crt
|
||||
GF_AUTH_ANONYMOUS_ENABLED: "false"
|
||||
GF_SERVER_PROTOCOL: https
|
||||
GF_SERVER_CERT_FILE: /run/otel/server.crt
|
||||
GF_SERVER_CERT_KEY: /run/otel/server.key
|
||||
GF_SECURITY_ADMIN_PASSWORD__FILE: /run/secrets/grafana_admin_password
|
||||
ports:
|
||||
- "127.0.0.1:${GRAFANA_HTTP_PORT:-3000}:3000"
|
||||
volumes:
|
||||
- ./secrets/tls/grafana-reader.crt:/run/otel/client.crt:ro
|
||||
- ./secrets/tls/grafana-reader.key:/run/otel/client.key:ro
|
||||
- *ca
|
||||
- ./secrets/tls/grafana.crt:/run/otel/server.crt:ro
|
||||
- ./secrets/tls/grafana.key:/run/otel/server.key:ro
|
||||
- ./infra/otel/grafana/provisioning:/etc/grafana/provisioning:ro
|
||||
- ./infra/otel/grafana/dashboards:/var/lib/grafana/dashboards:ro
|
||||
- ./secrets/grafana_admin_password:/run/secrets/grafana_admin_password:ro
|
||||
- grafana_data:/var/lib/grafana
|
||||
|
||||
volumes:
|
||||
openbao_data:
|
||||
prometheus_data:
|
||||
loki_data:
|
||||
tempo_data:
|
||||
grafana_data:
|
||||
rabbitmq_data:
|
||||
postgres_data:
|
||||
redis_data:
|
||||
s3_data:
|
||||
|
||||
networks:
|
||||
bao:
|
||||
internal: true
|
||||
edge:
|
||||
proxy:
|
||||
internal: true
|
||||
data:
|
||||
internal: true
|
||||
telemetry:
|
||||
internal: true
|
||||
ingest:
|
||||
internal: true
|
||||
+19
-32
@@ -1,37 +1,24 @@
|
||||
# Матрица доверия — заполнить в решении lab7
|
||||
# Матрица доверия
|
||||
|
||||
Это обязательная матрица покрытия. Для каждой строки укажите реальный порт, endpoint, CA, service identity, проверку SAN/ACL, срок жизни credentials и команду положительного/отрицательного теста. TLS шифрует и подтверждает сервер; mTLS также проверяет клиента. При доверенном CA всё равно нужно ограничивать разрешённую identity. Plaintext hop допустим только внутри закрытого локального transport/proxy boundary, без опубликованного обхода; границу и rationale нужно доказать.
|
||||
Внешний сертификат HAProxy использует ECDSA P-256 и отдельный CA `lab7-ingress-ca`. Остальные сетевые серверы предъявляют leaf со своим DNS SAN, подписанный локальным `lab7-ca`. Leaf действуют 30 дней, CA — 365 дней. Для доступа с хоста сертификаты HAProxy, MinIO, Grafana и OpenBao также содержат `localhost` и `127.0.0.1`. Пароли PostgreSQL, Redis, S3 и RabbitMQ статические, до смены оператором. Параметры действующего стенда приведены ниже.
|
||||
|
||||
| Связь | Минимум на 3 | Дополнительно на 4 | Доказательство / конфиг |
|
||||
| --- | --- | --- | --- |
|
||||
| Клиент → ingress/LB | TLS, CA + hostname, Bearer | Проверка expiry/revocation сессии и общего login rate limit | TODO |
|
||||
| LB → API (все реплики) | mTLS + allowlist LB identity | Ротация leaf без обхода проверки | TODO |
|
||||
| API/worker → PostgreSQL | TLS verify-full + раздельные ограниченные DB roles | Ротация credentials; mTLS — допустимое усиление | TODO |
|
||||
| API/worker → S3 | TLS + разные scoped credentials/prefix rights | Проверка denied prefix и ротация | TODO |
|
||||
| API/worker → broker | TLS + разные producer/consumer identities и ACL | mTLS + ограничение допустимых SAN/ролей | TODO |
|
||||
| API/worker → Collector | TLS; сеть/аутентификация запрещает произвольный ingest | mTLS с раздельными identities | TODO |
|
||||
| Collector → telemetry backends | TLS + scoped backend credentials или mTLS | Ротация и проверка least privilege | TODO |
|
||||
| Grafana → data sources | TLS с проверкой сервера, ограниченная identity | Отрицательные проверки доступа | TODO |
|
||||
| API/worker/agent → OpenBao | TLS + сервисный auth + scoped policy | Renewal/expiry/revocation | TODO |
|
||||
| Оператор → OpenBao/Grafana/S3 admin | TLS + authentication, доступ локальный/через защищённый канал | Разделение operator/application roles | TODO |
|
||||
| API → Redis/Valkey, если используется | TLS + ACL/пароль с минимальными правами | Ротация; mTLS по возможности | TODO |
|
||||
| Replication/кластерные каналы, если добавлены | TLS/mTLS либо подтверждённый шифрованный transport | Проверка заявленной модели отказа | TODO |
|
||||
| Связь | TLS, identity и права | Проверка |
|
||||
| --- | --- | --- |
|
||||
| Клиент → HAProxy | HTTPS `localhost:8443` с проверкой CA и SAN. Bearer-сессия по умолчанию 3600 с. | `make test test-security CA_FILE=secrets/tls/lab7-ingress-ca.crt`, `scripts/check_grade3_http.py`, `scripts/check_login_rate.py` и `scripts/check_session_expiry.py`. |
|
||||
| HAProxy → API | mTLS `api:8080`. HAProxy проверяет CA и `api`. API проверяет clientAuth и URI SAN `urn:lab7:load-balancer`. Порт API не опубликован. | `python3 scripts/check_ingress_mtls.py` проверяет разрешённый LB и три отказа на каждой реплике. |
|
||||
| API, worker, dispatcher, cleanup и migrate → PostgreSQL | `db:5432`, `sslmode=verify-full`, hostssl + SCRAM. Владелец запускает миграции, остальные используют отдельные login-роли и права на таблицы и последовательности из `infra/postgres-roles.sql`. | TLS probe с правильным и чужим CA, HTTPS smoke проверяет работу ролей API и worker. |
|
||||
| API, worker и cleanup → S3 | `https://s3:9000`, приватный bucket. У bootstrap root. API принимает staging и читает опубликованные файлы, worker читает staging и публикует оригиналы и варианты, cleanup удаляет по разрешённым префиксам. Права находятся в `s3_admin.go`. Direct upload использует подписанную форму через локальный `https://localhost:9000`. | TLS probe и smoke с побайтной сверкой оригинала и thumbnail. |
|
||||
| Dispatcher и worker → RabbitMQ | AMQPS `broker:5671`. Сервисные сертификаты от broker-client CA, RabbitMQ сопоставляет CN с учётной записью через SASL EXTERNAL. Dispatcher публикует через default exchange, worker читает `attachments.v1`. Обе роли могут объявлять только эту durable очередь. API пишет намерение в БД. | TLS probe, smoke queued → ready и `scripts/check_grade4_broker.py`: без cert и с чужим CA отказ TLS, незарегистрированная identity своего CA отклоняется при входе, обратные операции ролей запрещены. `infra/rabbitmq/bootstrap.sh` задаёт ACL. |
|
||||
| API, worker и dispatcher → OTel Collector | HTTPS `otel-collector:4318`. Collector закрепляет три конкретных leaf API, worker и dispatcher. Остальные leaf того же CA отклоняются. | TLS probe и проверка доставки свежих сигналов. |
|
||||
| Collector → Loki/Tempo, Prometheus → Collector | mTLS `loki:3100/otlp`, `tempo:4318`, scrape `otel-collector:8889`. Клиенты имеют сертификаты отдельного export CA. | `scripts/check_grade3_tls.py` и `scripts/check_grade3_signals.py`. |
|
||||
| Grafana → источники | mTLS `prometheus:9090`, `loki:3100`, `tempo:3200`. Grafana имеет ключ reader CA. Loki принимает reader и exporter CA на общем HTTP listener, поэтому граница контроля там ограничена этими двумя группами. | TLS probe с reader cert и запросы к каждому хранилищу в `scripts/check_grade3_signals.py`. |
|
||||
| API, worker, dispatcher, cleanup → OpenBao | `https://openbao:8200`. Cert auth закрепляет отдельный leaf каждой роли. Токен до 900 с читает только `messenger/data/<role>`. Сервисы получают credentials при запуске и держат их на tmpfs. | `python3 scripts/check_bao_roles.py`: свой путь разрешён, чужое чтение, чужой вход и запись запрещены. |
|
||||
| Оператор → OpenBao, Grafana и S3 | Локальные HTTPS-порты 8200, 3000, 9000. OpenBao — операторский токен, Grafana — login, MinIO — отдельные root credentials для bootstrap. | Проверка публикаций контейнеров и TLS. |
|
||||
| API → Redis | `rediss://redis:6379`, имя сервера проверяется. Отдельный пользователь messenger ограничен ключами `messenger:*` и операциями сессий и счётчиков. Plaintext listener отключён. | TLS probe, smoke, проверка expiry и общего ограничения входа. |
|
||||
| Replication/кластерные каналы | Не применимо. Реплики API не обмениваются данными напрямую, хранилища одиночные. | Проверка восстановления после остановки API. |
|
||||
|
||||
Нет выбранного компонента — отметьте N/A с причиной, а не оставляйте пустую строку. Внутренний публичный health listener, если нужен, минимален и изолирован; через него нельзя обращаться к бизнес-API без mTLS. Отдельно задокументируйте scrape/health/bootstrap каналы: они тоже не должны создавать обход.
|
||||
## Health, scrape и bootstrap
|
||||
|
||||
## Обязательные отрицательные проверки
|
||||
HAProxy проверяет `GET /health/ready` каждой реплики через её mTLS listener. Скрипт `scripts/wait_ingress.sh` проверяет готовность через HTTPS с доверенным CA. Служебные проверки Postgres и RabbitMQ запускаются внутри контейнеров, Redis проверяется локальным CLI, MinIO — HTTPS-запросом с проверкой CA. Redis healthcheck также использует TLS и ACL. Связь Prometheus с Collector учитывается отдельно в строке про scrape.
|
||||
|
||||
1. HTTPS с правильным CA/hostname проходит; с пустым/чужим CA и неправильным hostname — ошибка верификации, не «сервер недоступен».
|
||||
2. LB→API: без client cert, с cert чужого CA, с запрещённой identity от доверенного CA — отказ; с разрешённой LB identity — успех.
|
||||
3. На 4 повторите набор для broker и Collector; ограничение на network policy не заменяет проверку identity.
|
||||
4. Пользователь вне чата не читает metadata/content и не прикрепляет чужой файл; поддельный X-User-Id не меняет Bearer identity.
|
||||
5. OpenBao app policy читает только свой secret/path; запись policy и чтение чужого path запрещены. Worker не получает admin credentials API/БД/S3.
|
||||
6. На 4 отзовите secret/cert и дождитесь контролируемого эффекта с учётом действующих соединений, TLS resumption и cache/TTL. Покажите новое подключение, иначе тест отзыва может ничего не проверить.
|
||||
|
||||
## Bootstrap и эксплуатация
|
||||
|
||||
В репозитории только открытые конфиги, policy templates и scripts. CA private key, leaf keys, root token, unseal shares и SecretID не коммитятся, не запекаются в образ и не печатаются в отчёте. Для локального стенда оператор создаёт их вне Git и передаёт через files/secrets с ограниченными правами. Опишите, кто имеет доступ к CA/unseal, что нужно после рестарта OpenBao и как восстановить работу.
|
||||
|
||||
OpenBao не использует dev-mode. Выберите persistent storage и auth метод (например AppRole с короткоживущим SecretID/response wrapping, cert auth или иная обоснованная identity). Не подменяйте root token постоянной переменной BAO_TOKEN в приложении. Рабочие tokens имеют policy, TTL и renewal/re-auth сценарий. Статические KV secrets на 3 допустимы при реальном чтении через сервисную identity; динамические DB credentials — хорошее расширение, но не обязательное название технологии.
|
||||
|
||||
Разрешённый компромисс учебного стенда: ручной operator init/unseal, если он безопасно документирован и не нужен при каждом обычном рестарте API. Автоматическое хранение всех unseal shares рядом с OpenBao без защиты не считается решённым bootstrap.
|
||||
Bootstrap миграций и S3 выполняется одноразовыми контейнерами. Оператор запускает их с учётными данными владельцев и проверкой TLS. Прикладные контейнеры монтируют свои ключи cert auth и получают ограниченные credentials из OpenBao. Закрытые ключи, unseal shares и административные токены остаются у оператора вне Git и образов. `scripts/check_grade4_lifecycle.py` показывает замену leaf OpenBao и пароля PostgreSQL на новых соединениях. При рестарте сервис заново аутентифицируется по зарегистрированному leaf. Оператор выпускает новый leaf до истечения срока и обновляет доверие в выбранном получателе.
|
||||
|
||||
@@ -0,0 +1,83 @@
|
||||
# Стоимость проверки хеша Argon2id на лимитах API
|
||||
|
||||
Предпосылки: подготовленный, разблокированный учебный стенд; Docker Compose, Python и локальный CA.
|
||||
|
||||
`docker compose --profile test build backend-tests`
|
||||
|
||||
Код завершения: 0.
|
||||
|
||||
```text
|
||||
Image highload-lab7-backend-tests Building
|
||||
#1 [internal] load local bake definitions
|
||||
#1 reading from stdin 1.66kB done
|
||||
#1 DONE 0.0s
|
||||
|
||||
#2 [internal] load build definition from Dockerfile
|
||||
#2 transferring dockerfile: 816B done
|
||||
#2 DONE 0.0s
|
||||
|
||||
#3 [internal] load metadata for docker.io/library/golang:1.22.2-bookworm
|
||||
#3 DONE 1.1s
|
||||
|
||||
#4 [internal] load .dockerignore
|
||||
#4 transferring context: 327B done
|
||||
#4 DONE 0.0s
|
||||
|
||||
#5 [builder 1/7] FROM docker.io/library/golang:1.22.2-bookworm@sha256:d0902bacefdde1cf45528c098d14e55d78c107def8a22d148eabd71582d7a99f
|
||||
#5 resolve docker.io/library/golang:1.22.2-bookworm@sha256:d0902bacefdde1cf45528c098d14e55d78c107def8a22d148eabd71582d7a99f 0.0s done
|
||||
#5 DONE 0.0s
|
||||
|
||||
#6 [internal] load build context
|
||||
#6 transferring context: 13.70kB done
|
||||
#6 DONE 0.0s
|
||||
|
||||
#7 [builder 4/7] COPY go.mod go.sum ./
|
||||
#7 CACHED
|
||||
|
||||
#8 [builder 2/7] RUN apt-get update && apt-get install -y --no-install-recommends webp && rm -rf /var/lib/apt/lists/*
|
||||
#8 CACHED
|
||||
|
||||
#9 [builder 5/7] RUN go mod download
|
||||
#9 CACHED
|
||||
|
||||
#10 [builder 6/7] COPY backend ./backend
|
||||
#10 CACHED
|
||||
|
||||
#11 [builder 3/7] WORKDIR /src
|
||||
#11 CACHED
|
||||
|
||||
#12 [builder 7/7] RUN CGO_ENABLED=0 GOOS=linux go build -trimpath -ldflags="-s -w" -o /out/messenger ./backend
|
||||
#12 CACHED
|
||||
|
||||
#13 exporting to image
|
||||
#13 exporting layers
|
||||
#13 exporting layers 6.7s done
|
||||
#13 exporting manifest sha256:cc55619214f59b9b27ad25a2e23d1abce9f64566439be3b766526869522398f8 done
|
||||
#13 exporting config sha256:60158d0274ef38fabce1b1b41aef1dce15861f92bfe46cdd75183cc143df20f5 done
|
||||
#13 exporting attestation manifest sha256:facefdb8b90c9ec469b34788e90e8b0d663bf8c722e0e92d10cbc231bc2e6bc2 done
|
||||
#13 exporting manifest list sha256:a0ef01b3efc503922fd45a76b39ec30f5d34f2085c9a228eeb3277e39df87eab done
|
||||
#13 naming to docker.io/library/highload-lab7-backend-tests:latest done
|
||||
#13 unpacking to docker.io/library/highload-lab7-backend-tests:latest
|
||||
#13 unpacking to docker.io/library/highload-lab7-backend-tests:latest 1.2s done
|
||||
#13 DONE 7.9s
|
||||
|
||||
#14 resolving provenance for metadata file
|
||||
#14 DONE 0.0s
|
||||
Image highload-lab7-backend-tests Built
|
||||
```
|
||||
|
||||
`docker run --rm --network none --cpus 0.5 --memory 512m -e GOMAXPROCS=1 highload-lab7-backend-tests go test ./backend/services -run '^$' -bench '^BenchmarkPasswordVerification$' -benchtime=3x -count=3`
|
||||
|
||||
Код завершения: 0.
|
||||
|
||||
```text
|
||||
goos: linux
|
||||
goarch: amd64
|
||||
pkg: messenger/src/backend/services
|
||||
cpu: AMD Ryzen 9 PRO 7940HS w/ Radeon 780M Graphics
|
||||
BenchmarkPasswordVerification 3 194744721 ns/op 67115832 B/op 74 allocs/op
|
||||
BenchmarkPasswordVerification 3 176595711 ns/op 67115832 B/op 74 allocs/op
|
||||
BenchmarkPasswordVerification 3 196042348 ns/op 67115832 B/op 74 allocs/op
|
||||
PASS
|
||||
ok messenger/src/backend/services 3.606s
|
||||
```
|
||||
@@ -0,0 +1,35 @@
|
||||
# TLS хранилищ, брокера и телеметрии
|
||||
|
||||
Предпосылки: подготовленный, разблокированный учебный стенд; Docker Compose, Python и локальный CA.
|
||||
|
||||
`python3 scripts/check_grade3_tls.py`
|
||||
|
||||
Код завершения: 0.
|
||||
|
||||
```text
|
||||
PASS s3:9000: verified service SAN; foreign CA rejected
|
||||
PASS broker:5671: verified service SAN; foreign CA rejected; foreign client identity rejected
|
||||
PASS redis:6379: verified service SAN; foreign CA rejected
|
||||
PASS db:5432: verified service SAN; foreign CA rejected
|
||||
PASS otel-collector:4318: verified service SAN; foreign CA rejected; foreign client identity rejected
|
||||
PASS otel-collector:8889: verified service SAN; foreign CA rejected; foreign client identity rejected
|
||||
PASS prometheus:9090: verified service SAN; foreign CA rejected; foreign client identity rejected
|
||||
PASS loki:3100: verified service SAN; foreign CA rejected; foreign client identity rejected
|
||||
PASS loki:3100: verified service SAN; foreign CA rejected; foreign client identity rejected
|
||||
PASS tempo:4318: verified service SAN; foreign CA rejected; foreign client identity rejected
|
||||
PASS tempo:3200: verified service SAN; foreign CA rejected; foreign client identity rejected
|
||||
PASS grafana:3000: verified service SAN; foreign CA rejected
|
||||
PASS openbao:8200: verified service SAN; foreign CA rejected
|
||||
```
|
||||
|
||||
`./scripts/check_dependency_roles.sh`
|
||||
|
||||
Код завершения: 0.
|
||||
|
||||
```text
|
||||
PASS PostgreSQL: worker can update attachments; user table and schema creation denied
|
||||
PASS RabbitMQ: producer can publish and consumer can read attachments.v1
|
||||
PASS Redis: application role cannot read server configuration
|
||||
```
|
||||
|
||||
Итог: все команды завершились успешно.
|
||||
@@ -0,0 +1,67 @@
|
||||
# Истечение сессии на стенде с коротким TTL
|
||||
|
||||
Предпосылки: подготовленный, разблокированный учебный стенд; Docker Compose, Python и локальный CA.
|
||||
|
||||
`sleep 60`
|
||||
|
||||
Код завершения: 0.
|
||||
|
||||
```text
|
||||
|
||||
```
|
||||
|
||||
`env SESSION_TTL_SECONDS=5 docker compose up -d --no-deps api`
|
||||
|
||||
Код завершения: 0.
|
||||
|
||||
```text
|
||||
Container highload-lab7-api-1 Recreate
|
||||
Container highload-lab7-api-2 Recreate
|
||||
Container highload-lab7-api-1 Recreated
|
||||
Container highload-lab7-api-2 Recreated
|
||||
Container highload-lab7-api-1 Starting
|
||||
Container highload-lab7-api-1 Started
|
||||
Container highload-lab7-api-2 Starting
|
||||
Container highload-lab7-api-2 Started
|
||||
```
|
||||
|
||||
`./scripts/wait_ingress.sh`
|
||||
|
||||
Код завершения: 0.
|
||||
|
||||
```text
|
||||
|
||||
```
|
||||
|
||||
`python3 scripts/check_session_expiry.py --ca-file secrets/tls/lab7-ca.crt`
|
||||
|
||||
Код завершения: 0.
|
||||
|
||||
```text
|
||||
PASS: valid session accepted before expiry and rejected after expiry
|
||||
```
|
||||
|
||||
`docker compose up -d --no-deps api`
|
||||
|
||||
Код завершения: 0.
|
||||
|
||||
```text
|
||||
Container highload-lab7-api-2 Recreate
|
||||
Container highload-lab7-api-1 Recreate
|
||||
Container highload-lab7-api-2 Recreated
|
||||
Container highload-lab7-api-1 Recreated
|
||||
Container highload-lab7-api-1 Starting
|
||||
Container highload-lab7-api-1 Started
|
||||
Container highload-lab7-api-2 Starting
|
||||
Container highload-lab7-api-2 Started
|
||||
```
|
||||
|
||||
`./scripts/wait_ingress.sh`
|
||||
|
||||
Код завершения: 0.
|
||||
|
||||
```text
|
||||
|
||||
```
|
||||
|
||||
Итог: все команды завершились успешно.
|
||||
@@ -0,0 +1,13 @@
|
||||
# Лимиты, MIME, чужие вложения и неверный TLS hostname
|
||||
|
||||
Предпосылки: подготовленный, разблокированный учебный стенд; Docker Compose, Python и локальный CA.
|
||||
|
||||
`python3 scripts/check_grade3_http.py --ca-file secrets/tls/lab7-ca.crt`
|
||||
|
||||
Код завершения: 0.
|
||||
|
||||
```text
|
||||
PASS: upload size/MIME, forged identity, cross-chat attachment, wrong TLS hostname
|
||||
```
|
||||
|
||||
Итог: все команды завершились успешно.
|
||||
@@ -0,0 +1,25 @@
|
||||
# Бизнес-API, проверка CA, logout и вложения по HTTPS
|
||||
|
||||
Предпосылки: подготовленный, разблокированный учебный стенд; Docker Compose, Python и локальный CA.
|
||||
|
||||
`make check`
|
||||
|
||||
Код завершения: 0.
|
||||
|
||||
```text
|
||||
python3 scripts/check_contract.py
|
||||
PASS: offline skeleton structure, local refs and Python syntax; application NOT tested.
|
||||
```
|
||||
|
||||
`make test test-security CA_FILE=secrets/tls/lab7-ingress-ca.crt`
|
||||
|
||||
Код завершения: 0.
|
||||
|
||||
```text
|
||||
python3 tests/smoke.py --base-url "https://localhost:8443" --ca-file "secrets/tls/lab7-ingress-ca.crt"
|
||||
PASS: lab7 public API scenario; infrastructure and grading evidence remain separate.
|
||||
python3 tests/security.py --base-url "https://localhost:8443" --ca-file "secrets/tls/lab7-ingress-ca.crt"
|
||||
PASS: trusted HTTPS works; untrusted CA rejected. This does NOT verify internal mTLS, OpenBao or hardening.
|
||||
```
|
||||
|
||||
Итог: все команды завершились успешно.
|
||||
@@ -0,0 +1,21 @@
|
||||
# Общий лимит входа по адресу клиента
|
||||
|
||||
Предпосылки: подготовленный, разблокированный учебный стенд; Docker Compose, Python и локальный CA.
|
||||
|
||||
`sleep 60`
|
||||
|
||||
Код завершения: 0.
|
||||
|
||||
```text
|
||||
|
||||
```
|
||||
|
||||
`python3 scripts/check_login_rate.py`
|
||||
|
||||
Код завершения: 0.
|
||||
|
||||
```text
|
||||
PASS: request 13 returned 429 with Retry-After across 2 API replicas
|
||||
```
|
||||
|
||||
Итог: все команды завершились успешно.
|
||||
@@ -0,0 +1,14 @@
|
||||
# Допуск только роли LB на каждой реплике
|
||||
|
||||
Предпосылки: подготовленный, разблокированный учебный стенд; Docker Compose, Python и локальный CA.
|
||||
|
||||
`python3 scripts/check_ingress_mtls.py`
|
||||
|
||||
Код завершения: 0.
|
||||
|
||||
```text
|
||||
PASS b99988e01dc1: LB permitted; absent, foreign CA and disallowed SAN rejected
|
||||
PASS 10ef2f17b12c: LB permitted; absent, foreign CA and disallowed SAN rejected
|
||||
```
|
||||
|
||||
Итог: все команды завершились успешно.
|
||||
@@ -0,0 +1,16 @@
|
||||
# Сервисная аутентификация и scoped policy OpenBao
|
||||
|
||||
Предпосылки: подготовленный, разблокированный учебный стенд; Docker Compose, Python и локальный CA.
|
||||
|
||||
`python3 scripts/check_bao_roles.py`
|
||||
|
||||
Код завершения: 0.
|
||||
|
||||
```text
|
||||
PASS api: scoped read, cross-role login/read and write denied, TTL <= 900 s
|
||||
PASS worker: scoped read, cross-role login/read and write denied, TTL <= 900 s
|
||||
PASS dispatcher: scoped read, cross-role login/read and write denied, TTL <= 900 s
|
||||
PASS cleanup: scoped read, cross-role login/read and write denied, TTL <= 900 s
|
||||
```
|
||||
|
||||
Итог: все команды завершились успешно.
|
||||
@@ -0,0 +1,47 @@
|
||||
# Работа HTTPS API при остановленной второй реплике
|
||||
|
||||
Предпосылки: подготовленный, разблокированный учебный стенд; Docker Compose, Python и локальный CA.
|
||||
|
||||
`sleep 60`
|
||||
|
||||
Код завершения: 0.
|
||||
|
||||
```text
|
||||
|
||||
```
|
||||
|
||||
`./scripts/check_other_replica.sh`
|
||||
|
||||
Код завершения: 0.
|
||||
|
||||
```text
|
||||
Stopped API instance 3b3f131fd602
|
||||
python3 tests/smoke.py --base-url "https://localhost:8443" --ca-file "secrets/tls/lab7-ca.crt"
|
||||
PASS: lab7 public API scenario; infrastructure and grading evidence remain separate.
|
||||
Container highload-lab7-redis-1 Waiting
|
||||
Container highload-lab7-db-1 Waiting
|
||||
Container highload-lab7-s3-1 Waiting
|
||||
Container highload-lab7-db-1 Healthy
|
||||
Container highload-lab7-s3-1 Healthy
|
||||
Container highload-lab7-redis-1 Healthy
|
||||
Container highload-lab7-migrate-1 Starting
|
||||
Container highload-lab7-s3-bootstrap-1 Starting
|
||||
Container highload-lab7-migrate-1 Started
|
||||
Container highload-lab7-migrate-1 Waiting
|
||||
Container highload-lab7-s3-bootstrap-1 Started
|
||||
Container highload-lab7-s3-bootstrap-1 Waiting
|
||||
Container highload-lab7-migrate-1 Exited
|
||||
Container highload-lab7-s3-bootstrap-1 Exited
|
||||
Container highload-lab7-api-2 Starting
|
||||
Container highload-lab7-api-2 Started
|
||||
```
|
||||
|
||||
`./scripts/wait_ingress.sh`
|
||||
|
||||
Код завершения: 0.
|
||||
|
||||
```text
|
||||
|
||||
```
|
||||
|
||||
Итог: все команды завершились успешно.
|
||||
@@ -0,0 +1,139 @@
|
||||
# Сохранность после перезапуска хранилищ и отказ API
|
||||
|
||||
Предпосылки: подготовленный, разблокированный учебный стенд; Docker Compose, Python и локальный CA.
|
||||
|
||||
`sleep 60`
|
||||
|
||||
Код завершения: 0.
|
||||
|
||||
```text
|
||||
|
||||
```
|
||||
|
||||
`make test-persistence ACTION_SCRIPT=scripts/restart.sh CA_FILE=secrets/tls/lab7-ca.crt`
|
||||
|
||||
Код завершения: 0.
|
||||
|
||||
```text
|
||||
python3 tests/resilience.py --mode persistence --action-script "scripts/restart.sh" --base-url "https://localhost:8443" --ca-file "secrets/tls/lab7-ca.crt"
|
||||
Executing explicit persistence hook; hook output suppressed to avoid leaking secrets.
|
||||
PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence.
|
||||
```
|
||||
|
||||
`make test-failover ACTION_SCRIPT=scripts/stop-one.sh CA_FILE=secrets/tls/lab7-ca.crt`
|
||||
|
||||
Код завершения: 0.
|
||||
|
||||
```text
|
||||
python3 tests/resilience.py --mode failover --action-script "scripts/stop-one.sh" --base-url "https://localhost:8443" --ca-file "secrets/tls/lab7-ca.crt"
|
||||
Executing explicit failover hook; hook output suppressed to avoid leaking secrets.
|
||||
{"observed_instances_before": 2, "successful_writes": 124, "failed_attempts": 1, "first_recovery_seconds": 1.454}
|
||||
PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.
|
||||
```
|
||||
|
||||
`docker compose start api`
|
||||
|
||||
Код завершения: 0.
|
||||
|
||||
```text
|
||||
Container highload-lab7-db-1 Waiting
|
||||
Container highload-lab7-s3-1 Waiting
|
||||
Container highload-lab7-redis-1 Waiting
|
||||
Container highload-lab7-s3-1 Healthy
|
||||
Container highload-lab7-redis-1 Healthy
|
||||
Container highload-lab7-db-1 Healthy
|
||||
Container highload-lab7-s3-bootstrap-1 Starting
|
||||
Container highload-lab7-migrate-1 Starting
|
||||
Container highload-lab7-s3-bootstrap-1 Started
|
||||
Container highload-lab7-s3-bootstrap-1 Waiting
|
||||
Container highload-lab7-migrate-1 Started
|
||||
Container highload-lab7-migrate-1 Waiting
|
||||
Container highload-lab7-migrate-1 Exited
|
||||
Container highload-lab7-s3-bootstrap-1 Exited
|
||||
Container highload-lab7-api-1 Starting
|
||||
Container highload-lab7-api-1 Started
|
||||
```
|
||||
|
||||
`./scripts/wait_ingress.sh`
|
||||
|
||||
Код завершения: 0.
|
||||
|
||||
```text
|
||||
|
||||
```
|
||||
|
||||
`sleep 60`
|
||||
|
||||
Код завершения: 0.
|
||||
|
||||
```text
|
||||
|
||||
```
|
||||
|
||||
`make test-failover ACTION_SCRIPT=scripts/stop-one.sh CA_FILE=secrets/tls/lab7-ca.crt`
|
||||
|
||||
Код завершения: 0.
|
||||
|
||||
```text
|
||||
python3 tests/resilience.py --mode failover --action-script "scripts/stop-one.sh" --base-url "https://localhost:8443" --ca-file "secrets/tls/lab7-ca.crt"
|
||||
Executing explicit failover hook; hook output suppressed to avoid leaking secrets.
|
||||
{"observed_instances_before": 2, "successful_writes": 122, "failed_attempts": 2, "first_recovery_seconds": 1.58}
|
||||
PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.
|
||||
```
|
||||
|
||||
`docker compose start api`
|
||||
|
||||
Код завершения: 0.
|
||||
|
||||
```text
|
||||
Container highload-lab7-redis-1 Waiting
|
||||
Container highload-lab7-s3-1 Waiting
|
||||
Container highload-lab7-db-1 Waiting
|
||||
Container highload-lab7-s3-1 Healthy
|
||||
Container highload-lab7-s3-bootstrap-1 Starting
|
||||
Container highload-lab7-redis-1 Healthy
|
||||
Container highload-lab7-db-1 Healthy
|
||||
Container highload-lab7-migrate-1 Starting
|
||||
Container highload-lab7-s3-bootstrap-1 Started
|
||||
Container highload-lab7-s3-bootstrap-1 Waiting
|
||||
Container highload-lab7-migrate-1 Started
|
||||
Container highload-lab7-migrate-1 Waiting
|
||||
Container highload-lab7-migrate-1 Exited
|
||||
Container highload-lab7-s3-bootstrap-1 Exited
|
||||
Container highload-lab7-api-1 Starting
|
||||
Container highload-lab7-api-1 Started
|
||||
```
|
||||
|
||||
`./scripts/wait_ingress.sh`
|
||||
|
||||
Код завершения: 0.
|
||||
|
||||
```text
|
||||
|
||||
```
|
||||
|
||||
Состояния контейнеров (restart):
|
||||
|
||||
```text
|
||||
/highload-lab7-api-1 2026-11-14T17:24:55.651185572Z
|
||||
/highload-lab7-api-2 2026-11-14T17:24:55.819757058Z
|
||||
/highload-lab7-broker-1 2026-11-14T17:23:53.398757712Z
|
||||
/highload-lab7-db-1 2026-11-14T17:23:50.651938319Z
|
||||
/highload-lab7-redis-1 2026-11-14T17:23:50.615434358Z
|
||||
/highload-lab7-s3-1 2026-11-14T17:23:50.777141988Z
|
||||
/highload-lab7-api-1 2026-11-14T17:26:42.238931046Z
|
||||
/highload-lab7-api-2 2026-11-14T17:26:42.256578225Z
|
||||
/highload-lab7-broker-1 2026-11-14T17:26:34.665663438Z
|
||||
/highload-lab7-db-1 2026-11-14T17:26:33.677826831Z
|
||||
/highload-lab7-redis-1 2026-11-14T17:26:33.695777808Z
|
||||
/highload-lab7-s3-1 2026-11-14T17:26:33.780920926Z
|
||||
```
|
||||
|
||||
Состояния контейнеров (failover):
|
||||
|
||||
```text
|
||||
/highload-lab7-api-1 exited
|
||||
/highload-lab7-api-1 exited
|
||||
```
|
||||
|
||||
Итог: все команды завершились успешно.
|
||||
@@ -0,0 +1,13 @@
|
||||
# Изоляция, порты, лимиты и локальное размещение секретов
|
||||
|
||||
Предпосылки: подготовленный, разблокированный учебный стенд; Docker Compose, Python и локальный CA.
|
||||
|
||||
`python3 scripts/check_grade3_runtime.py`
|
||||
|
||||
Код завершения: 0.
|
||||
|
||||
```text
|
||||
PASS: local-only ports, runtime secrets on tmpfs, non-root app roles with limits and dropped capabilities
|
||||
```
|
||||
|
||||
Итог: все команды завершились успешно.
|
||||
@@ -0,0 +1,13 @@
|
||||
# Доставка логов, метрик и трейсов по защищённым связям
|
||||
|
||||
Предпосылки: подготовленный, разблокированный учебный стенд; Docker Compose, Python и локальный CA.
|
||||
|
||||
`python3 scripts/check_grade3_signals.py`
|
||||
|
||||
Код завершения: 0.
|
||||
|
||||
```text
|
||||
PASS: Prometheus scrape up; fresh messenger API log and trace found through mTLS
|
||||
```
|
||||
|
||||
Итог: все команды завершились успешно.
|
||||
@@ -0,0 +1,83 @@
|
||||
# Сетевой аудит, ACL и отсутствие контрольного секрета в телеметрии
|
||||
|
||||
Предпосылки: подготовленный, разблокированный учебный стенд; Docker Compose, Python и локальный CA.
|
||||
|
||||
`python3 scripts/check_grade3_tls.py`
|
||||
|
||||
Код завершения: 0.
|
||||
|
||||
```text
|
||||
PASS s3:9000: verified service SAN; foreign CA rejected
|
||||
PASS broker:5671: verified service SAN; foreign CA rejected; foreign client identity rejected
|
||||
PASS redis:6379: verified service SAN; foreign CA rejected
|
||||
PASS db:5432: verified service SAN; foreign CA rejected
|
||||
PASS Collector: missing client certificate rejected
|
||||
PASS otel-collector:4318: verified service SAN; foreign CA rejected; foreign client identity rejected
|
||||
PASS otel-collector:8889: verified service SAN; foreign CA rejected; foreign client identity rejected
|
||||
PASS prometheus:9090: verified service SAN; foreign CA rejected; foreign client identity rejected
|
||||
PASS loki:3100: verified service SAN; foreign CA rejected; foreign client identity rejected
|
||||
PASS loki:3100: verified service SAN; foreign CA rejected; foreign client identity rejected
|
||||
PASS tempo:4318: verified service SAN; foreign CA rejected; foreign client identity rejected
|
||||
PASS tempo:3200: verified service SAN; foreign CA rejected; foreign client identity rejected
|
||||
PASS grafana:3000: verified service SAN; foreign CA rejected
|
||||
PASS openbao:8200: verified service SAN; foreign CA rejected
|
||||
```
|
||||
|
||||
`python3 scripts/check_grade4_broker.py`
|
||||
|
||||
Код завершения: 0.
|
||||
|
||||
```text
|
||||
PASS: no certificate
|
||||
PASS: wrong-role
|
||||
PASS: messenger_unregistered
|
||||
PASS: messenger_worker
|
||||
PASS: messenger_dispatcher
|
||||
```
|
||||
|
||||
`./scripts/check_dependency_roles.sh`
|
||||
|
||||
Код завершения: 0.
|
||||
|
||||
```text
|
||||
PASS PostgreSQL: worker can update attachments; user table and schema creation denied
|
||||
PASS RabbitMQ: producer can publish and consumer can read attachments.v1
|
||||
PASS Redis: application role cannot read server configuration
|
||||
```
|
||||
|
||||
`python3 scripts/check_bao_roles.py`
|
||||
|
||||
Код завершения: 0.
|
||||
|
||||
```text
|
||||
PASS api: scoped read, cross-role login/read and write denied, TTL <= 900 s
|
||||
PASS worker: scoped read, cross-role login/read and write denied, TTL <= 900 s
|
||||
PASS dispatcher: scoped read, cross-role login/read and write denied, TTL <= 900 s
|
||||
PASS cleanup: scoped read, cross-role login/read and write denied, TTL <= 900 s
|
||||
```
|
||||
|
||||
`sleep 60`
|
||||
|
||||
Код завершения: 0.
|
||||
|
||||
```text
|
||||
|
||||
```
|
||||
|
||||
`python3 scripts/check_grade3_http.py --ca-file secrets/tls/lab7-ca.crt`
|
||||
|
||||
Код завершения: 0.
|
||||
|
||||
```text
|
||||
PASS: upload size/MIME, forged identity, cross-chat attachment, wrong TLS hostname
|
||||
```
|
||||
|
||||
`python3 scripts/check_grade4_telemetry.py`
|
||||
|
||||
Код завершения: 0.
|
||||
|
||||
```text
|
||||
PASS: synthetic Basic secret absent from correlated log, correlated trace, metric labels and container logs
|
||||
```
|
||||
|
||||
Итог: все команды завершились успешно.
|
||||
@@ -0,0 +1,13 @@
|
||||
# Права контейнеров, лимиты и эксплуатационные границы
|
||||
|
||||
Предпосылки: подготовленный, разблокированный учебный стенд; Docker Compose, Python и локальный CA.
|
||||
|
||||
`python3 scripts/check_grade3_runtime.py`
|
||||
|
||||
Код завершения: 0.
|
||||
|
||||
```text
|
||||
PASS: local ports, read-only app/ingress/telemetry, no-new-privileges, dropped capabilities, worker CPU/RAM/time budget
|
||||
```
|
||||
|
||||
Итог: все команды завершились успешно.
|
||||
@@ -0,0 +1,28 @@
|
||||
# Ротация секрета и leaf, отзыв, expiry и seal
|
||||
|
||||
Предпосылки: подготовленный, разблокированный учебный стенд; Docker Compose, Python и локальный CA.
|
||||
|
||||
`python3 scripts/check_grade4_lifecycle.py`
|
||||
|
||||
Код завершения: 0.
|
||||
|
||||
```text
|
||||
Container highload-lab7-api-2 Recreate
|
||||
Container highload-lab7-api-1 Recreate
|
||||
Container highload-lab7-api-1 Recreated
|
||||
Container highload-lab7-api-2 Recreated
|
||||
Container highload-lab7-api-1 Starting
|
||||
Container highload-lab7-api-1 Started
|
||||
Container highload-lab7-api-2 Starting
|
||||
Container highload-lab7-api-2 Started
|
||||
OpenBao unsealed
|
||||
PASS api: scoped read, cross-role login/read and write denied, TTL <= 900 s
|
||||
PASS worker: scoped read, cross-role login/read and write denied, TTL <= 900 s
|
||||
PASS dispatcher: scoped read, cross-role login/read and write denied, TTL <= 900 s
|
||||
PASS cleanup: scoped read, cross-role login/read and write denied, TTL <= 900 s
|
||||
PASS: replaced leaf, revoked old token, expired short-lived token
|
||||
PASS: database secret rotated, old password refused, API ready without rebuilding
|
||||
PASS: sealed OpenBao denies new login; running API remains ready using loaded credentials
|
||||
```
|
||||
|
||||
Итог: все команды завершились успешно.
|
||||
@@ -0,0 +1,86 @@
|
||||
# Rehash в PostgreSQL и бюджет защиты входа
|
||||
|
||||
Предпосылки: подготовленный, разблокированный учебный стенд; Docker Compose, Python и локальный CA.
|
||||
|
||||
`docker compose --profile test build backend-tests`
|
||||
|
||||
Код завершения: 0.
|
||||
|
||||
```text
|
||||
Image highload-lab7-backend-tests Building
|
||||
#1 [internal] load local bake definitions
|
||||
#1 reading from stdin 1.66kB done
|
||||
#1 DONE 0.0s
|
||||
|
||||
#2 [internal] load build definition from Dockerfile
|
||||
#2 transferring dockerfile: 816B done
|
||||
#2 DONE 0.0s
|
||||
|
||||
#3 [internal] load metadata for docker.io/library/golang:1.22.2-bookworm
|
||||
#3 DONE 1.3s
|
||||
|
||||
#4 [internal] load .dockerignore
|
||||
#4 transferring context: 327B done
|
||||
#4 DONE 0.0s
|
||||
|
||||
#5 [builder 1/7] FROM docker.io/library/golang:1.22.2-bookworm@sha256:d0902bacefdde1cf45528c098d14e55d78c107def8a22d148eabd71582d7a99f
|
||||
#5 resolve docker.io/library/golang:1.22.2-bookworm@sha256:d0902bacefdde1cf45528c098d14e55d78c107def8a22d148eabd71582d7a99f 0.0s done
|
||||
#5 DONE 0.0s
|
||||
|
||||
#6 [internal] load build context
|
||||
#6 transferring context: 13.81kB done
|
||||
#6 DONE 0.0s
|
||||
|
||||
#7 [builder 2/7] RUN apt-get update && apt-get install -y --no-install-recommends webp && rm -rf /var/lib/apt/lists/*
|
||||
#7 CACHED
|
||||
|
||||
#8 [builder 3/7] WORKDIR /src
|
||||
#8 CACHED
|
||||
|
||||
#9 [builder 4/7] COPY go.mod go.sum ./
|
||||
#9 CACHED
|
||||
|
||||
#10 [builder 5/7] RUN go mod download
|
||||
#10 CACHED
|
||||
|
||||
#11 [builder 6/7] COPY backend ./backend
|
||||
#11 CACHED
|
||||
|
||||
#12 [builder 7/7] RUN CGO_ENABLED=0 GOOS=linux go build -trimpath -ldflags="-s -w" -o /out/messenger ./backend
|
||||
#12 CACHED
|
||||
|
||||
#13 exporting to image
|
||||
#13 exporting layers done
|
||||
#13 exporting manifest sha256:283fe5f0706359d473b23cc42eb6f2a6eb75a0d12b0f5b23a591907e982c8181 done
|
||||
#13 exporting config sha256:f1628c981eb4ff32d1b96c99c17b0dc4556a0a9fc01033b1ad0802e2428aa0d7 done
|
||||
#13 exporting attestation manifest sha256:1d394f0779fff35a4bdb3df53f1c276fbaa3ad3826c1620e226215638ac81a3a 0.0s done
|
||||
#13 exporting manifest list sha256:16c3bf9e4a23386b4f3bfb72d5b57fdf8f98ca796a286c299699a2727eeac5ad done
|
||||
#13 naming to docker.io/library/highload-lab7-backend-tests:latest done
|
||||
#13 unpacking to docker.io/library/highload-lab7-backend-tests:latest done
|
||||
#13 DONE 0.1s
|
||||
|
||||
#14 resolving provenance for metadata file
|
||||
#14 DONE 0.0s
|
||||
Image highload-lab7-backend-tests Built
|
||||
```
|
||||
|
||||
`sleep 60`
|
||||
|
||||
Код завершения: 0.
|
||||
|
||||
```text
|
||||
|
||||
```
|
||||
|
||||
`python3 scripts/check_grade4_login.py`
|
||||
|
||||
Код завершения: 0.
|
||||
|
||||
```text
|
||||
Container highload-lab7-backend-tests-run-fd0b3ea519f7 Creating
|
||||
Container highload-lab7-backend-tests-run-fd0b3ea519f7 Created
|
||||
PASS: successful login upgraded the stored legacy Argon2id hash
|
||||
{"requests": 24, "status": {"201": 8, "429": 16}, "p50_ms": 548, "max_ms": 1156}
|
||||
```
|
||||
|
||||
Итог: все команды завершились успешно.
|
||||
@@ -0,0 +1,46 @@
|
||||
# Проверка ролей RabbitMQ и Collector
|
||||
|
||||
Предпосылки: подготовленный, разблокированный учебный стенд; Docker Compose, Python и локальный CA.
|
||||
|
||||
`python3 scripts/check_grade4_broker.py`
|
||||
|
||||
Код завершения: 0.
|
||||
|
||||
```text
|
||||
PASS: no certificate
|
||||
PASS: wrong-role
|
||||
PASS: messenger_unregistered
|
||||
PASS: messenger_worker
|
||||
PASS: messenger_dispatcher
|
||||
```
|
||||
|
||||
`python3 scripts/check_grade3_tls.py`
|
||||
|
||||
Код завершения: 0.
|
||||
|
||||
```text
|
||||
PASS s3:9000: verified service SAN; foreign CA rejected
|
||||
PASS broker:5671: verified service SAN; foreign CA rejected; foreign client identity rejected
|
||||
PASS redis:6379: verified service SAN; foreign CA rejected
|
||||
PASS db:5432: verified service SAN; foreign CA rejected
|
||||
PASS Collector: missing client certificate rejected
|
||||
PASS otel-collector:4318: verified service SAN; foreign CA rejected; foreign client identity rejected
|
||||
PASS otel-collector:8889: verified service SAN; foreign CA rejected; foreign client identity rejected
|
||||
PASS prometheus:9090: verified service SAN; foreign CA rejected; foreign client identity rejected
|
||||
PASS loki:3100: verified service SAN; foreign CA rejected; foreign client identity rejected
|
||||
PASS loki:3100: verified service SAN; foreign CA rejected; foreign client identity rejected
|
||||
PASS tempo:4318: verified service SAN; foreign CA rejected; foreign client identity rejected
|
||||
PASS tempo:3200: verified service SAN; foreign CA rejected; foreign client identity rejected
|
||||
PASS grafana:3000: verified service SAN; foreign CA rejected
|
||||
PASS openbao:8200: verified service SAN; foreign CA rejected
|
||||
```
|
||||
|
||||
`python3 scripts/check_grade3_signals.py`
|
||||
|
||||
Код завершения: 0.
|
||||
|
||||
```text
|
||||
PASS: Prometheus scrape up; fresh messenger API log and trace found through mTLS
|
||||
```
|
||||
|
||||
Итог: все команды завершились успешно.
|
||||
@@ -0,0 +1,25 @@
|
||||
# Применимость результатов сканирования
|
||||
|
||||
Trivy 0.69.3 проверил `src/go.mod` и `src/frontend/package-lock.json`.
|
||||
Для Go найдены 94 совпадения. Для npm на момент проверки совпадений нет.
|
||||
Оценки ниже учитывают пути выполнения и границы доверия.
|
||||
|
||||
| Результат | Применимость и решение |
|
||||
| --- | --- |
|
||||
| CVE-2026-77405, amqp091-go 1.10.0 | Проблема связана с неявным минимумом протокола из AMQPS URI. Рабочий клиент явно задаёт TLS 1.2, проверяет сервер и предъявляет сертификат. Ветка с URI для прежних локальных тестов продолжает зависеть от настроек Go. Обновление клиента до 1.13.0 также необходимо для закрытия остальных найденных дефектов. |
|
||||
| CVE-2026-33815 и CVE-2026-33816, pgx 5.5.5 | Драйвер исполняется в каждом приложении. PostgreSQL доступен через внутреннюю сеть с проверкой TLS и раздельными ролями. Компрометация доверенной БД остаётся релевантной. Снятие риска требует обновления pgx как минимум до 5.9.0 и повтора миграций и проверок отказа. |
|
||||
| CVE-2026-33186, gRPC-Go 1.69.4 | Уязвимость требует gRPC-сервера с соответствующей авторизацией. Мессенджер принимает HTTP, его OTLP-экспорт идёт по HTTP. Это заключение относится к бинарнику мессенджера. Collector и Tempo оцениваются отдельно по образам и включённым listeners. |
|
||||
| CVE-2025-22869, x/crypto 0.32.0 | Описанный путь проходит через SSH/SFTP-сервер. Стенд не предоставляет SSH/SFTP. Использование Argon2id из того же модуля не включает этот путь. |
|
||||
| CVE-2026-46603, x/image | Затронут VP8L-декодер. Код приложения принимает PNG/JPEG, а WebP создаёт через cwebp. Go-декодер WebP используется в тестах для проверки результата. При добавлении входного WebP нужно пересмотреть применимость. |
|
||||
|
||||
| Находка в образе | Применимость и решение |
|
||||
| --- | --- |
|
||||
| OpenBao, CVE-2026-33757 и CVE-2026-33758 | Обе проблемы затрагивают OIDC. На стенде настроен cert auth, операторский endpoint доступен локально. Перед включением OIDC нужно обновить OpenBao и проверить настройки callback. |
|
||||
| MinIO, CVE-2026-33322 и CVE-2026-33419 | Затронуты OIDC и LDAP. На стенде они отключены, операции S3 используют подпись SigV4. Включение внешней федерации требует повторной оценки. |
|
||||
| OpenSSL, CVE-2026-31789 | База указывает на 32-битную обработку крупных X.509. Стенд использует amd64. При переносе на 32-битную архитектуру вывод изменится. |
|
||||
| Tempo, CVE-2026-33186 | Библиотека есть в бинарнике Tempo. Внешняя поставка трейсов идёт по OTLP/HTTP, внутренний gRPC не публикуется на хосте. Влияние через скомпрометированный telemetry-компонент требует отдельной проверки. |
|
||||
| Grafana, CVE-2025-68121 | Сканер нашёл проблему проверки TLS при возобновлении сессии в собранном Go. Grafana использует TLS, поэтому требуется обновление образа с исправленным runtime и проверка datasource-подключений. Локальная публикация и закрытая сеть уменьшают достижимость. |
|
||||
|
||||
Состав образов и версии исправлений приведены в [результате Trivy](grade4-scan.md).
|
||||
Это анализ применимости для учебной конфигурации с текущим профилем.
|
||||
Невключённый путь выполнения не устраняет наличие уязвимого пакета.
|
||||
@@ -0,0 +1,50 @@
|
||||
# Сканирование зависимостей и образов
|
||||
|
||||
Предпосылки: подготовленный, разблокированный учебный стенд; Docker Compose, Python и локальный CA.
|
||||
|
||||
`./scripts/scan_grade4.sh`
|
||||
|
||||
Код завершения: 0.
|
||||
|
||||
```text
|
||||
Scanning highload-lab7-migrate (sha256:12ea72de68ee2d99865a1ea90ccb8162d8f1bbbde6b44d126781d39d5cc42b8c)
|
||||
Scanning highload-lab7-backend-tests (sha256:16c3bf9e4a23386b4f3bfb72d5b57fdf8f98ca796a286c299699a2727eeac5ad)
|
||||
Scanning postgres:16.10-bookworm (sha256:38471f330eb885e04de130b768d6db4e10469e2311879c7e5c699f6d2d8a1c74)
|
||||
Scanning grafana/tempo:2.7.1 (sha256:4443be217c396b065ee34845534199c36fdba4dc619cb96550e228d73fba6e69)
|
||||
Scanning redis:7.4.11-bookworm (sha256:44960b3100c48159a66d28c18b0588c193492e25f7551d60e51dd464fc3eca5b)
|
||||
Container image sha256:4f567dd06e97febea3c531bfdc7e8d58055e292c31132a4b64f3858f2a99d0ed is untagged/unavailable; scanning configured image highload-lab7-s3
|
||||
Scanning highload-lab7-s3 (sha256:cfd603a785197d29fb1509d599c7fdcac782b7e8f288ef22084f6e10ec891c3a)
|
||||
Scanning highload-lab7-upload-cleanup (sha256:4fe5584ba26276e2d003012bb32ad34f43d965d2d8c8a39bb17f7603b6aa35cc)
|
||||
Scanning grafana/loki:3.4.2 (sha256:58a6c186ce78ba04d58bfe2a927eff296ba733a430df09645d56cdc158f3ba08)
|
||||
Scanning openbao/openbao:2.4.4 (sha256:595c83b42614a4d2b044608e4593c05b019c5db25bc9c185d8fff3ac96c03ddd)
|
||||
Scanning haproxy:3.2.25-alpine (sha256:5d97434a423c2533cfeb42874d45cf6d69a840b60334582dfbfd5008e94c80be)
|
||||
Scanning rabbitmq:4.2.0-management (sha256:6431bbd0f22df99ace800138399367d7d6181442dfff09feb2795e15d4bd4b7a)
|
||||
Scanning prom/prometheus:v3.2.1 (sha256:6927e0919a144aa7616fd0137d4816816d42f6b816de3af269ab065250859a62)
|
||||
Scanning highload-lab7-s3-bootstrap (sha256:79b6e5b43792b2063e8154b3ef52bfb111ad2096eda8166ba8bcd392eef85ae2)
|
||||
Scanning otel/opentelemetry-collector-contrib:0.120.0 (sha256:85ac41c2db88d0df9bd6145e608a3cb023f5d8443868adbfbbf66efb51087917)
|
||||
Scanning grafana/grafana:11.5.2 (sha256:8b37a2f028f164ce7b9889e1765b9d6ee23fec80f871d156fbf436d6198d32b7)
|
||||
Scanning highload-lab7-worker (sha256:929b6d0c28efc0f004d2a7c3b0bcb06e92ddca5f6d2ec631f8bf4724862d7573)
|
||||
Scanning highload-lab7-dispatcher (sha256:9a11add2b62e99e9abb61f4292b54175778b5fd722069aeea1dfd3eaafca0f3e)
|
||||
Scanning highload-lab7-api (sha256:d33fb3ff589306b6bf4d47ec657196a198da2b25c36745fc63294163fe999436)
|
||||
dependencies.json {'CRITICAL': 6, 'HIGH': 38, 'LOW': 6, 'MEDIUM': 41, 'UNKNOWN': 3}
|
||||
image-sha256:12ea72de68ee2d99865a1ea90ccb8162d8f1bbbde6b44d126781d39d5cc42b8c.json {'CRITICAL': 10, 'HIGH': 121, 'LOW': 162, 'MEDIUM': 200, 'UNKNOWN': 10}
|
||||
image-sha256:16c3bf9e4a23386b4f3bfb72d5b57fdf8f98ca796a286c299699a2727eeac5ad.json {'CRITICAL': 44, 'HIGH': 1802, 'LOW': 2016, 'MEDIUM': 6804, 'UNKNOWN': 27}
|
||||
image-sha256:38471f330eb885e04de130b768d6db4e10469e2311879c7e5c699f6d2d8a1c74.json {'CRITICAL': 20, 'HIGH': 196, 'LOW': 225, 'MEDIUM': 313, 'UNKNOWN': 18}
|
||||
image-sha256:4443be217c396b065ee34845534199c36fdba4dc619cb96550e228d73fba6e69.json {'CRITICAL': 4, 'HIGH': 89, 'LOW': 44, 'MEDIUM': 112, 'UNKNOWN': 3}
|
||||
image-sha256:44960b3100c48159a66d28c18b0588c193492e25f7551d60e51dd464fc3eca5b.json {'CRITICAL': 1, 'HIGH': 41, 'LOW': 87, 'MEDIUM': 104, 'UNKNOWN': 7}
|
||||
image-sha256:4fe5584ba26276e2d003012bb32ad34f43d965d2d8c8a39bb17f7603b6aa35cc.json {'CRITICAL': 10, 'HIGH': 121, 'LOW': 162, 'MEDIUM': 200, 'UNKNOWN': 10}
|
||||
image-sha256:58a6c186ce78ba04d58bfe2a927eff296ba733a430df09645d56cdc158f3ba08.json {'CRITICAL': 2, 'HIGH': 58, 'LOW': 5, 'MEDIUM': 63, 'UNKNOWN': 6}
|
||||
image-sha256:595c83b42614a4d2b044608e4593c05b019c5db25bc9c185d8fff3ac96c03ddd.json {'CRITICAL': 9, 'HIGH': 91, 'LOW': 64, 'MEDIUM': 82, 'UNKNOWN': 3}
|
||||
image-sha256:5d97434a423c2533cfeb42874d45cf6d69a840b60334582dfbfd5008e94c80be.json {'HIGH': 4, 'MEDIUM': 1}
|
||||
image-sha256:6431bbd0f22df99ace800138399367d7d6181442dfff09feb2795e15d4bd4b7a.json {'HIGH': 7, 'LOW': 73, 'MEDIUM': 320}
|
||||
image-sha256:6927e0919a144aa7616fd0137d4816816d42f6b816de3af269ab065250859a62.json {'CRITICAL': 2, 'HIGH': 56, 'LOW': 8, 'MEDIUM': 65, 'UNKNOWN': 3}
|
||||
image-sha256:79b6e5b43792b2063e8154b3ef52bfb111ad2096eda8166ba8bcd392eef85ae2.json {'CRITICAL': 10, 'HIGH': 121, 'LOW': 162, 'MEDIUM': 200, 'UNKNOWN': 10}
|
||||
image-sha256:85ac41c2db88d0df9bd6145e608a3cb023f5d8443868adbfbbf66efb51087917.json {'CRITICAL': 7, 'HIGH': 81, 'LOW': 13, 'MEDIUM': 84, 'UNKNOWN': 3}
|
||||
image-sha256:8b37a2f028f164ce7b9889e1765b9d6ee23fec80f871d156fbf436d6198d32b7.json {'CRITICAL': 7, 'HIGH': 91, 'LOW': 35, 'MEDIUM': 109, 'UNKNOWN': 3}
|
||||
image-sha256:929b6d0c28efc0f004d2a7c3b0bcb06e92ddca5f6d2ec631f8bf4724862d7573.json {'CRITICAL': 10, 'HIGH': 121, 'LOW': 162, 'MEDIUM': 200, 'UNKNOWN': 10}
|
||||
image-sha256:9a11add2b62e99e9abb61f4292b54175778b5fd722069aeea1dfd3eaafca0f3e.json {'CRITICAL': 10, 'HIGH': 121, 'LOW': 162, 'MEDIUM': 200, 'UNKNOWN': 10}
|
||||
image-sha256:cfd603a785197d29fb1509d599c7fdcac782b7e8f288ef22084f6e10ec891c3a.json {'CRITICAL': 8, 'HIGH': 176, 'LOW': 157, 'MEDIUM': 212, 'UNKNOWN': 8}
|
||||
image-sha256:d33fb3ff589306b6bf4d47ec657196a198da2b25c36745fc63294163fe999436.json {'CRITICAL': 10, 'HIGH': 121, 'LOW': 162, 'MEDIUM': 200, 'UNKNOWN': 10}
|
||||
```
|
||||
|
||||
Итог: все команды завершились успешно.
|
||||
@@ -0,0 +1,42 @@
|
||||
global
|
||||
# Operator access only inside this container; no host port is published.
|
||||
stats socket ipv4@127.0.0.1:9999 level admin
|
||||
log stdout format raw local0
|
||||
maxconn 1024
|
||||
|
||||
defaults
|
||||
log global
|
||||
mode http
|
||||
option httplog
|
||||
timeout connect 1s
|
||||
timeout client 35s
|
||||
timeout server 35s
|
||||
timeout check 1s
|
||||
# Let the client decide which failed POST is safe to repeat.
|
||||
retries 0
|
||||
|
||||
# Docker returns the current addresses of all containers of the api service.
|
||||
resolvers docker
|
||||
nameserver docker_dns 127.0.0.11:53
|
||||
accepted_payload_size 8192
|
||||
resolve_retries 3
|
||||
timeout resolve 1s
|
||||
timeout retry 1s
|
||||
hold valid 1s
|
||||
hold obsolete 1s
|
||||
hold nx 1s
|
||||
hold timeout 1s
|
||||
hold refused 1s
|
||||
hold other 1s
|
||||
|
||||
frontend api_entry
|
||||
bind :8443 ssl crt /run/tls/ingress.pem ssl-min-ver TLSv1.2
|
||||
http-request set-header X-Forwarded-For %[src]
|
||||
default_backend api_replicas
|
||||
|
||||
backend api_replicas
|
||||
balance roundrobin
|
||||
option httpchk GET /health/ready
|
||||
http-check expect status 200
|
||||
# Separate slots receive distinct Docker DNS addresses. No sticky sessions.
|
||||
server-template api "${API_REPLICAS}" "api:${APP_PORT}" ssl verify required ca-file /run/tls/ca.crt crt /run/tls/lb.pem verifyhost api ssl-min-ver TLSv1.2 check inter 1s fall 2 rise 2 resolvers docker resolve-prefer ipv4 init-addr libc,none
|
||||
@@ -0,0 +1,19 @@
|
||||
FROM golang:1.24.2-bookworm AS builder
|
||||
WORKDIR /source
|
||||
RUN git init . \
|
||||
&& git remote add origin https://github.com/minio/minio.git \
|
||||
&& git fetch --depth 1 origin 0d7408fc9969caf07de6a8c3a84f9fbb10a6739e \
|
||||
&& git checkout --detach FETCH_HEAD
|
||||
RUN CGO_ENABLED=0 go build -trimpath \
|
||||
-ldflags="-s -w -X github.com/minio/minio/cmd.Version=2025-04-22T22:12:26Z -X github.com/minio/minio/cmd.ReleaseTag=RELEASE.2025-04-22T22-12-26Z -X github.com/minio/minio/cmd.CommitID=0d7408fc9969caf07de6a8c3a84f9fbb10a6739e" \
|
||||
-o /out/minio .
|
||||
|
||||
FROM debian:12.15-slim
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends ca-certificates curl \
|
||||
&& rm -rf /var/lib/apt/lists/* \
|
||||
&& groupadd --system storage && useradd --system --gid storage storage \
|
||||
&& mkdir /data && chown storage:storage /data
|
||||
COPY --from=builder /out/minio /usr/local/bin/minio
|
||||
USER storage:storage
|
||||
ENTRYPOINT ["minio"]
|
||||
CMD ["server", "/data", "--console-address", ":9001"]
|
||||
@@ -0,0 +1,9 @@
|
||||
api_addr = "https://openbao:8200"
|
||||
storage "file" {
|
||||
path = "/openbao/file"
|
||||
}
|
||||
listener "tcp" {
|
||||
address = "0.0.0.0:8200"
|
||||
tls_cert_file = "/run/tls/bao.crt"
|
||||
tls_key_file = "/run/tls/bao.key"
|
||||
}
|
||||
@@ -0,0 +1,20 @@
|
||||
groups:
|
||||
- name: messenger
|
||||
interval: 5s
|
||||
rules:
|
||||
- alert: MessengerHighErrorRate
|
||||
expr: (sum(rate(messenger_http_requests_total{http_status_class="5xx"}[1m])) / sum(rate(messenger_http_requests_total[1m]))) > 0.05
|
||||
for: 15s
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: More than 5% of business requests return 5xx over one minute
|
||||
runbook: Check API error spans and PostgreSQL operations in Tempo.
|
||||
- alert: MessengerQueueLag
|
||||
expr: max(messenger_queue_oldest_seconds) > 20
|
||||
for: 15s
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: Oldest outstanding attachment job is over 20 seconds old
|
||||
runbook: Check outstanding jobs, worker saturation and consumer traces.
|
||||
@@ -0,0 +1,61 @@
|
||||
receivers:
|
||||
otlp:
|
||||
protocols:
|
||||
http:
|
||||
endpoint: 0.0.0.0:4318
|
||||
tls:
|
||||
cert_file: /run/otel/server.crt
|
||||
key_file: /run/otel/server.key
|
||||
client_ca_file: /run/otel/ingest-ca.crt
|
||||
processors:
|
||||
memory_limiter:
|
||||
check_interval: 1s
|
||||
limit_mib: 128
|
||||
spike_limit_mib: 32
|
||||
batch:
|
||||
timeout: 1s
|
||||
send_batch_size: 256
|
||||
send_batch_max_size: 512
|
||||
exporters:
|
||||
prometheus:
|
||||
endpoint: 0.0.0.0:8889
|
||||
tls:
|
||||
cert_file: /run/otel/server.crt
|
||||
key_file: /run/otel/server.key
|
||||
client_ca_file: /run/otel/export-ca.crt
|
||||
resource_to_telemetry_conversion:
|
||||
enabled: true
|
||||
otlphttp/loki:
|
||||
endpoint: https://loki:3100/otlp
|
||||
tls:
|
||||
ca_file: /run/tls/ca.crt
|
||||
cert_file: /run/otel/client.crt
|
||||
key_file: /run/otel/client.key
|
||||
sending_queue:
|
||||
queue_size: 64
|
||||
retry_on_failure:
|
||||
max_elapsed_time: 30s
|
||||
otlphttp/tempo:
|
||||
endpoint: https://tempo:4318
|
||||
tls:
|
||||
ca_file: /run/tls/ca.crt
|
||||
cert_file: /run/otel/client.crt
|
||||
key_file: /run/otel/client.key
|
||||
sending_queue:
|
||||
queue_size: 64
|
||||
retry_on_failure:
|
||||
max_elapsed_time: 30s
|
||||
service:
|
||||
pipelines:
|
||||
metrics:
|
||||
receivers: [otlp]
|
||||
processors: [memory_limiter, batch]
|
||||
exporters: [prometheus]
|
||||
logs:
|
||||
receivers: [otlp]
|
||||
processors: [memory_limiter, batch]
|
||||
exporters: [otlphttp/loki]
|
||||
traces:
|
||||
receivers: [otlp]
|
||||
processors: [memory_limiter, batch]
|
||||
exporters: [otlphttp/tempo]
|
||||
@@ -0,0 +1,513 @@
|
||||
{
|
||||
"uid": "messenger-lab6",
|
||||
"title": "Мессенджер · API и worker",
|
||||
"schemaVersion": 39,
|
||||
"refresh": "5s",
|
||||
"time": {
|
||||
"from": "now-15m",
|
||||
"to": "now"
|
||||
},
|
||||
"panels": [
|
||||
{
|
||||
"id": 1,
|
||||
"title": "API · запросы в секунду",
|
||||
"type": "timeseries",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "lab6-prometheus"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "lab6-prometheus"
|
||||
},
|
||||
"expr": "sum(rate(messenger_http_requests_total[1m]))",
|
||||
"legendFormat": ""
|
||||
}
|
||||
],
|
||||
"gridPos": {
|
||||
"x": 0,
|
||||
"y": 0,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "reqps"
|
||||
},
|
||||
"overrides": []
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 2,
|
||||
"title": "API · 5xx в секунду",
|
||||
"type": "timeseries",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "lab6-prometheus"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "lab6-prometheus"
|
||||
},
|
||||
"expr": "sum(rate(messenger_http_requests_total{http_status_class=\"5xx\"}[1m])) or 0 * sum(rate(messenger_http_requests_total[1m]))",
|
||||
"legendFormat": ""
|
||||
}
|
||||
],
|
||||
"gridPos": {
|
||||
"x": 12,
|
||||
"y": 0,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "reqps"
|
||||
},
|
||||
"overrides": []
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 3,
|
||||
"title": "API · p95 длительности",
|
||||
"type": "timeseries",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "lab6-prometheus"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "lab6-prometheus"
|
||||
},
|
||||
"expr": "histogram_quantile(0.95, sum by (le) (rate(messenger_http_duration_seconds_bucket[1m])))",
|
||||
"legendFormat": ""
|
||||
}
|
||||
],
|
||||
"gridPos": {
|
||||
"x": 0,
|
||||
"y": 8,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "s"
|
||||
},
|
||||
"overrides": []
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 4,
|
||||
"title": "Worker · задания в минуту",
|
||||
"type": "timeseries",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "lab6-prometheus"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "lab6-prometheus"
|
||||
},
|
||||
"expr": "60 * sum by (job_outcome) (rate(messenger_worker_jobs_total[1m]))",
|
||||
"legendFormat": "{{job_outcome}}"
|
||||
}
|
||||
],
|
||||
"gridPos": {
|
||||
"x": 12,
|
||||
"y": 8,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "short"
|
||||
},
|
||||
"overrides": []
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 5,
|
||||
"title": "Worker · p95 длительности",
|
||||
"type": "timeseries",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "lab6-prometheus"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "lab6-prometheus"
|
||||
},
|
||||
"expr": "histogram_quantile(0.95, sum by (le,job_outcome) (rate(messenger_worker_duration_seconds_bucket[1m])))",
|
||||
"legendFormat": "{{job_outcome}}"
|
||||
}
|
||||
],
|
||||
"gridPos": {
|
||||
"x": 0,
|
||||
"y": 16,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "s"
|
||||
},
|
||||
"overrides": []
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 6,
|
||||
"title": "API и worker · события",
|
||||
"type": "logs",
|
||||
"datasource": {
|
||||
"type": "loki",
|
||||
"uid": "lab6-loki"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": {
|
||||
"type": "loki",
|
||||
"uid": "lab6-loki"
|
||||
},
|
||||
"expr": "{service_name=~\"messenger-api|messenger-worker\"}",
|
||||
"legendFormat": "",
|
||||
"queryType": "range"
|
||||
}
|
||||
],
|
||||
"gridPos": {
|
||||
"x": 12,
|
||||
"y": 16,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": ""
|
||||
},
|
||||
"overrides": []
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 7,
|
||||
"title": "API · p50 длительности",
|
||||
"type": "timeseries",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "lab6-prometheus"
|
||||
},
|
||||
"gridPos": {
|
||||
"x": 0,
|
||||
"y": 24,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "histogram_quantile(0.5, sum by (le) (rate(messenger_http_duration_seconds_bucket[1m])))",
|
||||
"legendFormat": ""
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "s"
|
||||
},
|
||||
"overrides": []
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 8,
|
||||
"title": "API · p99 длительности",
|
||||
"type": "timeseries",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "lab6-prometheus"
|
||||
},
|
||||
"gridPos": {
|
||||
"x": 12,
|
||||
"y": 24,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "histogram_quantile(0.99, sum by (le) (rate(messenger_http_duration_seconds_bucket[1m])))",
|
||||
"legendFormat": ""
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "s"
|
||||
},
|
||||
"overrides": []
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 9,
|
||||
"title": "API · доля 5xx",
|
||||
"type": "timeseries",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "lab6-prometheus"
|
||||
},
|
||||
"gridPos": {
|
||||
"x": 0,
|
||||
"y": 32,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "(sum(rate(messenger_http_requests_total{http_status_class=\"5xx\"}[1m])) or 0 * sum(rate(messenger_http_requests_total[1m]))) / sum(rate(messenger_http_requests_total[1m]))",
|
||||
"legendFormat": ""
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "percentunit"
|
||||
},
|
||||
"overrides": []
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 10,
|
||||
"title": "Очередь attachments.v1 · незавершённые задания",
|
||||
"type": "timeseries",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "lab6-prometheus"
|
||||
},
|
||||
"gridPos": {
|
||||
"x": 12,
|
||||
"y": 32,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "max(messenger_queue_depth)",
|
||||
"legendFormat": ""
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "short"
|
||||
},
|
||||
"overrides": []
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 11,
|
||||
"title": "Очередь attachments.v1 · возраст старейшего задания",
|
||||
"type": "timeseries",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "lab6-prometheus"
|
||||
},
|
||||
"gridPos": {
|
||||
"x": 0,
|
||||
"y": 40,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "max(messenger_queue_oldest_seconds)",
|
||||
"legendFormat": ""
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "s"
|
||||
},
|
||||
"overrides": []
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 12,
|
||||
"title": "PostgreSQL · занятые соединения",
|
||||
"type": "timeseries",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "lab6-prometheus"
|
||||
},
|
||||
"gridPos": {
|
||||
"x": 12,
|
||||
"y": 40,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "sum by (service_name) (messenger_db_pool_used)",
|
||||
"legendFormat": "{{service_name}}"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "short"
|
||||
},
|
||||
"overrides": []
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 13,
|
||||
"title": "PostgreSQL · запросы к пулу, ожидавшие соединение, в секунду",
|
||||
"type": "timeseries",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "lab6-prometheus"
|
||||
},
|
||||
"gridPos": {
|
||||
"x": 0,
|
||||
"y": 48,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "sum by (service_name) (rate(messenger_db_pool_waited_total[1m]))",
|
||||
"legendFormat": "{{service_name}}"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "ops"
|
||||
},
|
||||
"overrides": []
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 14,
|
||||
"title": "Worker · загрузка",
|
||||
"type": "timeseries",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "lab6-prometheus"
|
||||
},
|
||||
"gridPos": {
|
||||
"x": 12,
|
||||
"y": 48,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "sum(messenger_worker_busy) / sum(messenger_worker_capacity)",
|
||||
"legendFormat": ""
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "percentunit"
|
||||
},
|
||||
"overrides": []
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 15,
|
||||
"title": "Вложения · p95 до ready",
|
||||
"type": "timeseries",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "lab6-prometheus"
|
||||
},
|
||||
"gridPos": {
|
||||
"x": 0,
|
||||
"y": 56,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "histogram_quantile(0.95, sum by (le) (rate(messenger_attachment_ready_seconds_bucket[1m])))",
|
||||
"legendFormat": ""
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "s"
|
||||
},
|
||||
"overrides": []
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 16,
|
||||
"title": "Процессы · загрузка CPU",
|
||||
"type": "timeseries",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "lab6-prometheus"
|
||||
},
|
||||
"gridPos": {
|
||||
"x": 12,
|
||||
"y": 56,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "sum by (service_name) (rate(messenger_process_cpu_seconds_total[1m]))",
|
||||
"legendFormat": "{{service_name}}"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "cores"
|
||||
},
|
||||
"overrides": []
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 17,
|
||||
"title": "Процессы · резидентная память",
|
||||
"type": "timeseries",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "lab6-prometheus"
|
||||
},
|
||||
"gridPos": {
|
||||
"x": 0,
|
||||
"y": 64,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "sum by (service_name) (messenger_process_rss_bytes)",
|
||||
"legendFormat": "{{service_name}}"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "bytes"
|
||||
},
|
||||
"overrides": []
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,6 @@
|
||||
apiVersion: 1
|
||||
providers:
|
||||
- name: Messenger
|
||||
type: file
|
||||
options:
|
||||
path: /var/lib/grafana/dashboards
|
||||
@@ -0,0 +1,45 @@
|
||||
apiVersion: 1
|
||||
datasources:
|
||||
- name: Prometheus
|
||||
uid: lab6-prometheus
|
||||
type: prometheus
|
||||
access: proxy
|
||||
url: https://prometheus:9090
|
||||
isDefault: true
|
||||
secureJsonData:
|
||||
tlsClientCert: ${GRAFANA_READER_CERT}
|
||||
tlsClientKey: ${GRAFANA_READER_KEY}
|
||||
jsonData:
|
||||
tlsAuth: true
|
||||
- name: Loki
|
||||
uid: lab6-loki
|
||||
type: loki
|
||||
access: proxy
|
||||
url: https://loki:3100
|
||||
secureJsonData:
|
||||
tlsClientCert: ${GRAFANA_READER_CERT}
|
||||
tlsClientKey: ${GRAFANA_READER_KEY}
|
||||
jsonData:
|
||||
tlsAuth: true
|
||||
derivedFields:
|
||||
- name: Trace
|
||||
matcherType: label
|
||||
matcherRegex: trace_id
|
||||
datasourceUid: lab6-tempo
|
||||
url: '$${__value.raw}'
|
||||
- name: Tempo
|
||||
uid: lab6-tempo
|
||||
type: tempo
|
||||
access: proxy
|
||||
url: https://tempo:3200
|
||||
secureJsonData:
|
||||
tlsClientCert: ${GRAFANA_READER_CERT}
|
||||
tlsClientKey: ${GRAFANA_READER_KEY}
|
||||
jsonData:
|
||||
tlsAuth: true
|
||||
tracesToLogsV2:
|
||||
datasourceUid: lab6-loki
|
||||
spanStartTimeShift: '-5s'
|
||||
spanEndTimeShift: '5s'
|
||||
customQuery: true
|
||||
query: '{service_name=~"messenger-api|messenger-worker"} | trace_id="$${__span.traceId}"'
|
||||
@@ -0,0 +1,36 @@
|
||||
auth_enabled: false
|
||||
server:
|
||||
http_listen_port: 3100
|
||||
http_tls_config:
|
||||
cert_file: /run/otel/server.crt
|
||||
key_file: /run/otel/server.key
|
||||
client_auth_type: RequireAndVerifyClientCert
|
||||
client_ca_file: /run/otel/loki-client-ca.crt
|
||||
common:
|
||||
path_prefix: /loki
|
||||
replication_factor: 1
|
||||
ring:
|
||||
instance_addr: 127.0.0.1
|
||||
kvstore:
|
||||
store: inmemory
|
||||
storage:
|
||||
filesystem:
|
||||
chunks_directory: /loki/chunks
|
||||
rules_directory: /loki/rules
|
||||
schema_config:
|
||||
configs:
|
||||
- from: 2024-01-01
|
||||
store: tsdb
|
||||
object_store: filesystem
|
||||
schema: v13
|
||||
index:
|
||||
prefix: index_
|
||||
period: 24h
|
||||
limits_config:
|
||||
allow_structured_metadata: true
|
||||
otlp_config:
|
||||
resource_attributes:
|
||||
ignore_defaults: true
|
||||
attributes_config:
|
||||
- action: index_label
|
||||
attributes: [service.name, deployment.environment.name]
|
||||
@@ -0,0 +1,5 @@
|
||||
tls_server_config:
|
||||
cert_file: /run/otel/server.crt
|
||||
key_file: /run/otel/server.key
|
||||
client_auth_type: RequireAndVerifyClientCert
|
||||
client_ca_file: /run/otel/read-ca.crt
|
||||
@@ -0,0 +1,13 @@
|
||||
global:
|
||||
scrape_interval: 5s
|
||||
rule_files:
|
||||
- /etc/prometheus/alerts.yaml
|
||||
scrape_configs:
|
||||
- job_name: otel-collector
|
||||
scheme: https
|
||||
tls_config:
|
||||
ca_file: /run/tls/ca.crt
|
||||
cert_file: /run/otel/client.crt
|
||||
key_file: /run/otel/client.key
|
||||
static_configs:
|
||||
- targets: [otel-collector:8889]
|
||||
@@ -0,0 +1,27 @@
|
||||
# Диагностика API и обработки вложений
|
||||
|
||||
SLI доступности API — доля бизнес-запросов без 5xx. Цель — ≥99% за 30 минут при наличии трафика. Проверка:
|
||||
|
||||
```promql
|
||||
1 - (sum(increase(messenger_http_requests_total{http_status_class="5xx"}[30m])) or 0 * sum(increase(messenger_http_requests_total[30m]))) / sum(increase(messenger_http_requests_total[30m]))
|
||||
```
|
||||
|
||||
Health исключён из обоих счётчиков. `MessengerHighErrorRate` срабатывает при доле 5xx выше 5% за минуту в течение 15 секунд. В Explore → Tempo найти ошибочный серверный спан и дочерний `postgresql query`. Сравнить длительность дочернего спана с длительностью HTTP, затем сверить занятые соединения и rate запросов, ожидавших пул.
|
||||
|
||||
`MessengerQueueLag` срабатывает, когда старейшее незавершённое задание старше 20 секунд в течение 15 секунд. В dashboard проверить число незавершённых заданий и наличие worker. Если возраст растёт при отсутствии новых consumer spans, проверить процесс worker. После восстановления дождаться перехода заданий в ready и исчезновения активного alert.
|
||||
|
||||
Глубина очереди измеряется по устойчивым заданиям в PostgreSQL. В неё входят работа в RabbitMQ, leases и ожидание retry. Возраст измеряется от создания задания. Время до ready включает ожидание и повторные попытки. Гистограммы времени используют секунды. Загрузка worker — доля занятых слотов, по одному слоту на процесс. Ожидание пула отображается как частота успешных получений соединения после ожидания. Метрики CPU и RSS относятся к основным процессам Go.
|
||||
|
||||
Правила доступны через API Prometheus `/api/v1/alerts` или запрос `ALERTS{alertstate="firing"}` в Grafana. При недоступности Loki или Tempo приложение продолжает работу. Collector ограничивает память и очередь экспорта, повторяет отправку до 30 секунд. Сигналы сверх лимитов или после истечения retry могут быть потеряны.
|
||||
|
||||
Команды локальной проверки после `make app-up`:
|
||||
|
||||
```sh
|
||||
python3 scripts/check_observability.py pipeline
|
||||
python3 scripts/check_observability_incidents.py worker
|
||||
python3 scripts/check_observability_incidents.py database
|
||||
python3 scripts/check_observability_incidents.py retry
|
||||
python3 scripts/check_observability_incidents.py telemetry
|
||||
```
|
||||
|
||||
Сценарии выполняются последовательно на учебном стенде. DB-инцидент держит временную блокировку таблицы чатов. Проверка retry кратко останавливает S3 и проверяет номера попыток в одной трассе исходного upload. Проверка telemetry останавливает Tempo и одновременно отправляет сообщения и обрабатывает вложение.
|
||||
@@ -0,0 +1,27 @@
|
||||
server:
|
||||
http_listen_port: 3200
|
||||
http_tls_config:
|
||||
cert_file: /run/otel/server.crt
|
||||
key_file: /run/otel/server.key
|
||||
client_auth_type: RequireAndVerifyClientCert
|
||||
client_ca_file: /run/otel/read-ca.crt
|
||||
distributor:
|
||||
receivers:
|
||||
otlp:
|
||||
protocols:
|
||||
http:
|
||||
endpoint: 0.0.0.0:4318
|
||||
tls:
|
||||
cert_file: /run/otel/server.crt
|
||||
key_file: /run/otel/server.key
|
||||
client_ca_file: /run/otel/export-ca.crt
|
||||
storage:
|
||||
trace:
|
||||
backend: local
|
||||
wal:
|
||||
path: /var/tempo/wal
|
||||
local:
|
||||
path: /var/tempo/blocks
|
||||
compactor:
|
||||
compaction:
|
||||
block_retention: 48h
|
||||
@@ -0,0 +1,2 @@
|
||||
local all all trust
|
||||
hostssl all all all scram-sha-256
|
||||
@@ -0,0 +1,34 @@
|
||||
\getenv api_password api_password
|
||||
\getenv worker_password worker_password
|
||||
\getenv dispatcher_password dispatcher_password
|
||||
\getenv cleanup_password cleanup_password
|
||||
REVOKE CREATE ON SCHEMA public FROM PUBLIC;
|
||||
SELECT format('CREATE ROLE %I LOGIN', 'messenger_' || role)
|
||||
FROM unnest(ARRAY['api', 'worker', 'dispatcher', 'cleanup']) AS role
|
||||
WHERE NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'messenger_' || role)
|
||||
\gexec
|
||||
ALTER ROLE messenger_api PASSWORD :'api_password';
|
||||
ALTER ROLE messenger_worker PASSWORD :'worker_password';
|
||||
ALTER ROLE messenger_dispatcher PASSWORD :'dispatcher_password';
|
||||
ALTER ROLE messenger_cleanup PASSWORD :'cleanup_password';
|
||||
|
||||
GRANT SELECT, INSERT ON users, chats, chat_members, messages, message_attachments TO messenger_api;
|
||||
GRANT UPDATE (password_hash) ON users TO messenger_api;
|
||||
GRANT SELECT, INSERT, UPDATE ON attachments, direct_uploads TO messenger_api;
|
||||
GRANT SELECT, INSERT, DELETE ON upload_attempts, message_idempotency TO messenger_api;
|
||||
GRANT UPDATE ON message_idempotency TO messenger_api;
|
||||
GRANT UPDATE ON upload_attempts TO messenger_api;
|
||||
GRANT DELETE ON direct_uploads TO messenger_cleanup;
|
||||
GRANT INSERT ON attachment_jobs TO messenger_api;
|
||||
GRANT SELECT, UPDATE ON attachments, attachment_jobs TO messenger_worker;
|
||||
GRANT SELECT, INSERT, UPDATE ON attachment_job_attempts, attachment_job_failures TO messenger_worker;
|
||||
GRANT INSERT ON attachment_publications TO messenger_worker;
|
||||
GRANT SELECT ON attachments, attachment_jobs, attachment_job_failures TO messenger_dispatcher;
|
||||
GRANT UPDATE (published_at) ON attachment_jobs TO messenger_dispatcher;
|
||||
GRANT SELECT, UPDATE ON upload_attempts, direct_uploads TO messenger_cleanup;
|
||||
GRANT DELETE ON upload_attempts TO messenger_cleanup;
|
||||
GRANT SELECT ON attachments TO messenger_cleanup;
|
||||
GRANT USAGE ON SEQUENCE attachment_publications_id_seq TO messenger_worker;
|
||||
GRANT SELECT ON attachment_jobs, attachment_job_failures TO messenger_cleanup;
|
||||
GRANT UPDATE (staging_deleted_at) ON attachment_jobs TO messenger_cleanup;
|
||||
GRANT UPDATE (updated_at) ON attachments TO messenger_cleanup;
|
||||
@@ -0,0 +1,13 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
until rabbitmq-diagnostics -q check_running >/dev/null 2>&1; do sleep 1; done
|
||||
for role in dispatcher worker; do
|
||||
username="messenger_$role"
|
||||
if ! rabbitmqctl -q list_users | cut -f1 | grep -qx "$username"; then
|
||||
rabbitmqctl -q add_user "$username" ""
|
||||
fi
|
||||
rabbitmqctl -q clear_password "$username"
|
||||
if [ "$role" = dispatcher ]; then write='^amq\.default$'; read='^$'
|
||||
else write='^$'; read='^attachments\.v1$'; fi
|
||||
rabbitmqctl -q set_permissions "$username" '^attachments\.v1$' "$write" "$read"
|
||||
done
|
||||
@@ -0,0 +1 @@
|
||||
[rabbitmq_management,rabbitmq_prometheus,rabbitmq_auth_mechanism_ssl].
|
||||
@@ -0,0 +1,12 @@
|
||||
listeners.tcp = none
|
||||
listeners.ssl.default = 5671
|
||||
ssl_options.cacertfile = /run/tls/broker-client-ca.crt
|
||||
ssl_options.certfile = /run/tls/broker.crt
|
||||
ssl_options.keyfile = /run/tls/broker.key
|
||||
ssl_options.verify = verify_peer
|
||||
ssl_options.fail_if_no_peer_cert = true
|
||||
management.tcp.port = 15672
|
||||
management.tcp.ip = 127.0.0.1
|
||||
prometheus.tcp.ip = 127.0.0.1
|
||||
auth_mechanisms.1 = EXTERNAL
|
||||
ssl_cert_login_from = common_name
|
||||
@@ -0,0 +1,55 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Initialize or unseal the local OpenBao; keep operator material on the host."""
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import ssl
|
||||
from pathlib import Path
|
||||
from urllib.error import HTTPError
|
||||
from urllib.request import Request, urlopen
|
||||
|
||||
SECRETS = Path(__file__).resolve().parents[1] / "secrets"
|
||||
MATERIAL = SECRETS / "bao-operator.json"
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("action", choices=("init", "unseal"))
|
||||
action = parser.parse_args().action
|
||||
context = ssl.create_default_context(cafile=str(SECRETS / "tls/lab7-ca.crt"))
|
||||
|
||||
def request(path, data):
|
||||
req = Request(
|
||||
"https://localhost:8200/v1/sys/" + path,
|
||||
method="PUT",
|
||||
data=json.dumps(data).encode(),
|
||||
headers={"Content-Type": "application/json"},
|
||||
)
|
||||
with urlopen(req, context=context, timeout=15) as response:
|
||||
return json.load(response)
|
||||
|
||||
if action == "init":
|
||||
# Refuse to overwrite the operator's only copy of the material.
|
||||
fd = os.open(MATERIAL, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600)
|
||||
with os.fdopen(fd, "w") as output:
|
||||
try:
|
||||
data = request("init", {"secret_shares": 3, "secret_threshold": 2})
|
||||
except HTTPError:
|
||||
MATERIAL.unlink()
|
||||
raise
|
||||
json.dump(data, output)
|
||||
output.flush()
|
||||
os.fsync(output.fileno())
|
||||
print("Operator material saved in secrets/bao-operator.json")
|
||||
return
|
||||
|
||||
data = json.loads(MATERIAL.read_text())
|
||||
for share in data["keys_base64"]:
|
||||
if not request("unseal", {"key": share})["sealed"]:
|
||||
print("OpenBao unsealed")
|
||||
return
|
||||
raise SystemExit("OpenBao remains sealed")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Executable
+92
@@ -0,0 +1,92 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
umask 077
|
||||
dir=secrets/tls
|
||||
mkdir -p "$dir"
|
||||
chmod 700 "$dir"
|
||||
|
||||
make_key() {
|
||||
if [ "$algorithm" = ec ]; then
|
||||
openssl genpkey -algorithm EC -pkeyopt ec_paramgen_curve:P-256 -out "$dir/$name.key"
|
||||
else
|
||||
openssl genpkey -algorithm ED25519 -out "$dir/$name.key"
|
||||
fi
|
||||
}
|
||||
|
||||
make_ca() {
|
||||
name=$1 algorithm=${2:-ed25519}
|
||||
if [ ! -e "$dir/$name.crt" ]; then
|
||||
make_key
|
||||
openssl req -x509 -new -key "$dir/$name.key" -days 365 \
|
||||
-subj "/CN=$name" -addext "basicConstraints=critical,CA:TRUE,pathlen:0" \
|
||||
-addext "keyUsage=critical,keyCertSign,cRLSign" \
|
||||
-out "$dir/$name.crt"
|
||||
fi
|
||||
}
|
||||
|
||||
make_leaf() {
|
||||
name=$1 ca=$2 usage=$3 san=$4 algorithm=${5:-ed25519}
|
||||
if [ ! -e "$dir/$name.crt" ]; then
|
||||
make_key
|
||||
openssl req -new -key "$dir/$name.key" -subj "/CN=$name" -out "$dir/$name.csr"
|
||||
printf 'basicConstraints=critical,CA:FALSE\nkeyUsage=critical,digitalSignature\nextendedKeyUsage=%s\nsubjectAltName=%s\nsubjectKeyIdentifier=hash\nauthorityKeyIdentifier=keyid:always\n' \
|
||||
"$usage" "$san" > "$dir/$name.ext"
|
||||
openssl x509 -req -in "$dir/$name.csr" -CA "$dir/$ca.crt" -CAkey "$dir/$ca.key" \
|
||||
-CAcreateserial -days 30 -extfile "$dir/$name.ext" -out "$dir/$name.crt"
|
||||
rm "$dir/$name.ext" "$dir/$name.csr"
|
||||
fi
|
||||
# Containers can only read files individually mounted into them.
|
||||
chmod 644 "$dir/$name.crt" "$dir/$name.key"
|
||||
}
|
||||
|
||||
make_ca lab7-ca
|
||||
make_ca untrusted-ca
|
||||
chmod 644 "$dir/lab7-ca.crt"
|
||||
make_ca lab7-ingress-ca ec
|
||||
chmod 644 "$dir/lab7-ingress-ca.crt"
|
||||
make_leaf ingress-browser lab7-ingress-ca serverAuth DNS:localhost,IP:127.0.0.1 ec
|
||||
make_leaf api lab7-ca serverAuth DNS:api
|
||||
for service in db redis s3 broker; do
|
||||
san="DNS:$service"
|
||||
if [ "$service" = s3 ]; then san="$san,DNS:localhost,IP:127.0.0.1"; fi
|
||||
make_leaf "$service" lab7-ca serverAuth "$san"
|
||||
done
|
||||
for service in otel-collector loki tempo prometheus grafana; do
|
||||
san="DNS:$service"
|
||||
if [ "$service" = grafana ]; then san="$san,DNS:localhost,IP:127.0.0.1"; fi
|
||||
make_leaf "$service" lab7-ca serverAuth "$san"
|
||||
done
|
||||
make_ca telemetry-read-ca
|
||||
make_ca telemetry-export-ca
|
||||
chmod 644 "$dir/telemetry-read-ca.crt" "$dir/telemetry-export-ca.crt"
|
||||
make_leaf grafana-reader telemetry-read-ca clientAuth URI:urn:lab7:grafana
|
||||
make_leaf collector-exporter telemetry-export-ca clientAuth URI:urn:lab7:collector
|
||||
make_leaf prometheus-scraper telemetry-export-ca clientAuth URI:urn:lab7:prometheus
|
||||
cat "$dir/telemetry-read-ca.crt" "$dir/telemetry-export-ca.crt" > "$dir/loki-client-ca.crt"
|
||||
chmod 644 "$dir/loki-client-ca.crt"
|
||||
make_ca telemetry-ingest-ca
|
||||
chmod 644 "$dir/telemetry-ingest-ca.crt"
|
||||
for role in api worker dispatcher; do
|
||||
make_leaf "otel-$role" telemetry-ingest-ca clientAuth "URI:urn:lab7:otel:$role"
|
||||
done
|
||||
make_ca broker-client-ca
|
||||
chmod 644 "$dir/broker-client-ca.crt"
|
||||
for role in dispatcher worker unregistered; do
|
||||
make_leaf "messenger_$role" broker-client-ca clientAuth "URI:urn:lab7:broker:$role"
|
||||
done
|
||||
make_leaf openbao lab7-ca serverAuth DNS:openbao,DNS:localhost,IP:127.0.0.1
|
||||
for role in api worker dispatcher cleanup; do
|
||||
make_leaf "bao-$role" lab7-ca clientAuth "URI:urn:lab7:bao:$role"
|
||||
done
|
||||
make_leaf lb lab7-ca clientAuth URI:urn:lab7:load-balancer
|
||||
make_leaf wrong-role lab7-ca clientAuth URI:urn:lab7:probe
|
||||
make_leaf wrong-ca untrusted-ca clientAuth URI:urn:lab7:load-balancer
|
||||
# The Collector trusts only the leaf certificates registered for ingest.
|
||||
cat "$dir/otel-api.crt" "$dir/otel-worker.crt" "$dir/otel-dispatcher.crt" > "$dir/telemetry-ingest-clients.crt"
|
||||
chmod 644 "$dir/telemetry-ingest-clients.crt"
|
||||
make_leaf otel-wrong-role telemetry-ingest-ca clientAuth URI:urn:lab7:otel:unregistered
|
||||
for name in ingress-browser lb; do
|
||||
cat "$dir/$name.crt" "$dir/$name.key" > "$dir/$name.pem"
|
||||
chmod 644 "$dir/$name.pem"
|
||||
done
|
||||
@@ -0,0 +1,45 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Prove each runtime identity can read only its own OpenBao secret."""
|
||||
import json
|
||||
import ssl
|
||||
from pathlib import Path
|
||||
from urllib.error import HTTPError
|
||||
from urllib.request import Request, urlopen
|
||||
|
||||
TLS = Path(__file__).resolve().parents[1] / "secrets/tls"
|
||||
ROLES = ("api", "worker", "dispatcher", "cleanup")
|
||||
|
||||
|
||||
def request(context, path, *, body=None, token="", status=200):
|
||||
req = Request(
|
||||
f"https://localhost:8200/v1/{path}",
|
||||
data=None if body is None else json.dumps(body).encode(),
|
||||
headers={"X-Vault-Token": token, "Content-Type": "application/json"},
|
||||
)
|
||||
try:
|
||||
with urlopen(req, context=context, timeout=5) as response:
|
||||
assert response.status == status, (path, response.status, status)
|
||||
return json.load(response) if response.status != 204 else None
|
||||
except HTTPError as error:
|
||||
assert error.code == status, (path, error.code, status)
|
||||
|
||||
|
||||
def main():
|
||||
for role in ROLES:
|
||||
context = ssl.create_default_context(cafile=str(TLS / "lab7-ca.crt"))
|
||||
context.load_cert_chain(TLS / f"bao-{role}.crt", TLS / f"bao-{role}.key")
|
||||
auth = request(context, "auth/cert/login", body={"name": role})["auth"]
|
||||
assert set(auth["policies"]) == {role}
|
||||
assert 0 < auth["lease_duration"] <= 900
|
||||
token = auth["client_token"]
|
||||
request(context, f"messenger/data/{role}", token=token)
|
||||
request(context, f"messenger/data/{role}", token=token, body={"data": {}}, status=403)
|
||||
request(context, f"sys/policies/acl/{role}", token=token, body={"policy": ""}, status=403)
|
||||
for other in set(ROLES) - {role}:
|
||||
request(context, f"messenger/data/{other}", token=token, status=403)
|
||||
request(context, "auth/cert/login", body={"name": other}, status=400)
|
||||
print(f"PASS {role}: scoped read, cross-role login/read and write denied, TTL <= 900 s")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Executable
+26
@@ -0,0 +1,26 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
|
||||
docker compose exec -T db sh -ec '
|
||||
export PGPASSWORD=$(cat /run/secrets/database_worker_password)
|
||||
export PGCHANNELBINDING=disable PGSSLMODE=verify-full PGSSLROOTCERT=/run/tls/ca.crt
|
||||
psql -X -h db -U messenger_worker "$POSTGRES_DB" -Atc "
|
||||
SELECT has_table_privilege(current_user, '\''attachments'\'', '\''UPDATE'\'')
|
||||
AND NOT has_table_privilege(current_user, '\''users'\'', '\''SELECT'\'')
|
||||
AND NOT has_schema_privilege(current_user, '\''public'\'', '\''CREATE'\'');
|
||||
" | grep -qx t
|
||||
'
|
||||
printf '%s\n' 'PASS PostgreSQL: worker can update attachments; user table and schema creation denied'
|
||||
docker compose exec -T broker rabbitmqctl -q list_permissions -p / |
|
||||
awk -F '\t' '
|
||||
$1=="messenger_dispatcher" && $2=="^attachments\\.v1$" && $3=="^amq\\.default$" && $4=="^$" { p=1 }
|
||||
$1=="messenger_worker" && $2=="^attachments\\.v1$" && $3=="^$" && $4=="^attachments\\.v1$" { c=1 }
|
||||
END {exit !(p && c)}
|
||||
'
|
||||
printf '%s\n' 'PASS RabbitMQ: producer can publish and consumer can read attachments.v1'
|
||||
docker compose exec -T redis sh -ec '
|
||||
export REDISCLI_AUTH=$(cat /run/secrets/redis_password)
|
||||
redis-cli --tls --cacert /run/tls/ca.crt -h redis --user messenger CONFIG GET dir | grep -q NOPERM
|
||||
'
|
||||
printf '%s\n' 'PASS Redis: application role cannot read server configuration'
|
||||
@@ -0,0 +1,46 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Negative HTTP cases beyond the supplied smoke."""
|
||||
import ssl
|
||||
import socket
|
||||
import sys
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / "tests"))
|
||||
from client import Client, arguments, run # noqa: E402
|
||||
|
||||
|
||||
def main():
|
||||
args = arguments(__doc__)
|
||||
client = Client(args.base_url, args.ca_file)
|
||||
alice, _, eve, chat = client.bootstrap()
|
||||
upload = f"/api/v1/chats/{chat['id']}/attachments?filename=probe.png"
|
||||
headers = {"Content-Type": "image/png"}
|
||||
client.call("POST", upload, 413, actor=alice, raw=b"x" * (10 * 1024 * 1024 + 1), headers=headers)
|
||||
client.call("POST", upload, 415, actor=alice, raw=b"not an image", headers=headers)
|
||||
attachment, _ = client.attachment(alice, chat)
|
||||
path = "/api/v1/attachments/" + attachment["id"]
|
||||
spoofed = {"X-User-Id": alice["id"]}
|
||||
for suffix in ("", "/content"):
|
||||
client.call("GET", path + suffix, 404, actor=eve, headers=spoofed)
|
||||
other = client.call("POST", "/api/v1/chats", 201, actor=eve,
|
||||
body={"title": "ACL probe", "member_ids": [eve["id"], alice["id"]]}, schema="Chat")["json"]
|
||||
client.call("POST", f"/api/v1/chats/{other['id']}/messages", 404, actor=eve,
|
||||
body={"text": "", "attachment_ids": [attachment["id"]]},
|
||||
headers={"Idempotency-Key": uuid.uuid4().hex, **spoofed})
|
||||
context = ssl.create_default_context(cafile=args.ca_file)
|
||||
from urllib.parse import urlsplit
|
||||
endpoint = urlsplit(client.base)
|
||||
with socket.create_connection((endpoint.hostname, endpoint.port or 443), timeout=5) as connection:
|
||||
try:
|
||||
with context.wrap_socket(connection, server_hostname="wrong.lab.invalid"):
|
||||
raise AssertionError("TLS accepted a wrong hostname")
|
||||
except ssl.SSLCertVerificationError as error:
|
||||
if error.verify_code != 62:
|
||||
raise
|
||||
print("PASS: upload size/MIME, forged identity, cross-chat attachment, wrong TLS hostname")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
run(main)
|
||||
@@ -0,0 +1,46 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Inspect runtime constraints without printing inspect output or secrets."""
|
||||
import json
|
||||
import subprocess
|
||||
|
||||
APP_ROLES = {"api", "worker", "dispatcher", "upload-cleanup"}
|
||||
READ_ONLY = APP_ROLES | {"haproxy", "otel-collector", "prometheus", "loki", "tempo", "grafana"}
|
||||
ALLOWED_PORTS = {("haproxy", "8443/tcp"), ("grafana", "3000/tcp"),
|
||||
("openbao", "8200/tcp"), ("s3", "9000/tcp")}
|
||||
|
||||
|
||||
def main():
|
||||
ids = subprocess.check_output(["docker", "compose", "ps", "-q"], text=True).split()
|
||||
assert ids, "No running Compose services"
|
||||
services = json.loads(subprocess.check_output(["docker", "inspect", *ids], text=True))
|
||||
for service in services:
|
||||
name = service["Config"]["Labels"]["com.docker.compose.service"]
|
||||
config = service["HostConfig"]
|
||||
assert not config["Privileged"] and config["NetworkMode"] != "host", name
|
||||
mounts = [mount["Destination"] for mount in service["Mounts"]]
|
||||
assert not any("docker.sock" in path for path in mounts), name
|
||||
for port, bindings in (config.get("PortBindings") or {}).items():
|
||||
assert (name, port) in ALLOWED_PORTS, (name, port)
|
||||
assert all(binding["HostIp"] == "127.0.0.1" for binding in bindings), name
|
||||
if name in READ_ONLY:
|
||||
assert config["ReadonlyRootfs"] and config["CapDrop"] == ["ALL"], name
|
||||
assert "no-new-privileges:true" in config["SecurityOpt"], name
|
||||
assert config["Memory"] > 0 and config["NanoCpus"] > 0, name
|
||||
assert config["Tmpfs"], name
|
||||
if name == "worker":
|
||||
env = dict(item.split("=", 1) for item in service["Config"]["Env"] if "=" in item)
|
||||
assert 0 < int(env["JOB_TIMEOUT_SECONDS"]) <= 60, name
|
||||
assert config["Memory"] <= 512 * 1024 * 1024 and config["NanoCpus"] <= 10**9, name
|
||||
if name in APP_ROLES:
|
||||
assert service["Config"]["User"] == "app:app", name
|
||||
assert config["CapDrop"] == ["ALL"] and not config["CapAdd"], name
|
||||
assert config["Memory"] > 0 and config["NanoCpus"] > 0, name
|
||||
assert config["ReadonlyRootfs"] and "/run/runtime-secrets" in config["Tmpfs"], name
|
||||
assert not any(path.startswith("/run/secrets") for path in mounts), name
|
||||
assert not any("BAO_TOKEN=" in field for field in service["Config"]["Env"]), name
|
||||
assert APP_ROLES <= {svc["Config"]["Labels"]["com.docker.compose.service"] for svc in services}
|
||||
print("PASS: local ports, read-only app/ingress/telemetry, no-new-privileges, dropped capabilities, worker CPU/RAM/time budget")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,63 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Confirm metrics, logs and traces reach their HTTPS stores."""
|
||||
import json
|
||||
import ssl
|
||||
import subprocess
|
||||
import time
|
||||
from pathlib import Path
|
||||
from urllib.parse import urlencode
|
||||
from urllib.request import urlopen
|
||||
from urllib.error import HTTPError
|
||||
import http.client
|
||||
|
||||
TLS = Path(__file__).resolve().parents[1] / "secrets/tls"
|
||||
|
||||
|
||||
def query(service, port, path, params, *, allow_missing=False):
|
||||
context = ssl.create_default_context(cafile=str(TLS / "lab7-ca.crt"))
|
||||
context.load_cert_chain(TLS / "grafana-reader.crt", TLS / "grafana-reader.key")
|
||||
container = subprocess.check_output(["docker", "compose", "ps", "-q", service], text=True).strip()
|
||||
address = subprocess.check_output(["docker", "inspect", "-f",
|
||||
'{{(index .NetworkSettings.Networks "highload-lab7_telemetry").IPAddress}}', container],
|
||||
text=True).strip()
|
||||
connection = http.client.HTTPSConnection(service, port, context=context, timeout=5)
|
||||
create_connection = connection._create_connection
|
||||
connection._create_connection = lambda _, *args: create_connection((address, port), *args)
|
||||
try:
|
||||
connection.request("GET", f"{path}?{urlencode(params)}")
|
||||
response = connection.getresponse()
|
||||
if allow_missing and response.status == 404:
|
||||
return None
|
||||
assert response.status == 200, (service, response.status)
|
||||
return json.load(response)
|
||||
finally:
|
||||
connection.close()
|
||||
|
||||
|
||||
def main():
|
||||
try:
|
||||
with urlopen("https://localhost:8443/api/v1/users/me", context=ssl.create_default_context(
|
||||
cafile=str(TLS / "lab7-ingress-ca.crt")), timeout=5):
|
||||
raise AssertionError("Anonymous request should be rejected")
|
||||
except HTTPError as error:
|
||||
assert error.code == 401
|
||||
since = int(time.time()) - 10
|
||||
for _ in range(30):
|
||||
now = int(time.time())
|
||||
metrics = query("prometheus", 9090, "/api/v1/query", {
|
||||
"query": 'up{job="otel-collector"} == 1'})["data"]["result"]
|
||||
logs = query("loki", 3100, "/loki/api/v1/query_range", {
|
||||
"query": '{service_name="messenger-api"}', "start": since * 10**9,
|
||||
"end": now * 10**9, "limit": 1})["data"]["result"]
|
||||
traces = query("tempo", 3200, "/api/search", {
|
||||
"q": '{ resource.service.name = "messenger-api" }',
|
||||
"start": since, "end": now, "limit": 1}).get("traces")
|
||||
if metrics and logs and traces:
|
||||
print("PASS: Prometheus scrape up; fresh messenger API log and trace found through mTLS")
|
||||
return
|
||||
time.sleep(2)
|
||||
raise SystemExit("Telemetry signal missing within 60 seconds")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,75 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Verify trusted TLS on data-plane endpoints, from the Linux Docker host."""
|
||||
import json
|
||||
import socket
|
||||
import ssl
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
|
||||
TLS = Path(__file__).resolve().parents[1] / "secrets/tls"
|
||||
ENDPOINTS = (
|
||||
("s3", 9000, None, False),
|
||||
("broker", 5671, "messenger_worker", False),
|
||||
("redis", 6379, None, False),
|
||||
("db", 5432, None, True),
|
||||
("otel-collector", 4318, "otel-api", False),
|
||||
("otel-collector", 8889, "prometheus-scraper", False),
|
||||
("prometheus", 9090, "grafana-reader", False),
|
||||
("loki", 3100, "collector-exporter", False),
|
||||
("loki", 3100, "grafana-reader", False),
|
||||
("tempo", 4318, "collector-exporter", False),
|
||||
("tempo", 3200, "grafana-reader", False),
|
||||
("grafana", 3000, None, False),
|
||||
("openbao", 8200, None, False),
|
||||
)
|
||||
|
||||
|
||||
def probe(address, service, port, identity, pg, ca):
|
||||
context = ssl.create_default_context(cafile=str(TLS / f"{ca}.crt"))
|
||||
if identity:
|
||||
context.load_cert_chain(TLS / f"{identity}.crt", TLS / f"{identity}.key")
|
||||
with socket.create_connection((address, port), timeout=5) as conn:
|
||||
if pg:
|
||||
conn.sendall(bytes.fromhex("0000000804d2162f"))
|
||||
assert conn.recv(1) == b"S", "PostgreSQL did not accept TLS negotiation"
|
||||
with context.wrap_socket(conn, server_hostname=service) as encrypted:
|
||||
assert encrypted.version() in {"TLSv1.2", "TLSv1.3"}
|
||||
if identity or (service, port) == ("otel-collector", 4318):
|
||||
encrypted.sendall(b"GET / HTTP/1.0\r\n\r\n")
|
||||
encrypted.recv(1)
|
||||
|
||||
|
||||
def main():
|
||||
for service, port, identity, pg in ENDPOINTS:
|
||||
container = subprocess.check_output(["docker", "compose", "ps", "-q", service], text=True).strip()
|
||||
networks = json.loads(subprocess.check_output(
|
||||
["docker", "inspect", "-f", "{{json .NetworkSettings.Networks}}", container], text=True))
|
||||
address = next(network["IPAddress"] for network in networks.values() if network["IPAddress"])
|
||||
probe(address, service, port, identity, pg, "lab7-ca")
|
||||
try:
|
||||
probe(address, service, port, identity, pg, "untrusted-ca")
|
||||
except ssl.SSLCertVerificationError:
|
||||
pass
|
||||
else:
|
||||
raise AssertionError(f"{service}:{port} accepted a foreign CA")
|
||||
if (service, port) == ("otel-collector", 4318):
|
||||
try:
|
||||
probe(address, service, port, None, pg, "lab7-ca")
|
||||
except ssl.SSLError:
|
||||
print("PASS Collector: missing client certificate rejected")
|
||||
else:
|
||||
raise AssertionError("Collector accepted an anonymous client")
|
||||
if identity:
|
||||
foreign = "otel-wrong-role" if (service, port) == ("otel-collector", 4318) else "wrong-role"
|
||||
try:
|
||||
probe(address, service, port, foreign, pg, "lab7-ca")
|
||||
except ssl.SSLError:
|
||||
pass
|
||||
else:
|
||||
raise AssertionError(f"{service}:{port} accepted a foreign client identity")
|
||||
print(f"PASS {service}:{port}: verified service SAN; foreign CA rejected"
|
||||
+ ("; foreign client identity rejected" if identity else ""))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,91 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Exercise AMQP EXTERNAL authentication and role ACLs on new connections."""
|
||||
import socket
|
||||
import ssl
|
||||
import struct
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
|
||||
TLS = Path(__file__).resolve().parents[1] / "secrets/tls"
|
||||
|
||||
|
||||
def read_exact(conn, size):
|
||||
result = bytearray()
|
||||
while len(result) < size:
|
||||
chunk = conn.recv(size - len(result))
|
||||
if not chunk:
|
||||
raise EOFError("RabbitMQ closed the connection")
|
||||
result.extend(chunk)
|
||||
return bytes(result)
|
||||
|
||||
|
||||
def read_method(conn):
|
||||
kind, _, size = struct.unpack(">BHI", read_exact(conn, 7))
|
||||
assert kind == 1 and size >= 4
|
||||
body = read_exact(conn, size)
|
||||
assert read_exact(conn, 1) == b"\xce"
|
||||
return struct.unpack(">HH", body[:4]), body[4:]
|
||||
|
||||
|
||||
def send_method(conn, channel, klass, method, args):
|
||||
body = struct.pack(">HH", klass, method) + args
|
||||
conn.sendall(struct.pack(">BHI", 1, channel, len(body)) + body + b"\xce")
|
||||
|
||||
|
||||
def short(value):
|
||||
return bytes([len(value)]) + value
|
||||
|
||||
|
||||
def authenticate(conn):
|
||||
conn.sendall(b"AMQP\x00\x00\x09\x01")
|
||||
method, data = read_method(conn)
|
||||
assert method == (10, 10) and b"EXTERNAL" in data
|
||||
send_method(conn, 0, 10, 11, b"\x00" * 4 + short(b"EXTERNAL") + b"\x00" * 4 + short(b"en_US"))
|
||||
try:
|
||||
method, data = read_method(conn)
|
||||
except EOFError:
|
||||
return False
|
||||
if method == (10, 50):
|
||||
assert struct.unpack_from(">H", data)[0] == 403
|
||||
return False
|
||||
assert method == (10, 30)
|
||||
send_method(conn, 0, 10, 31, data)
|
||||
send_method(conn, 0, 10, 40, short(b"/") + short(b"") + b"\x00")
|
||||
assert read_method(conn)[0] == (10, 41)
|
||||
return True
|
||||
|
||||
|
||||
def main():
|
||||
container = subprocess.check_output(["docker", "compose", "ps", "-q", "broker"], text=True).strip()
|
||||
address = subprocess.check_output(["docker", "inspect", "-f",
|
||||
'{{(index .NetworkSettings.Networks "highload-lab7_data").IPAddress}}', container], text=True).strip()
|
||||
for identity in (None, "wrong-role", "messenger_unregistered", "messenger_worker", "messenger_dispatcher"):
|
||||
context = ssl.create_default_context(cafile=str(TLS / "lab7-ca.crt"))
|
||||
if identity:
|
||||
context.load_cert_chain(TLS / f"{identity}.crt", TLS / f"{identity}.key")
|
||||
with socket.create_connection((address, 5671), timeout=5) as tcp:
|
||||
try:
|
||||
with context.wrap_socket(tcp, server_hostname="broker") as conn:
|
||||
admitted = authenticate(conn)
|
||||
assert admitted == (identity in ("messenger_worker", "messenger_dispatcher"))
|
||||
if admitted:
|
||||
send_method(conn, 1, 20, 10, short(b""))
|
||||
assert read_method(conn)[0] == (20, 11)
|
||||
# Worker cannot publish. Dispatcher cannot read using basic.get.
|
||||
if identity == "messenger_worker":
|
||||
send_method(conn, 1, 60, 40, b"\x00\x00" + short(b"") + short(b"attachments.v1") + b"\x00")
|
||||
conn.sendall(struct.pack(">BHIHHQH", 2, 1, 14, 60, 0, 0, 0) + b"\xce")
|
||||
else:
|
||||
send_method(conn, 1, 60, 70, b"\x00\x00" + short(b"attachments.v1") + b"\x01")
|
||||
method, data = read_method(conn)
|
||||
assert method == (20, 40) and struct.unpack_from(">H", data)[0] == 403
|
||||
except ssl.SSLError:
|
||||
if identity not in (None, "wrong-role"):
|
||||
raise
|
||||
else:
|
||||
assert identity not in (None, "wrong-role"), "TLS accepted an untrusted peer"
|
||||
print(f"PASS: {identity or 'no certificate'}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,118 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Rotate a real API database password and OpenBao client leaf on the lab stand."""
|
||||
import json
|
||||
import os
|
||||
import secrets
|
||||
import ssl
|
||||
import subprocess
|
||||
import time
|
||||
from pathlib import Path
|
||||
from urllib.error import HTTPError
|
||||
from urllib.parse import urlsplit, urlunsplit
|
||||
|
||||
from check_bao_roles import request
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
FILES = ROOT / "secrets"
|
||||
TLS = FILES / "tls"
|
||||
CA = TLS / "lab7-ca.crt"
|
||||
|
||||
|
||||
def run(*args, **kwargs):
|
||||
return subprocess.run(args, cwd=ROOT, check=True, **kwargs)
|
||||
|
||||
|
||||
def identity(cert, key):
|
||||
context = ssl.create_default_context(cafile=str(CA))
|
||||
context.load_cert_chain(cert, key)
|
||||
return context
|
||||
|
||||
|
||||
def main():
|
||||
admin = json.loads((FILES / "bao-operator.json").read_text())["root_token"]
|
||||
base = ssl.create_default_context(cafile=str(CA))
|
||||
old = identity(TLS / "bao-api.crt", TLS / "bao-api.key")
|
||||
login = lambda ctx: request(ctx, "auth/cert/login", body={"name": "api"})["auth"]
|
||||
policy_path = "auth/cert/certs/api"
|
||||
secret_path = "messenger/data/api"
|
||||
old_auth = login(old)
|
||||
values = request(old, secret_path, token=old_auth["client_token"])["data"]["data"]
|
||||
old_password = urlsplit(values["database_url"]).password
|
||||
password = secrets.token_hex(24)
|
||||
for name in ("bao-api-next.crt", "bao-api-next.key"):
|
||||
if (TLS / name).exists():
|
||||
(TLS / name).unlink()
|
||||
run("openssl", "req", "-new", "-newkey", "ed25519", "-nodes", "-subj", "/CN=bao-api",
|
||||
"-keyout", str(TLS / "bao-api-next.key"), "-out", str(TLS / "bao-api-next.csr"),
|
||||
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
|
||||
run("openssl", "x509", "-req", "-in", str(TLS / "bao-api-next.csr"), "-CA", str(CA),
|
||||
"-CAkey", str(TLS / "lab7-ca.key"), "-CAcreateserial", "-days", "30",
|
||||
"-extfile", "/dev/stdin", "-out", str(TLS / "bao-api-next.crt"), input=
|
||||
"basicConstraints=critical,CA:FALSE\nkeyUsage=critical,digitalSignature\nextendedKeyUsage=clientAuth\n"
|
||||
"subjectAltName=URI:urn:lab7:bao:api\nsubjectKeyIdentifier=hash\nauthorityKeyIdentifier=keyid:always\n",
|
||||
text=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
|
||||
new_cert, new_key = TLS / "bao-api-next.crt", TLS / "bao-api-next.key"
|
||||
new = identity(new_cert, new_key)
|
||||
leaf = new_cert.read_text()
|
||||
|
||||
def role(ttl):
|
||||
request(base, policy_path, token=admin, status=204, body={
|
||||
"certificate": leaf, "token_policies": ["api"], "token_no_default_policy": True,
|
||||
"token_ttl": ttl, "token_max_ttl": ttl,
|
||||
})
|
||||
|
||||
def db_login(candidate, allowed):
|
||||
result = subprocess.run(["docker", "compose", "exec", "-T", "db", "sh", "-ec",
|
||||
'IFS= read -r PGPASSWORD; export PGPASSWORD PGCHANNELBINDING=disable '
|
||||
'PGSSLMODE=verify-full PGSSLROOTCERT=/run/tls/ca.crt; '
|
||||
'psql -X -h db -U messenger_api "$POSTGRES_DB" -Atc "SELECT 1"'],
|
||||
cwd=ROOT, input=candidate+"\n", text=True, capture_output=True)
|
||||
assert (result.returncode == 0 and result.stdout.strip() == "1") == allowed
|
||||
if not allowed:
|
||||
assert "password authentication failed" in result.stderr
|
||||
|
||||
role("3s")
|
||||
try:
|
||||
request(old, "auth/cert/login", body={"name": "api"}, status=400)
|
||||
request(base, "auth/token/revoke-accessor", token=admin, status=204,
|
||||
body={"accessor": old_auth["accessor"]})
|
||||
request(old, secret_path, token=old_auth["client_token"], status=403)
|
||||
auth = login(new)
|
||||
request(new, secret_path, token=auth["client_token"])
|
||||
time.sleep(auth["lease_duration"] + 1)
|
||||
request(new, secret_path, token=auth["client_token"], status=403)
|
||||
print("PASS: replaced leaf, revoked old token, expired short-lived token")
|
||||
finally:
|
||||
role("15m")
|
||||
for source, target in ((new_cert, TLS / "bao-api.crt"), (new_key, TLS / "bao-api.key")):
|
||||
source.chmod(0o644)
|
||||
os.replace(source, target)
|
||||
(TLS / "bao-api-next.csr").unlink(missing_ok=True)
|
||||
|
||||
db_login(old_password, True)
|
||||
run("docker", "compose", "exec", "-T", "db", "sh", "-ec",
|
||||
'psql -X -v ON_ERROR_STOP=1 -U "$POSTGRES_USER" "$POSTGRES_DB"',
|
||||
text=True, input=f"ALTER ROLE messenger_api PASSWORD '{password}';\n", stdout=subprocess.DEVNULL)
|
||||
parts = urlsplit(values["database_url"])
|
||||
values["database_url"] = urlunsplit(parts._replace(netloc=f"{parts.username}:{password}@{parts.hostname}:{parts.port}"))
|
||||
request(base, secret_path, token=admin, status=200, body={"data": values})
|
||||
for path, content in (("database_api_password", password), ("database_api_url", values["database_url"])):
|
||||
(FILES / path).write_text(content + "\n")
|
||||
db_login(old_password, False)
|
||||
db_login(password, True)
|
||||
run("docker", "compose", "up", "-d", "--no-deps", "--no-build", "--force-recreate", "api")
|
||||
run("./scripts/wait_ingress.sh")
|
||||
print("PASS: database secret rotated, old password refused, API ready without rebuilding")
|
||||
|
||||
try:
|
||||
request(base, "sys/seal", token=admin, status=204, body={})
|
||||
request(new, "auth/cert/login", body={"name": "api"}, status=503)
|
||||
run("./scripts/wait_ingress.sh")
|
||||
print("PASS: sealed OpenBao denies new login; running API remains ready using loaded credentials")
|
||||
finally:
|
||||
run("python3", "scripts/bao_operator.py", "unseal")
|
||||
run("python3", "scripts/check_bao_roles.py")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,66 @@
|
||||
"""Verify rehash in PostgreSQL and measure rejected concurrent login work."""
|
||||
import base64
|
||||
import concurrent.futures
|
||||
import json
|
||||
import os
|
||||
import ssl
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
from urllib.error import HTTPError
|
||||
from urllib.request import Request, urlopen
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / "tests"))
|
||||
from client import Client
|
||||
|
||||
|
||||
def main():
|
||||
ca = "secrets/tls/lab7-ingress-ca.crt"
|
||||
endpoint = "https://localhost:8443"
|
||||
api = Client(endpoint, ca)
|
||||
actor = api.user("rehash")
|
||||
registration = actor["registration"]
|
||||
user_id = actor["id"]
|
||||
|
||||
def sql(statement):
|
||||
return subprocess.check_output(["docker", "compose", "exec", "-T", "db", "sh", "-ec",
|
||||
'psql -X -v ON_ERROR_STOP=1 -U "$POSTGRES_USER" "$POSTGRES_DB" -At'],
|
||||
text=True, input=statement, cwd=ROOT).strip()
|
||||
|
||||
# Derive a synthetic legacy hash with the same Go implementation in the test image.
|
||||
legacy = subprocess.check_output(["docker", "compose", "--profile", "test", "run", "--rm", "-T",
|
||||
"--no-deps", "-e", "LAB7_REHASH_FIXTURE", "backend-tests",
|
||||
"go", "test", "./backend/services", "-run", "^TestPrintLegacyFixture$", "-v"],
|
||||
cwd=ROOT, text=True, env=os.environ | {"LAB7_REHASH_FIXTURE": registration["password"]})
|
||||
old = next(line for line in legacy.splitlines() if line.startswith("$argon2id$"))
|
||||
sql(f"UPDATE users SET password_hash=convert_to('{old}','UTF8') WHERE id='{user_id}';")
|
||||
auth = "Basic " + base64.b64encode(
|
||||
f"{registration['username']}:{registration['password']}".encode()).decode()
|
||||
api.call("POST", "/api/v1/auth/sessions", 201, headers={"Authorization": auth}, schema="Session")
|
||||
assert sql(f"SELECT split_part(convert_from(password_hash,'UTF8'),'$',4) FROM users WHERE id='{user_id}';") == "m=65536,t=3,p=2"
|
||||
print("PASS: successful login upgraded the stored legacy Argon2id hash")
|
||||
context = ssl.create_default_context(cafile=ca)
|
||||
|
||||
def login(_):
|
||||
started = time.monotonic()
|
||||
req = Request(endpoint + "/api/v1/auth/sessions", data=b"", headers={"Authorization": auth})
|
||||
try:
|
||||
with urlopen(req, context=context, timeout=10) as response:
|
||||
return response.status, time.monotonic()-started
|
||||
except HTTPError as error:
|
||||
assert error.code == 429 and error.headers["Retry-After"]
|
||||
return error.code, time.monotonic()-started
|
||||
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=24) as pool:
|
||||
result = list(pool.map(login, range(24)))
|
||||
status = {code: sum(c == code for c, _ in result) for code in (201, 429)}
|
||||
assert all(status.values()), status
|
||||
latencies = sorted(duration for _, duration in result)
|
||||
print(json.dumps({"requests": len(result), "status": status,
|
||||
"p50_ms": round(latencies[11]*1000), "max_ms": round(latencies[-1]*1000)}))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,56 @@
|
||||
"""Send a synthetic marker as a credential and check exported signals for it."""
|
||||
import base64
|
||||
import json
|
||||
import secrets
|
||||
import ssl
|
||||
import subprocess
|
||||
import time
|
||||
from urllib.error import HTTPError
|
||||
from urllib.request import Request, urlopen
|
||||
|
||||
from check_grade3_signals import TLS, query
|
||||
|
||||
|
||||
def main():
|
||||
marker = "lab7-secret-" + secrets.token_hex(16)
|
||||
encoded = base64.b64encode(("lab7_probe:" + marker).encode()).decode()
|
||||
context = ssl.create_default_context(cafile=str(TLS / "lab7-ingress-ca.crt"))
|
||||
started = int(time.time()) - 1
|
||||
req = Request("https://localhost:8443/api/v1/auth/sessions", data=b"",
|
||||
headers={"Authorization": "Basic " + encoded})
|
||||
try:
|
||||
with urlopen(req, context=context, timeout=5):
|
||||
raise AssertionError("Synthetic invalid login succeeded")
|
||||
except HTTPError as error:
|
||||
assert error.code == 401
|
||||
request_id = error.headers["X-Request-Id"]
|
||||
assert request_id
|
||||
for _ in range(30):
|
||||
now = int(time.time()) + 1
|
||||
selector = '{service_name="messenger-api"} | request_id=`' + request_id + '`'
|
||||
window = {"start": started * 10**9, "end": now * 10**9, "limit": 10}
|
||||
logs = query("loki", 3100, "/loki/api/v1/query_range",
|
||||
{**window, "query": selector})
|
||||
correlated = query("loki", 3100, "/loki/api/v1/query_range",
|
||||
{**window, "query": selector + ' | line_format "{{.trace_id}}"'})
|
||||
trace_id = next((entry[1] for stream in correlated["data"]["result"]
|
||||
for entry in stream["values"] if len(entry[1]) == 32), None)
|
||||
if trace_id:
|
||||
trace = query("tempo", 3200, "/api/traces/" + trace_id, {}, allow_missing=True)
|
||||
metrics = query("prometheus", 9090, "/api/v1/query", {
|
||||
"query": 'messenger_http_requests_total{http_route="/api/v1/auth/sessions",http_status_class="4xx"}',
|
||||
})["data"]["result"]
|
||||
if trace and metrics:
|
||||
break
|
||||
time.sleep(2)
|
||||
else:
|
||||
raise AssertionError("Missing correlated log, trace or ingress metric for synthetic request")
|
||||
container_logs = subprocess.check_output(
|
||||
["docker", "compose", "logs", "--since", str(started), "api", "otel-collector"], text=True)
|
||||
for output in (json.dumps(logs), json.dumps(trace), json.dumps(metrics), container_logs):
|
||||
assert marker not in output and encoded not in output
|
||||
print("PASS: synthetic Basic secret absent from correlated log, correlated trace, metric labels and container logs")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,56 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Probe each API replica directly from the Docker host on Linux."""
|
||||
import http.client
|
||||
import json
|
||||
import ssl
|
||||
import socket
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
TLS = ROOT / "secrets/tls"
|
||||
|
||||
|
||||
def check(address, port, identity, error):
|
||||
context = ssl.create_default_context(cafile=str(TLS / "lab7-ca.crt"))
|
||||
if identity:
|
||||
context.load_cert_chain(TLS / f"{identity}.crt", TLS / f"{identity}.key")
|
||||
with socket.create_connection((address, port), timeout=5) as tcp:
|
||||
try:
|
||||
with context.wrap_socket(tcp, server_hostname="api") as connection:
|
||||
connection.sendall(b"GET /health/live HTTP/1.1\r\nHost: api\r\nConnection: close\r\n\r\n")
|
||||
response = http.client.HTTPResponse(connection)
|
||||
response.begin()
|
||||
assert response.status == 200, response.status
|
||||
except ssl.SSLError as exc:
|
||||
if not error or error not in str(exc).lower():
|
||||
raise
|
||||
return
|
||||
if error:
|
||||
raise AssertionError(f"{identity or 'no certificate'} unexpectedly accepted")
|
||||
|
||||
|
||||
def main():
|
||||
config = subprocess.check_output(
|
||||
["docker", "compose", "config", "--format", "json"], cwd=ROOT, text=True
|
||||
)
|
||||
port = int(json.loads(config)["services"]["api"]["environment"]["APP_PORT"])
|
||||
replicas = subprocess.check_output(
|
||||
["docker", "compose", "ps", "-q", "api"], cwd=ROOT, text=True
|
||||
).split()
|
||||
assert len(replicas) >= 2, "Expected at least two running API replicas"
|
||||
for replica in replicas:
|
||||
address = subprocess.check_output(
|
||||
["docker", "inspect", "--format",
|
||||
'{{(index .NetworkSettings.Networks "highload-lab7_proxy").IPAddress}}', replica],
|
||||
text=True,
|
||||
).strip()
|
||||
check(address, port, "lb", None)
|
||||
check(address, port, None, "certificate required")
|
||||
check(address, port, "wrong-ca", "unknown ca")
|
||||
check(address, port, "wrong-role", "bad certificate")
|
||||
print(f"PASS {replica[:12]}: LB permitted; absent, foreign CA and disallowed SAN rejected")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,40 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Run against a quiet test ingress after at least 60 seconds without logins."""
|
||||
import argparse
|
||||
import ssl
|
||||
from urllib.error import HTTPError
|
||||
from urllib.request import Request, urlopen
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--base-url", default="https://localhost:8443")
|
||||
parser.add_argument("--ca-file", default="secrets/tls/lab7-ingress-ca.crt")
|
||||
args = parser.parse_args()
|
||||
context = ssl.create_default_context(cafile=args.ca_file)
|
||||
instances = set()
|
||||
for attempt in range(1, 14):
|
||||
request = Request(
|
||||
args.base_url.rstrip("/") + "/api/v1/auth/sessions",
|
||||
data=b"",
|
||||
headers={"X-Forwarded-For": f"192.0.2.{attempt}"},
|
||||
)
|
||||
try:
|
||||
with urlopen(request, context=context, timeout=5):
|
||||
raise AssertionError("Missing credentials unexpectedly accepted")
|
||||
except HTTPError as error:
|
||||
with error:
|
||||
instance = error.headers["X-Instance-Id"]
|
||||
assert instance, "Missing API instance identity"
|
||||
instances.add(instance)
|
||||
if attempt <= 12:
|
||||
assert error.code == 401, f"Attempt {attempt}: expected 401, got {error.code}"
|
||||
else:
|
||||
assert error.code == 429, f"Expected 429, got {error.code}"
|
||||
assert 0 < int(error.headers["Retry-After"]) <= 60
|
||||
assert len(instances) >= 2, "Expected requests to reach multiple API replicas"
|
||||
print(f"PASS: request 13 returned 429 with Retry-After across {len(instances)} API replicas")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Executable
+10
@@ -0,0 +1,10 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
replica=$(docker compose ps -q api | xargs docker inspect -f '{{.Name}} {{.Config.Hostname}}' |
|
||||
awk '$1=="/highload-lab7-api-2" {print $2}')
|
||||
test -n "$replica"
|
||||
trap 'docker compose start api >/dev/null' EXIT
|
||||
TARGET_INSTANCE_ID="$replica" ./scripts/stop-one.sh
|
||||
./scripts/wait_ingress.sh
|
||||
make test CA_FILE=secrets/tls/lab7-ingress-ca.crt
|
||||
@@ -0,0 +1,33 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Use a stand started with SESSION_TTL_SECONDS between 3 and 15."""
|
||||
import base64
|
||||
import datetime as dt
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "tests"))
|
||||
from client import Client, arguments, check, run, timestamp # noqa: E402
|
||||
|
||||
|
||||
def main():
|
||||
args = arguments(__doc__)
|
||||
client = Client(args.base_url, args.ca_file)
|
||||
actor = client.user("expiry")
|
||||
registration = actor["registration"]
|
||||
credentials = base64.b64encode(
|
||||
f"{registration['username']}:{registration['password']}".encode()
|
||||
).decode()
|
||||
session = client.call("POST", "/api/v1/auth/sessions", 201,
|
||||
headers={"Authorization": "Basic " + credentials}, schema="Session")["json"]
|
||||
actor["token"] = session["token"]
|
||||
client.call("GET", "/api/v1/users/me", actor=actor, schema="User")
|
||||
remaining = (timestamp(session["expires_at"]) - dt.datetime.now(dt.timezone.utc)).total_seconds()
|
||||
check(0 < remaining <= 15, "Start rehearsal API with a short SESSION_TTL_SECONDS")
|
||||
time.sleep(remaining + 0.3)
|
||||
client.call("GET", "/api/v1/users/me", 401, actor=actor)
|
||||
print("PASS: valid session accepted before expiry and rejected after expiry")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
run(main)
|
||||
@@ -0,0 +1,101 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Run level-3 checks on the local rehearsal stand; write sanitized Markdown."""
|
||||
import argparse
|
||||
import os
|
||||
import shlex
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
CA = "secrets/tls/lab7-ingress-ca.crt"
|
||||
OUTPUT = ROOT / "evidence"
|
||||
VALIDATION = {"CA_FILE": str(ROOT / CA)}
|
||||
CHECKS = {
|
||||
"grade3-https": ("Бизнес-API, проверка CA, logout и вложения по HTTPS", [
|
||||
["make", "check"], ["make", "test", "test-security", f"CA_FILE={CA}"],
|
||||
]),
|
||||
"grade3-mtls": ("Допуск только роли LB на каждой реплике", [
|
||||
["python3", "scripts/check_ingress_mtls.py"],
|
||||
]),
|
||||
"grade3-dependencies": ("TLS хранилищ, брокера и телеметрии", [
|
||||
["python3", "scripts/check_grade3_tls.py"],
|
||||
["./scripts/check_dependency_roles.sh"],
|
||||
]),
|
||||
"grade3-signals": ("Доставка логов, метрик и трейсов по защищённым связям", [
|
||||
["python3", "scripts/check_grade3_signals.py"],
|
||||
]),
|
||||
"grade3-openbao": ("Сервисная аутентификация и scoped policy OpenBao", [
|
||||
["python3", "scripts/check_bao_roles.py"],
|
||||
]),
|
||||
"grade3-runtime": ("Изоляция, порты, лимиты и локальное размещение секретов", [
|
||||
["python3", "scripts/check_grade3_runtime.py"],
|
||||
]),
|
||||
"grade3-http-security": ("Лимиты, MIME, чужие вложения и неверный TLS hostname", [
|
||||
["python3", "scripts/check_grade3_http.py", "--ca-file", CA],
|
||||
]),
|
||||
"grade3-resilience": ("Сохранность после перезапуска хранилищ и отказ API", [
|
||||
["sleep", "60"],
|
||||
["make", "test-persistence", "ACTION_SCRIPT=scripts/restart.sh", f"CA_FILE={CA}"],
|
||||
["make", "test-failover", "ACTION_SCRIPT=scripts/stop-one.sh", f"CA_FILE={CA}"],
|
||||
["docker", "compose", "start", "api"], ["./scripts/wait_ingress.sh"],
|
||||
["sleep", "60"],
|
||||
["make", "test-failover", "ACTION_SCRIPT=scripts/stop-one.sh", f"CA_FILE={CA}"],
|
||||
["docker", "compose", "start", "api"], ["./scripts/wait_ingress.sh"],
|
||||
]),
|
||||
"grade3-other-replica": ("Работа HTTPS API при остановленной второй реплике", [
|
||||
["sleep", "60"], ["./scripts/check_other_replica.sh"], ["./scripts/wait_ingress.sh"],
|
||||
]),
|
||||
"grade3-login": ("Общий лимит входа по адресу клиента", [
|
||||
["sleep", "60"], ["python3", "scripts/check_login_rate.py"],
|
||||
]),
|
||||
"grade3-expiry": ("Истечение сессии на стенде с коротким TTL", [
|
||||
["sleep", "60"],
|
||||
["env", "SESSION_TTL_SECONDS=5", "docker", "compose", "up", "-d", "--no-deps", "api"],
|
||||
["./scripts/wait_ingress.sh"],
|
||||
["python3", "scripts/check_session_expiry.py", "--ca-file", CA],
|
||||
["docker", "compose", "up", "-d", "--no-deps", "api"], ["./scripts/wait_ingress.sh"],
|
||||
]),
|
||||
"grade3-argon2": ("Стоимость проверки хеша Argon2id на лимитах API", [
|
||||
["docker", "compose", "--profile", "test", "build", "backend-tests"],
|
||||
["docker", "run", "--rm", "--network", "none", "--cpus", "0.5", "--memory", "512m",
|
||||
"-e", "GOMAXPROCS=1", "highload-lab7-backend-tests", "go", "test", "./backend/services",
|
||||
"-run", "^$", "-bench", "^BenchmarkPasswordVerification$", "-benchtime=3x", "-count=3"],
|
||||
]),
|
||||
}
|
||||
|
||||
|
||||
def main(checks=CHECKS):
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--only", nargs="+", choices=checks)
|
||||
selected = parser.parse_args().only or checks
|
||||
OUTPUT.mkdir(exist_ok=True)
|
||||
if "grade3-resilience" in selected:
|
||||
for name in ("restart", "failover"):
|
||||
path = ROOT / ".memory" / f"{name}-containers.txt"
|
||||
path.parent.mkdir(exist_ok=True)
|
||||
path.write_text("")
|
||||
VALIDATION[f"{name.upper()}_EVIDENCE_FILE"] = str(path)
|
||||
passed = True
|
||||
for slug in selected:
|
||||
title, commands = checks[slug]
|
||||
report = [f"# {title}", "", "Предпосылки: подготовленный, разблокированный учебный стенд; Docker Compose, Python и локальный CA.", ""]
|
||||
for command in commands:
|
||||
result = subprocess.run(command, cwd=ROOT, env=os.environ | VALIDATION,
|
||||
stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True)
|
||||
passed &= result.returncode == 0
|
||||
report.extend([f"`{shlex.join(command)}`", "", f"Код завершения: {result.returncode}.", "",
|
||||
"```text", "\n".join(line.rstrip() for line in result.stdout.splitlines()).rstrip(), "```", ""])
|
||||
if slug == "grade3-resilience":
|
||||
for name in ("restart", "failover"):
|
||||
rows = Path(VALIDATION[f"{name.upper()}_EVIDENCE_FILE"]).read_text().strip()
|
||||
report.extend([f"Состояния контейнеров ({name}):", "", "```text", rows, "```", ""])
|
||||
report.extend(["Итог: " + ("все команды завершились успешно." if all(
|
||||
line == "Код завершения: 0." for line in report if line.startswith("Код завершения:")
|
||||
) else "есть команды с ошибками."), ""])
|
||||
(OUTPUT / f"{slug}.md").write_text("\n".join(report), encoding="utf-8")
|
||||
print(f"Recorded {slug}", flush=True)
|
||||
raise SystemExit(0 if passed else 1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,35 @@
|
||||
"""Collect evidence for the level-4 criteria on the rehearsal stack."""
|
||||
from collect_grade3 import main
|
||||
|
||||
CA = "secrets/tls/lab7-ingress-ca.crt"
|
||||
CHECKS = {
|
||||
"grade4-mtls": ("Проверка ролей RabbitMQ и Collector", [
|
||||
["python3", "scripts/check_grade4_broker.py"],
|
||||
["python3", "scripts/check_grade3_tls.py"],
|
||||
["python3", "scripts/check_grade3_signals.py"],
|
||||
]),
|
||||
"grade4-login": ("Rehash в PostgreSQL и бюджет защиты входа", [
|
||||
["docker", "compose", "--profile", "test", "build", "backend-tests"],
|
||||
["sleep", "60"], ["python3", "scripts/check_grade4_login.py"],
|
||||
]),
|
||||
"grade4-lifecycle": ("Ротация секрета и leaf, отзыв, expiry и seal", [
|
||||
["python3", "scripts/check_grade4_lifecycle.py"],
|
||||
]),
|
||||
"grade4-hardening": ("Права контейнеров, лимиты и эксплуатационные границы", [
|
||||
["python3", "scripts/check_grade3_runtime.py"],
|
||||
]),
|
||||
"grade4-audit": ("Сетевой аудит, ACL и отсутствие контрольного секрета в телеметрии", [
|
||||
["python3", "scripts/check_grade3_tls.py"],
|
||||
["python3", "scripts/check_grade4_broker.py"],
|
||||
["./scripts/check_dependency_roles.sh"],
|
||||
["python3", "scripts/check_bao_roles.py"],
|
||||
["sleep", "60"], ["python3", "scripts/check_grade3_http.py", "--ca-file", CA],
|
||||
["python3", "scripts/check_grade4_telemetry.py"],
|
||||
]),
|
||||
"grade4-scan": ("Сканирование зависимостей и образов", [
|
||||
["./scripts/scan_grade4.sh"],
|
||||
]),
|
||||
}
|
||||
|
||||
if __name__ == "__main__":
|
||||
main(CHECKS)
|
||||
@@ -0,0 +1,66 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Configure an initialized, unsealed local OpenBao using an operator token."""
|
||||
import argparse
|
||||
import getpass
|
||||
import json
|
||||
import ssl
|
||||
from pathlib import Path
|
||||
from urllib.error import HTTPError
|
||||
from urllib.request import Request, urlopen
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
SECRETS = ROOT / "secrets"
|
||||
TLS = SECRETS / "tls"
|
||||
|
||||
|
||||
def main():
|
||||
context = ssl.create_default_context(cafile=str(TLS / "lab7-ca.crt"))
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--operator-file", type=Path)
|
||||
args = parser.parse_args()
|
||||
token = (json.loads(args.operator_file.read_text())["root_token"]
|
||||
if args.operator_file else getpass.getpass("OpenBao operator token: "))
|
||||
|
||||
def request(path, body=None):
|
||||
data = None if body is None else json.dumps(body).encode()
|
||||
req = Request(
|
||||
"https://localhost:8200/v1/" + path,
|
||||
data=data,
|
||||
headers={"Content-Type": "application/json", "X-Vault-Token": token},
|
||||
)
|
||||
try:
|
||||
with urlopen(req, context=context, timeout=5) as response:
|
||||
return json.load(response) if response.status != 204 else {}
|
||||
except HTTPError as error:
|
||||
raise SystemExit(f"OpenBao {path}: HTTP {error.code}")
|
||||
|
||||
if "cert/" not in request("sys/auth"):
|
||||
request("sys/auth/cert", {"type": "cert"})
|
||||
if "messenger/" not in request("sys/mounts"):
|
||||
request("sys/mounts/messenger", {"type": "kv", "options": {"version": "2"}})
|
||||
|
||||
for role in ("api", "worker", "dispatcher", "cleanup"):
|
||||
request(f"sys/policies/acl/{role}", {
|
||||
"policy": f'path "messenger/data/{role}" {{ capabilities = ["read"] }}',
|
||||
})
|
||||
request(f"auth/cert/certs/{role}", {
|
||||
"certificate": (TLS / f"bao-{role}.crt").read_text(),
|
||||
"token_policies": [role],
|
||||
"token_no_default_policy": True,
|
||||
"token_ttl": "15m",
|
||||
"token_max_ttl": "15m",
|
||||
})
|
||||
|
||||
names = {"database_url": f"database_{role}_url"}
|
||||
if role == "api":
|
||||
names["cache_url"] = "cache_tls_url"
|
||||
if role != "dispatcher":
|
||||
for key in ("access_key_id", "secret_access_key"):
|
||||
names[f"s3_{key}"] = f"s3_{role}_{key}"
|
||||
values = {key: (SECRETS / name).read_text().strip() for key, name in names.items()}
|
||||
request(f"messenger/data/{role}", {"data": values})
|
||||
print(f"Configured OpenBao role {role}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,105 @@
|
||||
import os
|
||||
import secrets
|
||||
from pathlib import Path
|
||||
from urllib.parse import quote
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
|
||||
|
||||
def read_values(path):
|
||||
values = {}
|
||||
if path.exists():
|
||||
for line in path.read_text(encoding="utf-8").splitlines():
|
||||
line = line.strip()
|
||||
if line and not line.startswith("#") and "=" in line:
|
||||
key, value = line.split("=", 1)
|
||||
values[key.strip()] = value.strip().strip("\"'")
|
||||
return values
|
||||
|
||||
|
||||
def main():
|
||||
path = ROOT / ".env"
|
||||
existing = read_values(path)
|
||||
defaults = read_values(ROOT / ".env.example")
|
||||
defaults.update(
|
||||
POSTGRES_DB="messenger",
|
||||
POSTGRES_USER="messenger",
|
||||
POSTGRES_PASSWORD=secrets.token_hex(24),
|
||||
)
|
||||
defaults.update(API_HTTPS_PORT="8443", S3_ENDPOINT="https://s3:9000",
|
||||
OTEL_EXPORTER_OTLP_ENDPOINT="https://otel-collector:4318", UPLOAD_CLEANUP_GRACE_SECONDS="600", UPLOAD_TMP_DIR="/tmp/uploads",
|
||||
S3_PUBLIC_ENDPOINT="https://localhost:9000",
|
||||
S3_CORS_ORIGINS="http://127.0.0.1:5173,http://localhost:5173")
|
||||
values = defaults | existing
|
||||
for name, lower, upper in (
|
||||
("APP_PORT", 1, 65535),
|
||||
("API_REPLICAS", 2, 32),
|
||||
("SESSION_TTL_SECONDS", 1, 31536000),
|
||||
("IDEMPOTENCY_TTL_SECONDS", 1, 31536000),
|
||||
):
|
||||
try:
|
||||
valid = lower <= int(values[name]) <= upper
|
||||
except (ValueError, KeyError):
|
||||
valid = False
|
||||
if not valid:
|
||||
raise SystemExit(f"{name} must be between {lower} and {upper}")
|
||||
for name in (
|
||||
"POSTGRES_DB",
|
||||
"POSTGRES_USER",
|
||||
"POSTGRES_PASSWORD",
|
||||
"APP_HOST",
|
||||
"APP_ENV",
|
||||
):
|
||||
if not values[name]:
|
||||
raise SystemExit(f"{name} must not be empty")
|
||||
# Existing values, including credentials, are never replaced.
|
||||
missing = {key: value for key, value in values.items() if key not in existing}
|
||||
if missing:
|
||||
prefix = "\n" if path.exists() and path.stat().st_size else ""
|
||||
with path.open("a", encoding="utf-8") as file:
|
||||
file.write(
|
||||
prefix + "".join(f"{key}={value}\n" for key, value in missing.items())
|
||||
)
|
||||
os.chmod(path, 0o600)
|
||||
secret_dir = ROOT / "secrets"
|
||||
secret_dir.mkdir(mode=0o700, exist_ok=True)
|
||||
def create_secret(name, value):
|
||||
target = secret_dir / name
|
||||
if not target.exists():
|
||||
target.write_text(value + "\n", encoding="utf-8")
|
||||
target.chmod(0o644)
|
||||
return target.read_text(encoding="utf-8").strip()
|
||||
|
||||
create_secret("grafana_admin_password", secrets.token_hex(24))
|
||||
database_url = "postgres://{}:{}@db:5432/{}?sslmode=verify-full&sslrootcert=/run/tls/ca.crt".format(
|
||||
quote(values["POSTGRES_USER"], safe=""),
|
||||
quote(values["POSTGRES_PASSWORD"], safe=""),
|
||||
quote(values["POSTGRES_DB"], safe=""),
|
||||
)
|
||||
create_secret("broker_user", "messenger")
|
||||
create_secret("broker_password", secrets.token_hex(32))
|
||||
redis_password = create_secret("redis_password", secrets.token_hex(32))
|
||||
cache_url = f"rediss://messenger:{redis_password}@redis:6379/0"
|
||||
create_secret("redis_acl", "user default off\nuser messenger on >" + redis_password
|
||||
+ " ~messenger:* +ping +get +set +del +incr +expire +ttl +eval +evalsha +hello +client|setinfo")
|
||||
for name, content in (
|
||||
("database_tls_url", database_url),
|
||||
("cache_tls_url", cache_url),
|
||||
):
|
||||
create_secret(name, content)
|
||||
create_secret("s3_access_key_id", secrets.token_hex(12))
|
||||
create_secret("s3_secret_access_key", secrets.token_hex(32))
|
||||
database = quote(values["POSTGRES_DB"], safe="")
|
||||
for role in ("api", "worker", "dispatcher", "cleanup"):
|
||||
password = create_secret(f"database_{role}_password", secrets.token_hex(24))
|
||||
create_secret(f"database_{role}_url",
|
||||
f"postgres://messenger_{role}:{password}@db:5432/{database}"
|
||||
"?sslmode=verify-full&sslrootcert=/run/tls/ca.crt")
|
||||
if role != "dispatcher":
|
||||
create_secret(f"s3_{role}_access_key_id", secrets.token_hex(12))
|
||||
create_secret(f"s3_{role}_secret_access_key", secrets.token_hex(32))
|
||||
print("Local connection files ready; existing values and files preserved.")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Executable
+13
@@ -0,0 +1,13 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
docker compose up -d --wait db redis broker s3
|
||||
docker compose run --rm migrate
|
||||
docker compose exec -T db sh -ec '
|
||||
for role in api worker dispatcher cleanup; do
|
||||
export "${role}_password=$(cat "/run/secrets/database_${role}_password")"
|
||||
done
|
||||
psql -X -v ON_ERROR_STOP=1 -U "$POSTGRES_USER" "$POSTGRES_DB" -f -
|
||||
' < infra/postgres-roles.sql
|
||||
docker compose exec -T broker sh /bootstrap.sh
|
||||
docker compose run --rm s3-bootstrap
|
||||
Executable
+15
@@ -0,0 +1,15 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
if [ -n "${RESTART_EVIDENCE_FILE:-}" ]; then
|
||||
docker compose ps -q db redis s3 broker api | xargs docker inspect \
|
||||
-f '{{.Name}} {{.State.StartedAt}}' > "$RESTART_EVIDENCE_FILE"
|
||||
fi
|
||||
docker compose --project-name highload-lab7 --env-file .env -f compose.yml restart db redis s3 broker >/dev/null
|
||||
docker compose start --wait --wait-timeout 45 db redis s3 broker >/dev/null
|
||||
docker compose --project-name highload-lab7 --env-file .env -f compose.yml restart api upload-cleanup worker dispatcher >/dev/null
|
||||
./scripts/wait_ingress.sh
|
||||
if [ -n "${RESTART_EVIDENCE_FILE:-}" ]; then
|
||||
docker compose ps -q db redis s3 broker api | xargs docker inspect \
|
||||
-f '{{.Name}} {{.State.StartedAt}}' >> "$RESTART_EVIDENCE_FILE"
|
||||
fi
|
||||
Executable
+26
@@ -0,0 +1,26 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
cache=highload-lab7-trivy-cache
|
||||
trivy=aquasec/trivy:0.69.3
|
||||
mkdir -p .memory/scans
|
||||
docker run --rm -v "$cache:/root/.cache/trivy" -v "$PWD/src:/src:ro" "$trivy" \
|
||||
fs --timeout 20m --no-progress --quiet --scanners vuln --format json /src > .memory/scans/dependencies.raw
|
||||
mv .memory/scans/dependencies.raw .memory/scans/dependencies.json
|
||||
docker compose ps -aq | xargs docker inspect --format '{{.Image}} {{.Config.Image}}' |
|
||||
sort -u | while read -r image configured; do
|
||||
archive=$(mktemp)
|
||||
trap 'rm -f "$archive"' EXIT HUP INT TERM
|
||||
if ! docker image inspect "$image" >/dev/null 2>&1; then
|
||||
echo "Container image $image is untagged/unavailable; scanning configured image $configured"
|
||||
image=$(docker image inspect --format '{{.Id}}' "$configured")
|
||||
fi
|
||||
printf 'Scanning %s (%s)\n' "$configured" "$image"
|
||||
docker image save "$image" -o "$archive"
|
||||
docker run --rm -v "$cache:/root/.cache/trivy" -v "$archive:/scan.tar:ro" "$trivy" \
|
||||
image --timeout 15m --no-progress --quiet --input /scan.tar --scanners vuln --format json > ".memory/scans/image-$image.raw"
|
||||
mv ".memory/scans/image-$image.raw" ".memory/scans/image-$image.json"
|
||||
rm -f "$archive"
|
||||
trap - EXIT HUP INT TERM
|
||||
done
|
||||
python3 scripts/summarize_grade4_scan.py .memory/scans
|
||||
Executable
+19
@@ -0,0 +1,19 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
|
||||
: "${TARGET_INSTANCE_ID:?Set TARGET_INSTANCE_ID to the X-Instance-Id of an API replica}"
|
||||
for container in $(docker compose --project-name highload-lab7 --env-file .env -f compose.yml ps -q api); do
|
||||
hostname=$(docker inspect --format '{{.Config.Hostname}}' "$container")
|
||||
if [ "$hostname" = "$TARGET_INSTANCE_ID" ]; then
|
||||
# Only containers from this Compose project's api service are eligible.
|
||||
docker stop --timeout 8 "$container" >/dev/null
|
||||
if [ -n "${FAILOVER_EVIDENCE_FILE:-}" ]; then
|
||||
docker inspect -f '{{.Name}} {{.State.Status}}' "$container" >> "$FAILOVER_EVIDENCE_FILE"
|
||||
fi
|
||||
printf 'Stopped API instance %s\n' "$hostname"
|
||||
exit 0
|
||||
fi
|
||||
done
|
||||
printf 'No running API replica matches TARGET_INSTANCE_ID\n' >&2
|
||||
exit 1
|
||||
@@ -0,0 +1,15 @@
|
||||
"""Print a compact inventory from raw Trivy JSON."""
|
||||
import collections
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
for path in sorted(Path(sys.argv[1]).glob("*.json")):
|
||||
report = json.loads(path.read_text())
|
||||
matches = {}
|
||||
for target in report.get("Results", []):
|
||||
for entry in target.get("Vulnerabilities") or []:
|
||||
key = entry["VulnerabilityID"], entry["PkgName"], entry["InstalledVersion"]
|
||||
matches[key] = entry
|
||||
counts = collections.Counter(entry["Severity"] for entry in matches.values())
|
||||
print(path.name, dict(sorted(counts.items())))
|
||||
Executable
+13
@@ -0,0 +1,13 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
port=$(docker compose port haproxy 8443)
|
||||
for attempt in $(seq 1 30); do
|
||||
if curl --silent --fail --max-time 2 --cacert secrets/tls/lab7-ingress-ca.crt \
|
||||
"https://localhost:${port##*:}/health/ready" >/dev/null; then
|
||||
exit 0
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
printf 'HTTPS ingress did not become ready\n' >&2
|
||||
exit 1
|
||||
@@ -1,19 +1,48 @@
|
||||
# Модель угроз — заполнить
|
||||
# Модель угроз мессенджера
|
||||
|
||||
## Активы и акторы
|
||||
|
||||
TODO: пользовательские credentials/сессии, сообщения/вложения, service identities, CA/secrets, telemetry. Акторы: участник чата, посторонний пользователь, внешний клиент, скомпрометированный сервис, оператор. Опишите доступ атакующего и границы учебного стенда.
|
||||
- Пароли и их Argon2id-хеши, Bearer-токены и записи сессий. Владение действующим токеном даёт права соответствующего пользователя.
|
||||
- Сообщения, состав чатов, метаданные и байты вложений, включая staging и производные изображения. PostgreSQL хранит бизнес-состояние, S3 — файлы, RabbitMQ — доставку фоновых заданий, Redis — сессии и счётчики входа.
|
||||
- Ключи CA и сервисов, учётные данные хранилищ и токены OpenBao. Отдельно защищаются телеметрия и её доступность, поскольку поддельные сигналы могут скрыть отказ, а утечка логов — раскрыть секреты.
|
||||
|
||||
Внешний клиент может формировать произвольные HTTP-запросы к ingress. Аутентифицированный посторонний знает или угадывает идентификаторы ресурсов в чужом чате. Участник чата может загружать специально подготовленные файлы. При компрометации контейнера атакующий получает доступ к его файлам, учётным данным и доступным ему сетям. Оператор управляет хостом, Docker, сертификатами и хранилищами. Компрометация хоста, Docker daemon или ключа CA выходит за границы защиты приложения. Стенд развёрнут на одном хосте и проверяет отказ отдельной API-реплики.
|
||||
|
||||
## Границы доверия
|
||||
|
||||
TODO: схема и ссылка на заполненную contracts/trust-matrix.md; публичные/внутренние интерфейсы, процессы, секреты и их выдача.
|
||||
Клиент обращается к HAProxy по HTTPS. HAProxy подключается к API по mTLS, после чего приложение независимо проверяет пользовательскую сессию и членство в чате. API, worker, dispatcher и служебные задачи пересекают отдельную границу при доступе к данным. Collector принимает сигналы от приложений, а Grafana читает их из хранилищ. Административный доступ и выдача credentials образуют отдельную границу. Текущие и целевые настройки каждой связи перечислены в [матрице доверия](../contracts/trust-matrix.md).
|
||||
|
||||
| Угроза и конкретный путь | Нарушаемый инвариант | Контроль | Отрицательный тест | Остаточное ограничение |
|
||||
| --- | --- | --- | --- | --- |
|
||||
| TODO | TODO | TODO | TODO | TODO |
|
||||
|
||||
На 4 минимум 6 конкретных угроз. Не заменяйте сценарии словами «SQL injection», «безопасный Docker», «mTLS включён» без пути и проверки.
|
||||
| Угроза и путь | Защищаемое свойство | Контроль и проверка | Остаточное ограничение |
|
||||
| --- | --- | --- | --- |
|
||||
| Подмена HTTPS-узла и перехват Basic или Bearer | Клиент передаёт credentials нужному серверу | HTTPS с проверкой CA и SAN. `make test-security CA_FILE=secrets/tls/lab7-ingress-ca.crt` подтвердил отказ без доверия к CA. Неверное имя проверяется в `scripts/check_grade3_http.py`. | Токен, украденный на клиенте, пригоден до logout или expiry. |
|
||||
| Прямой запрос к API из соседнего контейнера или предъявление сертификата другого сервиса | API принимает только HAProxy | API не публикует порт. `python3 scripts/check_ingress_mtls.py` проверил обе реплики с сертификатом LB, без сертификата, с чужим CA и с чужим URI SAN от своего CA. | Кража ключа LB позволяет имитировать его до смены доверия. Пользовательские ACL продолжают действовать. |
|
||||
| Чтение чужих сообщений или файлов, подмена `X-User-Id`, добавление чужого attachment ID | Ресурсы доступны участникам соответствующего чата | Bearer определяет пользователя, членство проверяется API. Публичный smoke прошёл по HTTPS. `scripts/check_grade3_http.py` проверяет metadata/content и сообщение с чужим вложением и поддельным заголовком. | Участник может скопировать легально полученные данные. |
|
||||
| Повтор использования токена после logout или истечения срока | Отозванная сессия отклоняется всеми API | В Redis хранится ключ на основе SHA-256 токена с TTL, logout удаляет запись. Smoke проверил logout. Для expiry создать сессию на отдельном стенде с коротким TTL и ожидать 401 с тем же токеном после истечения. | Захваченная активная сессия имеет права пользователя. |
|
||||
| Перебор пароля и нагрузка на Argon2id через чередование реплик и поддельный `X-Forwarded-For` | Ограничение входа действует независимо от реплики | HAProxy заменяет заголовок фактическим адресом, Redis атомарно ведёт общий счётчик. `python3 scripts/check_login_rate.py` подтвердил 429 и `Retry-After` на 13-й попытке. Пароли хешируются Argon2id с индивидуальной солью. | Распределённые источники имеют отдельные окна, пользователи за одним NAT делят лимит. |
|
||||
| Загрузка слишком большого файла, изображения с чрезмерным числом пикселей или подменённым MIME | Ограничены память, CPU и объём входных данных | API ограничивает upload 10 МиБ и проверяет формат и размеры, обработка выполняется отдельно в worker. Проверить 413 для превышения размера/пикселей и 415 для ложного MIME, затем загрузить штатный файл и дождаться ready. | Параллельные загрузки конкурируют за общие ресурсы. |
|
||||
| Компрометация worker и выход за пределы его роли | Worker имеет доступ только к ресурсам обработки | Действуют отдельные DB/S3/AMQP identities и scoped policy OpenBao. TLS, обработка в своей роли и запрет чтения чужого пути проверяются отдельно. | Компрометация затронет разрешённую область worker. |
|
||||
| Поддельный OTLP ingest или чтение телеметрии соседом | Источник сигналов авторизован, хранилища доступны по ролям | Действует сетевая сегментация и mTLS с отдельными CA для ingest, export и read. Проверяется доставка в три хранилища и отказ чужого CA. | Скомпрометированное приложение может посылать ложные сигналы в пределах своей роли. |
|
||||
| Утечка bootstrap credentials или остановка OpenBao | Сервис получает только свои секреты, отказ закрывает новый доступ | Cert auth выдаёт отдельный токен на 15 минут для чтения своего KV-пути. Проверяются запрет чтения соседнего пути, записи и отсутствие административных секретов в контейнерах. | Действующие подключения и уже загруженные credentials могут жить до закрытия соединения или отзыва в целевой системе. |
|
||||
|
||||
## Lifecycle и инциденты
|
||||
|
||||
TODO: bootstrap, expiry/renewal, revoke/rotation, компрометация identity, sealed OpenBao и восстановление. Указать владельца CA/unseal и способ защиты bootstrap credentials. Сопоставить scan findings с реальной применимостью и исправлением/остаточным риском.
|
||||
Сейчас оператор выпускает локальный CA и leaf-сертификаты через `scripts/bootstrap_tls.sh`. CA действует 365 дней, leaf — 30 дней. Закрытый ключ CA остаётся у оператора, контейнеры получают отдельные файлы. Повторный запуск bootstrap сохраняет имеющиеся сертификаты.
|
||||
|
||||
OpenBao использует файловый backend на томе. Bootstrap сохраняет операторский материал на хосте с правами 0600. При эксплуатации оператор должен хранить доли ключа и административные credentials раздельно. После рестарта оператор выполняет unseal. API и worker заново получают scoped токены по своим клиентским сертификатам. При компрометации отзыв identity блокирует новую аутентификацию, отзыв токенов закрывает доступ к OpenBao, а смена выданных паролей и закрытие активных подключений ограничивают доступ к зависимостям. Проверки expiry, renewal и ротации выполняются на новых соединениях. результаты сканирования и применимость уязвимостей фиксируются в доказательствах.
|
||||
|
||||
## Проверки уровня «4»
|
||||
|
||||
- Подмена identity на брокере и Collector. Сертификат незарегистрированной роли
|
||||
от доверенного CA отклоняется в новом сеансе. Негативные сценарии
|
||||
воспроизводятся через `scripts/check_grade4_broker.py` и
|
||||
`scripts/check_grade3_tls.py`. AMQP ACL дополнительно запрещают публикацию
|
||||
воркером и чтение диспетчером.
|
||||
- Отзыв OpenBao и компрометация пароля PostgreSQL. `scripts/check_grade4_lifecycle.py`
|
||||
меняет leaf и пароль, проверяет отказ старым credentials на новом соединении,
|
||||
истечение короткого токена и вход при sealed backend.
|
||||
- Преднамеренная нагрузка на Argon2id. `scripts/check_grade4_login.py` подаёт
|
||||
параллельные запросы с корректной парой имени и пароля и проверяет ограничение
|
||||
до запуска лишней работы.
|
||||
- Утечка credentials в наблюдаемость. `scripts/check_grade4_telemetry.py`
|
||||
посылает синтетический секрет и проверяет соответствующую журнальную запись,
|
||||
трейсы и метрики за выбранный интервал.
|
||||
|
||||
@@ -0,0 +1,26 @@
|
||||
.git
|
||||
.gitignore
|
||||
.github
|
||||
.env
|
||||
.env.*
|
||||
!.env.example
|
||||
secrets
|
||||
certs
|
||||
volumes
|
||||
data
|
||||
backups
|
||||
*.key
|
||||
*.pem
|
||||
*.p12
|
||||
*.pfx
|
||||
*.jks
|
||||
.venv
|
||||
__pycache__
|
||||
node_modules
|
||||
evidence
|
||||
.DS_Store
|
||||
README.md
|
||||
REPORT.md
|
||||
COURSE.md
|
||||
CONTRIBUTING.md
|
||||
MAINTAINERS.md
|
||||
@@ -0,0 +1,39 @@
|
||||
FROM golang:1.22.2-bookworm AS builder
|
||||
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends webp && rm -rf /var/lib/apt/lists/*
|
||||
|
||||
WORKDIR /src
|
||||
|
||||
COPY go.mod go.sum ./
|
||||
|
||||
RUN go mod download
|
||||
|
||||
COPY backend ./backend
|
||||
|
||||
RUN CGO_ENABLED=0 \
|
||||
GOOS=linux \
|
||||
go build \
|
||||
-trimpath \
|
||||
-ldflags="-s -w" \
|
||||
-o /out/messenger \
|
||||
./backend
|
||||
|
||||
|
||||
FROM debian:12.15-slim
|
||||
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends webp && rm -rf /var/lib/apt/lists/* \
|
||||
&& groupadd --system app \
|
||||
&& useradd --system --gid app --no-create-home app
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
COPY --from=builder /out/messenger ./messenger
|
||||
|
||||
ENV APP_HOST=0.0.0.0
|
||||
ENV APP_PORT=8080
|
||||
|
||||
EXPOSE 8080
|
||||
|
||||
USER app:app
|
||||
|
||||
CMD ["./messenger"]
|
||||
@@ -0,0 +1,155 @@
|
||||
package app
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log"
|
||||
"net"
|
||||
"net/http"
|
||||
"os"
|
||||
"os/signal"
|
||||
"runtime"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"messenger/src/backend/handlers"
|
||||
"messenger/src/backend/repositories"
|
||||
"messenger/src/backend/services"
|
||||
"messenger/src/backend/telemetry"
|
||||
)
|
||||
|
||||
func serverAddress() string {
|
||||
host := os.Getenv("APP_HOST")
|
||||
if host == "" {
|
||||
host = "0.0.0.0"
|
||||
}
|
||||
|
||||
port := os.Getenv("APP_PORT")
|
||||
if port == "" {
|
||||
port = "8080"
|
||||
}
|
||||
|
||||
return net.JoinHostPort(host, port)
|
||||
}
|
||||
|
||||
func RunHTTP() error {
|
||||
tlsConfig, err := ingressTLS()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := repositories.UploadCleanupGrace(); err != nil {
|
||||
return err
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
|
||||
db, err := repositories.OpenDatabase(ctx)
|
||||
cancel()
|
||||
if err != nil {
|
||||
return fmt.Errorf("database initialization failed: %w", err)
|
||||
}
|
||||
defer db.Close()
|
||||
// Opt-in experiment telemetry goes to container logs, never a public port.
|
||||
if os.Getenv("BENCH_POOL_STATS") == "1" {
|
||||
go func() {
|
||||
for range time.Tick(time.Second) {
|
||||
s := db.Stat()
|
||||
data, _ := json.Marshal(map[string]any{
|
||||
"time": time.Now().UTC(), "acquired": s.AcquiredConns(),
|
||||
"total": s.TotalConns(), "max": s.MaxConns(),
|
||||
"gomaxprocs": runtime.GOMAXPROCS(0), "goroutines": runtime.NumGoroutine(),
|
||||
"acquire_count": s.AcquireCount(), "acquire_duration_ns": s.AcquireDuration().Nanoseconds(),
|
||||
"empty_acquire_count": s.EmptyAcquireCount(), "canceled_acquire_count": s.CanceledAcquireCount(),
|
||||
})
|
||||
log.Printf("benchmark_pool %s", data)
|
||||
}
|
||||
}()
|
||||
}
|
||||
redisClient, err := repositories.RedisClientFromEnvironment()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer redisClient.Close()
|
||||
sessionStore := repositories.NewSessionStore(redisClient, "messenger:session:")
|
||||
userStore := repositories.NewUserStore(db)
|
||||
chatStore := repositories.NewChatStore(db)
|
||||
idempotencyTTL, err := repositories.IdempotencyTTL()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
messageStore := repositories.NewMessageStoreWithTTL(db, idempotencyTTL)
|
||||
ttl, err := services.SessionTTL()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
userService := services.NewUserService(userStore)
|
||||
sessionService := services.NewSessionService(userService, sessionStore, ttl)
|
||||
chatService := services.NewChatService(chatStore)
|
||||
messageService := services.NewMessageService(messageStore)
|
||||
objects, err := repositories.NewObjectStoreFromEnvironment()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
attachmentService := services.NewAttachmentService(repositories.NewAttachmentStore(db), objects)
|
||||
mux := handlers.NewRouter(services.NewHealthService(repositories.NewHealthStore(db), sessionStore, objects), userService, sessionService, chatService, messageService, attachmentService)
|
||||
|
||||
signals, stopTelemetry, err := telemetry.Start("messenger-api")
|
||||
if err != nil {
|
||||
return fmt.Errorf("telemetry initialization failed: %w", err)
|
||||
}
|
||||
defer stopTelemetry()
|
||||
if err := signals.ObservePool(db); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
server := &http.Server{
|
||||
Addr: serverAddress(),
|
||||
TLSConfig: tlsConfig,
|
||||
Handler: signals.HTTP(mux),
|
||||
ReadHeaderTimeout: 5 * time.Second,
|
||||
ReadTimeout: 30 * time.Second,
|
||||
WriteTimeout: 30 * time.Second,
|
||||
IdleTimeout: 60 * time.Second,
|
||||
}
|
||||
|
||||
shutdownContext, stop := signal.NotifyContext(
|
||||
context.Background(),
|
||||
os.Interrupt,
|
||||
syscall.SIGTERM,
|
||||
)
|
||||
defer stop()
|
||||
|
||||
serverErrors := make(chan error, 1)
|
||||
|
||||
go func() {
|
||||
log.Printf("server is listening on %s", server.Addr)
|
||||
|
||||
if err := server.ListenAndServeTLS(os.Getenv("TLS_CERT_FILE"), os.Getenv("TLS_KEY_FILE")); err != nil &&
|
||||
!errors.Is(err, http.ErrServerClosed) {
|
||||
serverErrors <- err
|
||||
}
|
||||
}()
|
||||
|
||||
select {
|
||||
case err := <-serverErrors:
|
||||
return err
|
||||
|
||||
case <-shutdownContext.Done():
|
||||
log.Printf("shutdown signal received")
|
||||
handlers.BeginShutdown()
|
||||
|
||||
shutdownDeadline, cancel := context.WithTimeout(
|
||||
context.Background(),
|
||||
10*time.Second,
|
||||
)
|
||||
defer cancel()
|
||||
|
||||
if err := server.Shutdown(shutdownDeadline); err != nil {
|
||||
log.Printf("graceful shutdown failed: %v", err)
|
||||
return nil
|
||||
}
|
||||
|
||||
log.Printf("server stopped gracefully")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,32 @@
|
||||
package app
|
||||
|
||||
import (
|
||||
"crypto/tls"
|
||||
"crypto/x509"
|
||||
"errors"
|
||||
"os"
|
||||
)
|
||||
|
||||
func ingressTLS() (*tls.Config, error) {
|
||||
ca, err := os.ReadFile(os.Getenv("TLS_CA_FILE"))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
roots := x509.NewCertPool()
|
||||
if !roots.AppendCertsFromPEM(ca) {
|
||||
return nil, errors.New("invalid ingress client CA")
|
||||
}
|
||||
return &tls.Config{
|
||||
MinVersion: tls.VersionTLS12,
|
||||
ClientAuth: tls.RequireAndVerifyClientCert,
|
||||
ClientCAs: roots,
|
||||
VerifyConnection: func(state tls.ConnectionState) error {
|
||||
for _, identity := range state.PeerCertificates[0].URIs {
|
||||
if identity.String() == "urn:lab7:load-balancer" {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
return errors.New("client certificate does not identify the load balancer")
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
@@ -0,0 +1,111 @@
|
||||
package commands
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/tls"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"time"
|
||||
)
|
||||
|
||||
var secretFiles = map[string]string{
|
||||
"database_url": "DATABASE_URL_FILE",
|
||||
"cache_url": "CACHE_URL_FILE",
|
||||
"broker_url": "BROKER_URL_FILE",
|
||||
"s3_access_key_id": "S3_ACCESS_KEY_ID_FILE",
|
||||
"s3_secret_access_key": "S3_SECRET_ACCESS_KEY_FILE",
|
||||
}
|
||||
|
||||
func loadBaoSecrets() error {
|
||||
role := os.Getenv("BAO_ROLE")
|
||||
address, err := url.Parse(os.Getenv("BAO_ADDR"))
|
||||
if err != nil || address.Scheme != "https" || address.Host == "" {
|
||||
return errors.New("invalid OpenBao HTTPS address")
|
||||
}
|
||||
cert, err := tls.LoadX509KeyPair("/run/bao/client.crt", "/run/bao/client.key")
|
||||
if err != nil {
|
||||
return fmt.Errorf("load OpenBao service identity: %w", err)
|
||||
}
|
||||
transport := http.DefaultTransport.(*http.Transport).Clone()
|
||||
transport.TLSClientConfig = &tls.Config{MinVersion: tls.VersionTLS12, Certificates: []tls.Certificate{cert}}
|
||||
defer transport.CloseIdleConnections()
|
||||
client := &http.Client{
|
||||
Transport: transport,
|
||||
Timeout: 5 * time.Second,
|
||||
CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse },
|
||||
}
|
||||
request := func(method, path string, body []byte, token string, result any) error {
|
||||
req, err := http.NewRequest(method, address.String()+path, bytes.NewReader(body))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
if token != "" {
|
||||
req.Header.Set("X-Vault-Token", token)
|
||||
}
|
||||
response, err := client.Do(req)
|
||||
if err != nil {
|
||||
return errors.New("OpenBao connection failed")
|
||||
}
|
||||
defer response.Body.Close()
|
||||
if response.StatusCode != http.StatusOK {
|
||||
return fmt.Errorf("OpenBao returned HTTP %d", response.StatusCode)
|
||||
}
|
||||
return json.NewDecoder(response.Body).Decode(result)
|
||||
}
|
||||
var login struct {
|
||||
Auth struct {
|
||||
Token string `json:"client_token"`
|
||||
}
|
||||
}
|
||||
body, err := json.Marshal(map[string]string{"name": role})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err = request("POST", "/v1/auth/cert/login", body, "", &login); err != nil {
|
||||
return err
|
||||
}
|
||||
if login.Auth.Token == "" {
|
||||
return errors.New("OpenBao returned an empty token")
|
||||
}
|
||||
var secret struct {
|
||||
Data struct{ Data map[string]string }
|
||||
}
|
||||
if err = request("GET", "/v1/messenger/data/"+url.PathEscape(role), nil, login.Auth.Token, &secret); err != nil {
|
||||
return err
|
||||
}
|
||||
for key, env := range secretFiles {
|
||||
if os.Getenv(env) == "" {
|
||||
continue
|
||||
}
|
||||
value := secret.Data.Data[key]
|
||||
if value == "" {
|
||||
return fmt.Errorf("missing %s in OpenBao", key)
|
||||
}
|
||||
dir := "/run/runtime-secrets"
|
||||
path := filepath.Join(dir, key)
|
||||
tmp, err := os.CreateTemp(dir, ".secret-")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, writeErr := tmp.WriteString(value)
|
||||
closeErr := tmp.Close()
|
||||
if err := errors.Join(writeErr, closeErr); err != nil {
|
||||
os.Remove(tmp.Name())
|
||||
return err
|
||||
}
|
||||
if err := os.Rename(tmp.Name(), path); err != nil {
|
||||
os.Remove(tmp.Name())
|
||||
return err
|
||||
}
|
||||
if err := os.Setenv(env, path); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,64 @@
|
||||
package commands
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"flag"
|
||||
"log"
|
||||
"os"
|
||||
"os/signal"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"messenger/src/backend/repositories"
|
||||
)
|
||||
|
||||
func runUploadCleanup(args []string) error {
|
||||
flags := flag.NewFlagSet("attachments-cleanup", flag.ContinueOnError)
|
||||
loop := flags.Bool("loop", false, "run every 60 seconds")
|
||||
dryRun := flags.Bool("dry-run", false, "report without deleting anything")
|
||||
if err := flags.Parse(args); err != nil {
|
||||
return err
|
||||
}
|
||||
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
|
||||
defer stop()
|
||||
init, cancel := context.WithTimeout(ctx, 20*time.Second)
|
||||
db, err := repositories.OpenDatabase(init)
|
||||
cancel()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer db.Close()
|
||||
objects, err := repositories.NewObjectStoreFromEnvironment()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
store := repositories.NewAttachmentStore(db)
|
||||
for {
|
||||
pass, cancel := context.WithTimeout(ctx, 30*time.Second)
|
||||
result, err := store.CleanupUploads(pass, *dryRun, objects.Delete)
|
||||
direct, directErr := store.CleanupDirectUploads(pass, *dryRun, objects.Delete)
|
||||
jobs, jobsErr := store.CleanupJobs(pass, 24*time.Hour, *dryRun, objects.Delete)
|
||||
cancel()
|
||||
if encodeErr := json.NewEncoder(os.Stdout).Encode(map[string]any{"originals": result, "direct": direct, "job_staging": jobs}); encodeErr != nil {
|
||||
return encodeErr
|
||||
}
|
||||
if !*loop {
|
||||
if err == nil {
|
||||
err = directErr
|
||||
if err == nil {
|
||||
err = jobsErr
|
||||
}
|
||||
}
|
||||
return err
|
||||
}
|
||||
if err != nil || directErr != nil || jobsErr != nil {
|
||||
log.Print("upload cleanup pass failed; journal retained for retry")
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return nil
|
||||
case <-time.After(60 * time.Second):
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,76 @@
|
||||
package commands
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"os"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
|
||||
"messenger/src/backend/repositories"
|
||||
)
|
||||
|
||||
func runJobAdmin(args []string) error {
|
||||
if len(args) < 1 {
|
||||
return errors.New("usage: jobs list-failed | replay JOB_ID REASON | cleanup-staging JOB_ID | upgrade ATTACHMENT_ID 2 REASON")
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
|
||||
defer cancel()
|
||||
db, err := repositories.OpenDatabase(ctx)
|
||||
if err != nil {
|
||||
return errors.New("database unavailable")
|
||||
}
|
||||
defer db.Close()
|
||||
store := repositories.NewAttachmentStore(db)
|
||||
if args[0] == "list-failed" && len(args) == 1 {
|
||||
jobs, e := store.FailedJobs(ctx)
|
||||
if e != nil {
|
||||
return errors.New("cannot read failed registry")
|
||||
}
|
||||
return json.NewEncoder(os.Stdout).Encode(jobs)
|
||||
}
|
||||
if len(args) < 2 {
|
||||
return errors.New("job id required")
|
||||
}
|
||||
if _, err = uuid.Parse(args[1]); err != nil {
|
||||
return errors.New("invalid job id")
|
||||
}
|
||||
objects, err := repositories.NewObjectStoreFromEnvironment()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var changed bool
|
||||
switch {
|
||||
case args[0] == "upgrade" && len(args) == 4:
|
||||
version, e := strconv.Atoi(args[2])
|
||||
if e != nil {
|
||||
return errors.New("invalid pipeline version")
|
||||
}
|
||||
changed, err = store.UpgradePipeline(ctx, args[1], version, args[3], func(ctx context.Context, key string) error {
|
||||
r, e := objects.Get(ctx, key)
|
||||
if e == nil {
|
||||
r.Close()
|
||||
}
|
||||
return e
|
||||
})
|
||||
case args[0] == "replay" && len(args) == 3:
|
||||
changed, err = store.ReplayJob(ctx, args[1], args[2], func(ctx context.Context, key string) error {
|
||||
r, e := objects.Get(ctx, key)
|
||||
if e == nil {
|
||||
r.Close()
|
||||
}
|
||||
return e
|
||||
})
|
||||
case args[0] == "cleanup-staging" && len(args) == 2:
|
||||
changed, err = store.CleanupJobStaging(ctx, args[1], 24*time.Hour, false, objects.Delete)
|
||||
default:
|
||||
return errors.New("usage: jobs list-failed | replay JOB_ID REASON | cleanup-staging JOB_ID | upgrade ATTACHMENT_ID 2 REASON")
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return json.NewEncoder(os.Stdout).Encode(map[string]bool{"changed": changed})
|
||||
}
|
||||
@@ -0,0 +1,84 @@
|
||||
package commands
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"log"
|
||||
"net"
|
||||
"net/http"
|
||||
"os"
|
||||
"time"
|
||||
|
||||
"messenger/src/backend/app"
|
||||
"messenger/src/backend/repositories"
|
||||
"messenger/src/backend/workers"
|
||||
)
|
||||
|
||||
// Run dispatches the existing executable modes; the binary and CLI stay the same.
|
||||
func Run(args []string) error {
|
||||
if os.Getenv("BAO_ROLE") != "" && (len(args) == 0 || args[0] != "image-process") {
|
||||
if err := loadBaoSecrets(); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if len(args) == 0 {
|
||||
if environment := os.Getenv("APP_ENV"); environment != "" {
|
||||
log.SetPrefix("[" + environment + "] ")
|
||||
}
|
||||
return app.RunHTTP()
|
||||
}
|
||||
switch args[0] {
|
||||
case "image-process":
|
||||
return workers.RunImageChild(args[1:])
|
||||
case "jobs":
|
||||
return runJobAdmin(args[1:])
|
||||
case "worker", "dispatcher", "broker-publish":
|
||||
return workers.RunPipeline(args[0], args[1:])
|
||||
}
|
||||
if environment := os.Getenv("APP_ENV"); environment != "" {
|
||||
log.SetPrefix("[" + environment + "] ")
|
||||
}
|
||||
switch {
|
||||
case args[0] == "s3-bootstrap" && len(args) == 1:
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
if err := repositories.BootstrapS3(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
log.Print("private S3 bucket ready")
|
||||
return nil
|
||||
case args[0] == "attachments-cleanup":
|
||||
return runUploadCleanup(args[1:])
|
||||
case args[0] == "healthcheck" && len(args) == 1:
|
||||
return healthcheck()
|
||||
case args[0] == "migrate" && len(args) == 2 && (args[1] == "up" || args[1] == "down"):
|
||||
url, err := repositories.DatabaseURL()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err = repositories.RunMigrations(url, args[1]); err != nil {
|
||||
return err
|
||||
}
|
||||
log.Printf("migration %s complete", args[1])
|
||||
return nil
|
||||
default:
|
||||
return errors.New("usage: messenger migrate up|down")
|
||||
}
|
||||
}
|
||||
|
||||
func healthcheck() error {
|
||||
client := http.Client{Timeout: 2 * time.Second}
|
||||
port := os.Getenv("APP_PORT")
|
||||
if port == "" {
|
||||
port = "8080"
|
||||
}
|
||||
response, err := client.Get("http://" + net.JoinHostPort("127.0.0.1", port) + "/health/ready")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
response.Body.Close()
|
||||
if response.StatusCode != http.StatusOK {
|
||||
return errors.New("API is not ready")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,165 @@
|
||||
package handlers
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"github.com/google/uuid"
|
||||
"io"
|
||||
"log"
|
||||
"messenger/src/backend/repositories"
|
||||
"messenger/src/backend/services"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
func uploadAttachmentHandler(chats *services.ChatService, attachments *services.AttachmentService) http.HandlerFunc {
|
||||
return func(w http.ResponseWriter, r *http.Request) {
|
||||
requestID := newRequestID()
|
||||
w.Header().Set("X-Request-Id", requestID)
|
||||
if r.Method != http.MethodPost {
|
||||
writeError(w, 405, "method_not_allowed", "method not allowed", requestID)
|
||||
return
|
||||
}
|
||||
userID, ok := requestUserID(w, r, requestID)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
raw := strings.TrimSuffix(strings.TrimPrefix(r.URL.Path, "/api/v1/chats/"), "/attachments")
|
||||
chatID, err := uuid.Parse(raw)
|
||||
if err != nil {
|
||||
writeError(w, 400, "invalid_request", "invalid chat id", requestID)
|
||||
return
|
||||
}
|
||||
if _, err = chats.Get(r.Context(), userID, chatID.String()); err != nil {
|
||||
writeServiceError(w, err, requestID)
|
||||
return
|
||||
}
|
||||
query, err := urlQuery(r)
|
||||
if err != nil {
|
||||
writeServiceError(w, err, requestID)
|
||||
return
|
||||
}
|
||||
filenames := query["filename"]
|
||||
if len(filenames) != 1 {
|
||||
writeError(w, 400, "invalid_request", "exactly one filename is required", requestID)
|
||||
return
|
||||
}
|
||||
if err = services.ValidateFilename(filenames[0]); err != nil {
|
||||
writeServiceError(w, err, requestID)
|
||||
return
|
||||
}
|
||||
if r.ContentLength > services.MaxUploadBytes {
|
||||
writeServiceError(w, services.ErrPayloadTooLarge, requestID)
|
||||
return
|
||||
}
|
||||
r.Body = http.MaxBytesReader(w, r.Body, services.MaxUploadBytes)
|
||||
ctx, cancel := context.WithTimeout(r.Context(), 25*time.Second)
|
||||
defer cancel()
|
||||
attachment, err := attachments.Upload(ctx, userID, chatID.String(), filenames[0], r.Header.Get("Content-Type"), r.Body)
|
||||
var tooLarge *http.MaxBytesError
|
||||
if errors.As(err, &tooLarge) {
|
||||
err = services.ErrPayloadTooLarge
|
||||
}
|
||||
if err != nil {
|
||||
writeServiceError(w, err, requestID)
|
||||
return
|
||||
}
|
||||
writeJSON(w, 202, attachment)
|
||||
}
|
||||
}
|
||||
|
||||
func attachmentHandler(attachments *services.AttachmentService) http.HandlerFunc {
|
||||
return func(w http.ResponseWriter, r *http.Request) {
|
||||
requestID := newRequestID()
|
||||
w.Header().Set("X-Request-Id", requestID)
|
||||
if r.Method != http.MethodGet {
|
||||
writeError(w, 405, "method_not_allowed", "method not allowed", requestID)
|
||||
return
|
||||
}
|
||||
userID, ok := requestUserID(w, r, requestID)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
path := strings.TrimPrefix(r.URL.Path, "/api/v1/attachments/")
|
||||
parts := strings.Split(path, "/")
|
||||
if len(parts) > 2 || (len(parts) == 2 && parts[1] != "content") {
|
||||
writeServiceError(w, services.ErrAttachmentNotFound, requestID)
|
||||
return
|
||||
}
|
||||
id, err := uuid.Parse(parts[0])
|
||||
if err != nil {
|
||||
writeError(w, 400, "invalid_request", "invalid attachment id", requestID)
|
||||
return
|
||||
}
|
||||
a, err := attachments.Get(r.Context(), id.String(), userID)
|
||||
if err != nil {
|
||||
writeServiceError(w, err, requestID)
|
||||
return
|
||||
}
|
||||
if len(parts) == 1 {
|
||||
writeJSON(w, 200, a)
|
||||
return
|
||||
}
|
||||
query, err := urlQuery(r)
|
||||
if err != nil {
|
||||
writeServiceError(w, err, requestID)
|
||||
return
|
||||
}
|
||||
if a.Status != "ready" {
|
||||
writeServiceError(w, services.ErrConflict, requestID)
|
||||
return
|
||||
}
|
||||
if variants, exists := query["variant"]; exists {
|
||||
if len(variants) != 1 {
|
||||
writeError(w, 400, "invalid_request", "invalid variant", requestID)
|
||||
return
|
||||
}
|
||||
if variants[0] == "thumbnail" || variants[0] == "crop" || variants[0] == "optimized" {
|
||||
found := false
|
||||
for _, raw := range a.Variants {
|
||||
if v, ok := raw.(map[string]any); ok && v["name"] == variants[0] {
|
||||
a.ObjectKey = repositories.VariantObjectKey(a.ID, a.PipelineVersion, variants[0])
|
||||
a.ContentType = v["content_type"].(string)
|
||||
a.SizeBytes = int64(v["size_bytes"].(float64))
|
||||
found = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
writeServiceError(w, services.ErrAttachmentNotFound, requestID)
|
||||
return
|
||||
}
|
||||
} else if variants[0] != "original" {
|
||||
writeServiceError(w, services.ErrAttachmentNotFound, requestID)
|
||||
return
|
||||
}
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(r.Context(), 25*time.Second)
|
||||
defer cancel()
|
||||
content, err := attachments.Content(ctx, a)
|
||||
if err != nil {
|
||||
writeServiceError(w, err, requestID)
|
||||
return
|
||||
}
|
||||
defer content.Close()
|
||||
w.Header().Set("Content-Type", a.ContentType)
|
||||
w.Header().Set("Content-Length", strconv.FormatInt(a.SizeBytes, 10))
|
||||
w.Header().Set("X-Content-Type-Options", "nosniff")
|
||||
w.Header().Set("Cache-Control", "private, no-store")
|
||||
w.WriteHeader(200)
|
||||
if _, err = io.Copy(w, content); err != nil {
|
||||
log.Printf("attachment download interrupted, request %s", requestID)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func urlQuery(r *http.Request) (url.Values, error) {
|
||||
q, err := url.ParseQuery(r.URL.RawQuery)
|
||||
if err != nil {
|
||||
return nil, services.InvalidInput{Message: "invalid query"}
|
||||
}
|
||||
return q, nil
|
||||
}
|
||||
@@ -0,0 +1,137 @@
|
||||
package handlers
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"strings"
|
||||
|
||||
"messenger/src/backend/services"
|
||||
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
|
||||
type createChatRequest struct {
|
||||
Title string `json:"title"`
|
||||
MemberIDs []string `json:"member_ids"`
|
||||
}
|
||||
|
||||
func createChatHandler(chats *services.ChatService) http.HandlerFunc {
|
||||
return func(w http.ResponseWriter, r *http.Request) {
|
||||
requestID := newRequestID()
|
||||
w.Header().Set("X-Request-Id", requestID)
|
||||
if r.Method != http.MethodPost {
|
||||
writeError(w, 405, "method_not_allowed", "method not allowed", requestID)
|
||||
return
|
||||
}
|
||||
userID, ok := requestUserID(w, r, requestID)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
if !strings.HasPrefix(r.Header.Get("Content-Type"), "application/json") {
|
||||
writeError(w, 415, "unsupported_media_type", "content type must be application/json", requestID)
|
||||
return
|
||||
}
|
||||
var request createChatRequest
|
||||
if err := decodeJSONBody(r, &request); err != nil {
|
||||
writeError(w, 400, "invalid_request", "invalid JSON body", requestID)
|
||||
return
|
||||
}
|
||||
chat, err := chats.Create(r.Context(), userID, request.Title, request.MemberIDs)
|
||||
if err != nil {
|
||||
writeServiceError(w, err, requestID)
|
||||
return
|
||||
}
|
||||
writeJSON(w, 201, chat)
|
||||
}
|
||||
}
|
||||
|
||||
func getChatHandler(chats *services.ChatService) http.HandlerFunc {
|
||||
return func(w http.ResponseWriter, r *http.Request) {
|
||||
requestID := newRequestID()
|
||||
w.Header().Set("X-Request-Id", requestID)
|
||||
if r.Method != http.MethodGet {
|
||||
writeError(w, 405, "method_not_allowed", "method not allowed", requestID)
|
||||
return
|
||||
}
|
||||
userID, ok := requestUserID(w, r, requestID)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
chatID := strings.TrimPrefix(r.URL.Path, "/api/v1/chats/")
|
||||
if chatID == "" || strings.Contains(chatID, "/") {
|
||||
writeError(w, 400, "invalid_request", "invalid chat id", requestID)
|
||||
return
|
||||
}
|
||||
if _, err := uuid.Parse(chatID); err != nil {
|
||||
writeError(w, 400, "invalid_request", "invalid chat id", requestID)
|
||||
return
|
||||
}
|
||||
chat, err := chats.Get(r.Context(), userID, chatID)
|
||||
if err != nil {
|
||||
writeServiceError(w, err, requestID)
|
||||
return
|
||||
}
|
||||
writeJSON(w, 200, chat)
|
||||
}
|
||||
}
|
||||
|
||||
func listChatsHandler(chats *services.ChatService) http.HandlerFunc {
|
||||
return func(w http.ResponseWriter, r *http.Request) {
|
||||
requestID := newRequestID()
|
||||
w.Header().Set("X-Request-Id", requestID)
|
||||
if r.Method != http.MethodGet {
|
||||
writeError(w, 405, "method_not_allowed", "method not allowed", requestID)
|
||||
return
|
||||
}
|
||||
userID, ok := requestUserID(w, r, requestID)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
limit, cursor, ok := pageParams(w, r, requestID)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
page, err := chats.List(r.Context(), userID, limit, cursor)
|
||||
if err != nil {
|
||||
writeServiceError(w, err, requestID)
|
||||
return
|
||||
}
|
||||
writeJSON(w, 200, page)
|
||||
}
|
||||
}
|
||||
|
||||
func chatCollectionHandler(chats *services.ChatService) http.HandlerFunc {
|
||||
create := createChatHandler(chats)
|
||||
list := listChatsHandler(chats)
|
||||
return func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Method == http.MethodGet {
|
||||
list(w, r)
|
||||
} else {
|
||||
create(w, r)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func chatSubresourceHandler(chats *services.ChatService, messages *services.MessageService, attachments *services.AttachmentService) http.HandlerFunc {
|
||||
get := getChatHandler(chats)
|
||||
createMessage := createMessageHandler(chats, messages)
|
||||
listMessages := listMessagesHandler(chats, messages)
|
||||
return func(w http.ResponseWriter, r *http.Request) {
|
||||
if attachments != nil && strings.Contains(r.URL.Path, "/attachment-uploads") {
|
||||
directUploadHandler(chats, attachments)(w, r)
|
||||
return
|
||||
}
|
||||
if attachments != nil && strings.HasSuffix(r.URL.Path, "/attachments") {
|
||||
uploadAttachmentHandler(chats, attachments)(w, r)
|
||||
return
|
||||
}
|
||||
if strings.HasSuffix(r.URL.Path, "/messages") {
|
||||
if r.Method == http.MethodGet {
|
||||
listMessages(w, r)
|
||||
} else {
|
||||
createMessage(w, r)
|
||||
}
|
||||
return
|
||||
}
|
||||
get(w, r)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,81 @@
|
||||
package handlers
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"io"
|
||||
"net/http"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"messenger/src/backend/services"
|
||||
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
|
||||
func directUploadHandler(chats *services.ChatService, attachments *services.AttachmentService) http.HandlerFunc {
|
||||
return func(w http.ResponseWriter, r *http.Request) {
|
||||
requestID := newRequestID()
|
||||
w.Header().Set("X-Request-Id", requestID)
|
||||
if r.Method != http.MethodPost {
|
||||
writeError(w, 405, "method_not_allowed", "method not allowed", requestID)
|
||||
return
|
||||
}
|
||||
user, ok := requestUserID(w, r, requestID)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
parts := strings.Split(strings.TrimPrefix(r.URL.Path, "/api/v1/chats/"), "/")
|
||||
if (len(parts) != 2 && (len(parts) != 4 || parts[3] != "complete")) || parts[1] != "attachment-uploads" {
|
||||
writeError(w, 404, "not_found", "upload endpoint not found", requestID)
|
||||
return
|
||||
}
|
||||
chat, err := uuid.Parse(parts[0])
|
||||
if err != nil {
|
||||
writeError(w, 400, "invalid_request", "invalid chat id", requestID)
|
||||
return
|
||||
}
|
||||
if _, err = chats.Get(r.Context(), user, chat.String()); err != nil {
|
||||
writeServiceError(w, err, requestID)
|
||||
return
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(r.Context(), 25*time.Second)
|
||||
defer cancel()
|
||||
if len(parts) == 4 {
|
||||
id, err := uuid.Parse(parts[2])
|
||||
if err != nil {
|
||||
writeError(w, 400, "invalid_request", "invalid upload id", requestID)
|
||||
return
|
||||
}
|
||||
a, err := attachments.CompleteDirect(ctx, user, chat.String(), id.String())
|
||||
if err != nil {
|
||||
writeServiceError(w, err, requestID)
|
||||
return
|
||||
}
|
||||
writeJSON(w, 202, a)
|
||||
return
|
||||
}
|
||||
r.Body = http.MaxBytesReader(w, r.Body, 4096)
|
||||
var input services.DirectUploadInput
|
||||
decoder := json.NewDecoder(r.Body)
|
||||
decoder.DisallowUnknownFields()
|
||||
err = decoder.Decode(&input)
|
||||
if err == nil {
|
||||
var extra any
|
||||
if next := decoder.Decode(&extra); !errors.Is(next, io.EOF) {
|
||||
err = errors.New("expected one JSON object")
|
||||
}
|
||||
}
|
||||
if err != nil {
|
||||
writeError(w, 400, "invalid_request", "invalid upload JSON (maximum 4 KiB)", requestID)
|
||||
return
|
||||
}
|
||||
result, err := attachments.InitDirect(ctx, user, chat.String(), input)
|
||||
if err != nil {
|
||||
writeServiceError(w, err, requestID)
|
||||
return
|
||||
}
|
||||
writeJSON(w, 201, result)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,163 @@
|
||||
package handlers
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"log"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"messenger/src/backend/repositories"
|
||||
"messenger/src/backend/services"
|
||||
)
|
||||
|
||||
type healthResponse struct {
|
||||
Status string `json:"status"`
|
||||
}
|
||||
type apiError struct {
|
||||
Code string `json:"code"`
|
||||
Message string `json:"message"`
|
||||
RequestID string `json:"request_id"`
|
||||
}
|
||||
type errorResponse struct {
|
||||
Error apiError `json:"error"`
|
||||
}
|
||||
|
||||
func newRequestID() string {
|
||||
bytes := make([]byte, 16)
|
||||
if _, err := rand.Read(bytes); err != nil {
|
||||
return "request-id-generation-failed"
|
||||
}
|
||||
return hex.EncodeToString(bytes)
|
||||
}
|
||||
|
||||
func writeError(w http.ResponseWriter, status int, code, message, requestID string) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.WriteHeader(status)
|
||||
response := errorResponse{Error: apiError{Code: code, Message: message, RequestID: requestID}}
|
||||
if err := json.NewEncoder(w).Encode(response); err != nil {
|
||||
log.Printf("failed to encode error response: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func writeServiceError(w http.ResponseWriter, err error, requestID string) {
|
||||
var upload repositories.UploadError
|
||||
if errors.As(err, &upload) {
|
||||
writeError(w, upload.Status, upload.Code, upload.Message, requestID)
|
||||
return
|
||||
}
|
||||
var invalid services.InvalidInput
|
||||
switch {
|
||||
case errors.As(err, &invalid):
|
||||
writeError(w, 400, "invalid_request", invalid.Message, requestID)
|
||||
case errors.Is(err, services.ErrPayloadTooLarge):
|
||||
writeError(w, 413, "payload_too_large", "image exceeds upload limits", requestID)
|
||||
case errors.Is(err, services.ErrUnsupportedMedia):
|
||||
writeError(w, 415, "unsupported_media_type", "a valid PNG or JPEG matching Content-Type is required", requestID)
|
||||
case errors.Is(err, services.ErrAttachmentNotFound):
|
||||
writeError(w, 404, "not_found", "attachment not found", requestID)
|
||||
case errors.Is(err, services.ErrConflict):
|
||||
writeError(w, 409, "conflict", "request conflicts with an existing resource", requestID)
|
||||
case errors.Is(err, services.ErrUnknownUser):
|
||||
writeError(w, 401, "unauthorized", "unknown user", requestID)
|
||||
case errors.Is(err, services.ErrChatNotFound):
|
||||
writeError(w, 404, "not_found", "chat not found", requestID)
|
||||
default:
|
||||
writeError(w, 503, "unavailable", "dependency unavailable", requestID)
|
||||
}
|
||||
}
|
||||
|
||||
func writeJSON(w http.ResponseWriter, status int, value any) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.WriteHeader(status)
|
||||
if err := json.NewEncoder(w).Encode(value); err != nil {
|
||||
log.Printf("failed to encode response: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func requestUserID(w http.ResponseWriter, r *http.Request, requestID string) (string, bool) {
|
||||
id := currentUserID(r)
|
||||
if id == "" {
|
||||
writeError(w, 401, "unauthorized", "valid Bearer session is required", requestID)
|
||||
return "", false
|
||||
}
|
||||
return id, true
|
||||
}
|
||||
|
||||
func pageParams(w http.ResponseWriter, r *http.Request, requestID string) (int, string, bool) {
|
||||
limit := 50
|
||||
if values, exists := r.URL.Query()["limit"]; exists {
|
||||
if len(values) != 1 || values[0] == "" {
|
||||
writeError(w, 400, "invalid_request", "invalid limit", requestID)
|
||||
return 0, "", false
|
||||
}
|
||||
parsed, err := strconv.Atoi(values[0])
|
||||
if err != nil {
|
||||
writeError(w, 400, "invalid_request", "limit must be between 1 and 100", requestID)
|
||||
return 0, "", false
|
||||
}
|
||||
limit = parsed
|
||||
}
|
||||
var cursor string
|
||||
if values, exists := r.URL.Query()["cursor"]; exists {
|
||||
if len(values) != 1 || values[0] == "" {
|
||||
writeError(w, 400, "invalid_request", "invalid cursor", requestID)
|
||||
return 0, "", false
|
||||
}
|
||||
cursor = values[0]
|
||||
}
|
||||
return limit, cursor, true
|
||||
}
|
||||
|
||||
func healthHandler(w http.ResponseWriter, r *http.Request) {
|
||||
requestID := newRequestID()
|
||||
w.Header().Set("X-Request-Id", requestID)
|
||||
if r.Method != http.MethodGet {
|
||||
writeError(w, 405, "method_not_allowed", "method not allowed", requestID)
|
||||
return
|
||||
}
|
||||
writeJSON(w, 200, healthResponse{"ok"})
|
||||
}
|
||||
|
||||
func readyHandler(health *services.HealthService) http.HandlerFunc {
|
||||
return func(w http.ResponseWriter, r *http.Request) {
|
||||
requestID := newRequestID()
|
||||
w.Header().Set("X-Request-Id", requestID)
|
||||
if r.Method != http.MethodGet {
|
||||
writeError(w, 405, "method_not_allowed", "method not allowed", requestID)
|
||||
return
|
||||
}
|
||||
// Readiness must finish before HAProxy's one-second check read timeout.
|
||||
ctx, cancel := context.WithTimeout(r.Context(), 500*time.Millisecond)
|
||||
defer cancel()
|
||||
if err := health.Ready(ctx); err != nil {
|
||||
writeError(w, 503, "unavailable", "dependency unavailable", requestID)
|
||||
return
|
||||
}
|
||||
writeJSON(w, 200, healthResponse{"ok"})
|
||||
}
|
||||
}
|
||||
|
||||
func NewRouter(health *services.HealthService, users *services.UserService, sessions *services.SessionService, chats *services.ChatService, messages *services.MessageService, attachmentServices ...*services.AttachmentService) http.Handler {
|
||||
var attachments *services.AttachmentService
|
||||
if len(attachmentServices) > 0 {
|
||||
attachments = attachmentServices[0]
|
||||
}
|
||||
mux := http.NewServeMux()
|
||||
mux.HandleFunc("/health/live", healthHandler)
|
||||
mux.HandleFunc("/health/ready", readyHandler(health))
|
||||
mux.Handle("/api/v1/users", createUserHandler(users))
|
||||
mux.Handle("/api/v1/auth/sessions", sessionHandler(sessions))
|
||||
mux.Handle("/api/v1/auth/sessions/current", authRequired(sessions, logoutHandler(sessions)))
|
||||
mux.Handle("/api/v1/users/me", authRequired(sessions, currentUserHandler(users)))
|
||||
mux.Handle("/api/v1/chats", authRequired(sessions, chatCollectionHandler(chats)))
|
||||
mux.Handle("/api/v1/chats/", authRequired(sessions, chatSubresourceHandler(chats, messages, attachments)))
|
||||
if attachments != nil {
|
||||
mux.Handle("/api/v1/attachments/", authRequired(sessions, attachmentHandler(attachments)))
|
||||
}
|
||||
return withInstance(mux)
|
||||
}
|
||||
@@ -0,0 +1,34 @@
|
||||
package handlers
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"os"
|
||||
"sync/atomic"
|
||||
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
|
||||
// Docker's hostname lets the operator map an API response to its container.
|
||||
var instanceID = func() string {
|
||||
id, err := os.Hostname()
|
||||
if err != nil || id == "" {
|
||||
return uuid.NewString()
|
||||
}
|
||||
return id
|
||||
}()
|
||||
|
||||
var draining atomic.Bool
|
||||
|
||||
func BeginShutdown() { draining.Store(true) }
|
||||
|
||||
func withInstance(next http.Handler) http.Handler {
|
||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("X-Instance-Id", instanceID)
|
||||
w.Header().Set("X-Request-Id", newRequestID())
|
||||
if draining.Load() && r.URL.Path != "/health/live" {
|
||||
writeError(w, 503, "unavailable", "instance is shutting down", w.Header().Get("X-Request-Id"))
|
||||
return
|
||||
}
|
||||
next.ServeHTTP(w, r)
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,127 @@
|
||||
package handlers
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"strings"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"messenger/src/backend/services"
|
||||
)
|
||||
|
||||
type createMessageRequest struct {
|
||||
Text *string `json:"text"`
|
||||
AttachmentIDs json.RawMessage `json:"attachment_ids"`
|
||||
}
|
||||
|
||||
func messageChatID(path string) (string, bool) {
|
||||
const prefix = "/api/v1/chats/"
|
||||
const suffix = "/messages"
|
||||
if !strings.HasPrefix(path, prefix) || !strings.HasSuffix(path, suffix) {
|
||||
return "", false
|
||||
}
|
||||
chatID := strings.TrimSuffix(strings.TrimPrefix(path, prefix), suffix)
|
||||
if chatID == "" || strings.Contains(chatID, "/") {
|
||||
return "", false
|
||||
}
|
||||
return chatID, true
|
||||
}
|
||||
|
||||
func createMessageHandler(chats *services.ChatService, messages *services.MessageService) http.HandlerFunc {
|
||||
return func(w http.ResponseWriter, r *http.Request) {
|
||||
requestID := newRequestID()
|
||||
w.Header().Set("X-Request-Id", requestID)
|
||||
if r.Method != http.MethodPost {
|
||||
writeError(w, 405, "method_not_allowed", "method not allowed", requestID)
|
||||
return
|
||||
}
|
||||
userID, ok := requestUserID(w, r, requestID)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
chatID, ok := messageChatID(r.URL.Path)
|
||||
if !ok {
|
||||
writeError(w, 400, "invalid_request", "invalid chat id", requestID)
|
||||
return
|
||||
}
|
||||
if _, err := uuid.Parse(chatID); err != nil {
|
||||
writeError(w, 400, "invalid_request", "invalid chat id", requestID)
|
||||
return
|
||||
}
|
||||
chat, err := chats.Get(r.Context(), userID, chatID)
|
||||
if err != nil {
|
||||
writeServiceError(w, err, requestID)
|
||||
return
|
||||
}
|
||||
if !strings.HasPrefix(r.Header.Get("Content-Type"), "application/json") {
|
||||
writeError(w, 415, "unsupported_media_type", "content type must be application/json", requestID)
|
||||
return
|
||||
}
|
||||
var request createMessageRequest
|
||||
if err := decodeJSONBody(r, &request); err != nil {
|
||||
writeError(w, 400, "invalid_request", "invalid JSON body", requestID)
|
||||
return
|
||||
}
|
||||
ids := []string{}
|
||||
if request.Text == nil {
|
||||
writeError(w, 400, "invalid_request", "text must be a string", requestID)
|
||||
return
|
||||
}
|
||||
if len(request.AttachmentIDs) > 0 {
|
||||
if err := json.Unmarshal(request.AttachmentIDs, &ids); err != nil || ids == nil {
|
||||
writeError(w, 400, "invalid_request", "attachment_ids must be an array of UUIDs", requestID)
|
||||
return
|
||||
}
|
||||
}
|
||||
keys := r.Header.Values("Idempotency-Key")
|
||||
if len(keys) != 1 {
|
||||
writeError(w, 400, "invalid_request", "exactly one Idempotency-Key is required", requestID)
|
||||
return
|
||||
}
|
||||
message, err := messages.CreateIdempotent(r.Context(), userID, chat, *request.Text, keys[0], ids)
|
||||
if err != nil {
|
||||
writeServiceError(w, err, requestID)
|
||||
return
|
||||
}
|
||||
writeJSON(w, 201, message)
|
||||
}
|
||||
}
|
||||
|
||||
func listMessagesHandler(chats *services.ChatService, messages *services.MessageService) http.HandlerFunc {
|
||||
return func(w http.ResponseWriter, r *http.Request) {
|
||||
requestID := newRequestID()
|
||||
w.Header().Set("X-Request-Id", requestID)
|
||||
if r.Method != http.MethodGet {
|
||||
writeError(w, 405, "method_not_allowed", "method not allowed", requestID)
|
||||
return
|
||||
}
|
||||
userID, ok := requestUserID(w, r, requestID)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
chatID, ok := messageChatID(r.URL.Path)
|
||||
if !ok {
|
||||
writeError(w, 400, "invalid_request", "invalid chat id", requestID)
|
||||
return
|
||||
}
|
||||
if _, err := uuid.Parse(chatID); err != nil {
|
||||
writeError(w, 400, "invalid_request", "invalid chat id", requestID)
|
||||
return
|
||||
}
|
||||
chat, err := chats.Get(r.Context(), userID, chatID)
|
||||
if err != nil {
|
||||
writeServiceError(w, err, requestID)
|
||||
return
|
||||
}
|
||||
limit, cursor, ok := pageParams(w, r, requestID)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
page, err := messages.List(r.Context(), userID, chat, limit, cursor)
|
||||
if err != nil {
|
||||
writeServiceError(w, err, requestID)
|
||||
return
|
||||
}
|
||||
writeJSON(w, 200, page)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,123 @@
|
||||
package handlers
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"net"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"messenger/src/backend/services"
|
||||
)
|
||||
|
||||
type authenticatedUserKey struct{}
|
||||
|
||||
func currentUserID(r *http.Request) string {
|
||||
id, _ := r.Context().Value(authenticatedUserKey{}).(string)
|
||||
return id
|
||||
}
|
||||
|
||||
func authRequired(sessions *services.SessionService, next http.Handler) http.Handler {
|
||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
authorization := r.Header.Get("Authorization")
|
||||
if !strings.HasPrefix(authorization, "Bearer ") || strings.TrimSpace(strings.TrimPrefix(authorization, "Bearer ")) == "" {
|
||||
unauthorized(w, "Bearer", newRequestID())
|
||||
return
|
||||
}
|
||||
user, err := sessions.User(r.Context(), strings.TrimPrefix(authorization, "Bearer "))
|
||||
if errors.Is(err, services.ErrInvalidSession) {
|
||||
unauthorized(w, "Bearer", newRequestID())
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
requestID := newRequestID()
|
||||
w.Header().Set("X-Request-Id", requestID)
|
||||
writeError(w, 503, "unavailable", "dependency unavailable", requestID)
|
||||
return
|
||||
}
|
||||
next.ServeHTTP(w, r.WithContext(context.WithValue(r.Context(), authenticatedUserKey{}, user.ID)))
|
||||
})
|
||||
}
|
||||
|
||||
func unauthorized(w http.ResponseWriter, scheme, requestID string) {
|
||||
w.Header().Set("X-Request-Id", requestID)
|
||||
if scheme == "Basic" {
|
||||
w.Header().Set("WWW-Authenticate", `Basic realm="messenger", charset="UTF-8"`)
|
||||
} else {
|
||||
w.Header().Set("WWW-Authenticate", `Bearer realm="messenger"`)
|
||||
}
|
||||
writeError(w, 401, "unauthorized", "invalid credentials", requestID)
|
||||
}
|
||||
|
||||
func sessionHandler(sessions *services.SessionService) http.HandlerFunc {
|
||||
return func(w http.ResponseWriter, r *http.Request) {
|
||||
requestID := newRequestID()
|
||||
w.Header().Set("X-Request-Id", requestID)
|
||||
if r.Method != http.MethodPost {
|
||||
writeError(w, 405, "method_not_allowed", "method not allowed", requestID)
|
||||
return
|
||||
}
|
||||
// HAProxy overwrites this header and is the only mTLS identity allowed to reach the API.
|
||||
if r.TLS != nil {
|
||||
source := net.ParseIP(r.Header.Get("X-Forwarded-For"))
|
||||
if source == nil {
|
||||
writeError(w, 503, "unavailable", "missing ingress client address", requestID)
|
||||
return
|
||||
}
|
||||
retryAfter, err := sessions.LoginRetryAfter(r.Context(), source.String())
|
||||
if err != nil {
|
||||
writeError(w, 503, "unavailable", "dependency unavailable", requestID)
|
||||
return
|
||||
}
|
||||
if retryAfter > 0 {
|
||||
w.Header().Set("Retry-After", strconv.Itoa(retryAfter))
|
||||
writeError(w, 429, "rate_limited", "too many login attempts", requestID)
|
||||
return
|
||||
}
|
||||
}
|
||||
username, password, ok := r.BasicAuth()
|
||||
if !ok || r.ContentLength > 0 {
|
||||
unauthorized(w, "Basic", requestID)
|
||||
return
|
||||
}
|
||||
user, token, expires, valid, err := sessions.Login(r.Context(), username, password)
|
||||
if errors.Is(err, services.ErrPasswordBusy) {
|
||||
w.Header().Set("Retry-After", "1")
|
||||
writeError(w, 429, "rate_limited", "login capacity exhausted", requestID)
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
writeError(w, 503, "unavailable", "dependency unavailable", requestID)
|
||||
return
|
||||
}
|
||||
if !valid {
|
||||
unauthorized(w, "Basic", requestID)
|
||||
return
|
||||
}
|
||||
writeJSON(w, 201, struct {
|
||||
Token string `json:"token"`
|
||||
TokenType string `json:"token_type"`
|
||||
ExpiresAt time.Time `json:"expires_at"`
|
||||
User any `json:"user"`
|
||||
}{token, "Bearer", expires, user})
|
||||
}
|
||||
}
|
||||
|
||||
func logoutHandler(sessions *services.SessionService) http.HandlerFunc {
|
||||
return func(w http.ResponseWriter, r *http.Request) {
|
||||
requestID := newRequestID()
|
||||
w.Header().Set("X-Request-Id", requestID)
|
||||
if r.Method != http.MethodDelete {
|
||||
writeError(w, 405, "method_not_allowed", "method not allowed", requestID)
|
||||
return
|
||||
}
|
||||
token := strings.TrimPrefix(r.Header.Get("Authorization"), "Bearer ")
|
||||
if err := sessions.Revoke(r.Context(), token); err != nil {
|
||||
writeError(w, 503, "unavailable", "dependency unavailable", requestID)
|
||||
return
|
||||
}
|
||||
w.WriteHeader(204)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,103 @@
|
||||
package handlers
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"reflect"
|
||||
"strings"
|
||||
|
||||
"messenger/src/backend/services"
|
||||
)
|
||||
|
||||
type createUserRequest struct {
|
||||
Username string `json:"username"`
|
||||
DisplayName string `json:"display_name"`
|
||||
Password string `json:"password"`
|
||||
}
|
||||
|
||||
func decodeJSONBody(r *http.Request, destination any) error {
|
||||
decoder := json.NewDecoder(r.Body)
|
||||
var body json.RawMessage
|
||||
if err := decoder.Decode(&body); err != nil {
|
||||
return err
|
||||
}
|
||||
var extra any
|
||||
if err := decoder.Decode(&extra); err != io.EOF {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return fmt.Errorf("more than one JSON value")
|
||||
}
|
||||
var fields map[string]json.RawMessage
|
||||
if err := json.Unmarshal(body, &fields); err != nil {
|
||||
return err
|
||||
}
|
||||
if fields == nil {
|
||||
return fmt.Errorf("JSON body must be an object")
|
||||
}
|
||||
// encoding/json matches struct fields case-insensitively, even with
|
||||
// DisallowUnknownFields. Check exact contract names before decoding values.
|
||||
typeOfRequest := reflect.TypeOf(destination).Elem()
|
||||
allowed := make(map[string]bool, typeOfRequest.NumField())
|
||||
for i := 0; i < typeOfRequest.NumField(); i++ {
|
||||
name := strings.Split(typeOfRequest.Field(i).Tag.Get("json"), ",")[0]
|
||||
if name != "" && name != "-" {
|
||||
allowed[name] = true
|
||||
}
|
||||
}
|
||||
for name := range fields {
|
||||
if !allowed[name] {
|
||||
return fmt.Errorf("unknown JSON field %q", name)
|
||||
}
|
||||
}
|
||||
return json.Unmarshal(body, destination)
|
||||
}
|
||||
|
||||
func createUserHandler(users *services.UserService) http.HandlerFunc {
|
||||
return func(w http.ResponseWriter, r *http.Request) {
|
||||
requestID := newRequestID()
|
||||
w.Header().Set("X-Request-Id", requestID)
|
||||
if r.Method != http.MethodPost {
|
||||
writeError(w, 405, "method_not_allowed", "method not allowed", requestID)
|
||||
return
|
||||
}
|
||||
if !strings.HasPrefix(r.Header.Get("Content-Type"), "application/json") {
|
||||
writeError(w, 415, "unsupported_media_type", "content type must be application/json", requestID)
|
||||
return
|
||||
}
|
||||
var request createUserRequest
|
||||
if err := decodeJSONBody(r, &request); err != nil {
|
||||
writeError(w, 400, "invalid_request", "invalid JSON body", requestID)
|
||||
return
|
||||
}
|
||||
user, err := users.Create(r.Context(), request.Username, request.DisplayName, request.Password)
|
||||
if err != nil {
|
||||
writeServiceError(w, err, requestID)
|
||||
return
|
||||
}
|
||||
writeJSON(w, 201, user)
|
||||
}
|
||||
}
|
||||
|
||||
func currentUserHandler(users *services.UserService) http.HandlerFunc {
|
||||
return func(w http.ResponseWriter, r *http.Request) {
|
||||
requestID := newRequestID()
|
||||
w.Header().Set("X-Request-Id", requestID)
|
||||
if r.Method != http.MethodGet {
|
||||
writeError(w, 405, "method_not_allowed", "method not allowed", requestID)
|
||||
return
|
||||
}
|
||||
userID, ok := requestUserID(w, r, requestID)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
user, err := users.Get(r.Context(), userID)
|
||||
if err != nil {
|
||||
writeServiceError(w, err, requestID)
|
||||
return
|
||||
}
|
||||
writeJSON(w, 200, user)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,64 @@
|
||||
package faults
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
|
||||
// Only an operator-mounted directory enables faults; clients cannot set them.
|
||||
// Rename makes each injected crash one-shot across automatic container restarts.
|
||||
func Take(point, id string) bool {
|
||||
directory := os.Getenv("PIPELINE_FAULT_DIR")
|
||||
if directory == "" {
|
||||
return false
|
||||
}
|
||||
if _, err := uuid.Parse(id); err != nil {
|
||||
return false
|
||||
}
|
||||
name := filepath.Join(directory, point+"."+id)
|
||||
return os.Rename(name, name+".hit") == nil
|
||||
}
|
||||
func Crash(point, id string) {
|
||||
if Take(point, id) {
|
||||
log.Printf("injected pipeline crash: %s, attachment %s", point, id)
|
||||
os.Exit(86)
|
||||
}
|
||||
}
|
||||
|
||||
// Observe records a test-only branch without exposing an API or a lease token.
|
||||
func Observe(point, id string) {
|
||||
directory := os.Getenv("PIPELINE_FAULT_DIR")
|
||||
if directory == "" {
|
||||
return
|
||||
}
|
||||
if _, err := uuid.Parse(id); err != nil {
|
||||
return
|
||||
}
|
||||
_ = os.WriteFile(filepath.Join(directory, point+"."+id+".observed"), []byte("observed\n"), 0644)
|
||||
}
|
||||
|
||||
// Pause is operator-only and ends on release or context expiry.
|
||||
func Pause(ctx context.Context, point, id string) {
|
||||
if !Take(point, id) {
|
||||
return
|
||||
}
|
||||
release := filepath.Join(os.Getenv("PIPELINE_FAULT_DIR"), point+"."+id+".release")
|
||||
ticker := time.NewTicker(50 * time.Millisecond)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-ticker.C:
|
||||
if _, err := os.Stat(release); err == nil {
|
||||
_ = os.WriteFile(release+".observed", []byte("released\n"), 0644)
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"log"
|
||||
"os"
|
||||
|
||||
"messenger/src/backend/commands"
|
||||
)
|
||||
|
||||
func main() {
|
||||
if err := commands.Run(os.Args[1:]); err != nil {
|
||||
if len(os.Args) > 1 && (os.Args[1] == "image-process" || os.Args[1] == "healthcheck") {
|
||||
os.Exit(1)
|
||||
}
|
||||
log.Fatal(err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,48 @@
|
||||
package repositories
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
"time"
|
||||
)
|
||||
|
||||
type Attachment struct {
|
||||
ID string `json:"id"`
|
||||
ChatID string `json:"chat_id"`
|
||||
OwnerID string `json:"owner_id"`
|
||||
Filename string `json:"filename"`
|
||||
ContentType string `json:"content_type"`
|
||||
SizeBytes int64 `json:"size_bytes"`
|
||||
SHA256 string `json:"sha256"`
|
||||
Status string `json:"status"`
|
||||
Variants []any `json:"variants"`
|
||||
ErrorCode *string `json:"error_code"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
UpdatedAt time.Time `json:"updated_at"`
|
||||
ObjectKey string `json:"-"`
|
||||
PipelineVersion int `json:"-"`
|
||||
}
|
||||
|
||||
type AttachmentStore struct{ db *pgxpool.Pool }
|
||||
|
||||
func NewAttachmentStore(db *pgxpool.Pool) *AttachmentStore { return &AttachmentStore{db: db} }
|
||||
func (s *AttachmentStore) Create(ctx context.Context, a Attachment) error {
|
||||
ctx, cancel := boundedQuery(ctx)
|
||||
defer cancel()
|
||||
_, err := s.db.Exec(ctx, `INSERT INTO attachments(id,chat_id,owner_id,filename,object_key,content_type,size_bytes,sha256,created_at,updated_at) VALUES($1,$2,$3,$4,$5,$6,$7,$8,$9,$10)`, a.ID, a.ChatID, a.OwnerID, a.Filename, a.ObjectKey, a.ContentType, a.SizeBytes, a.SHA256, a.CreatedAt, a.UpdatedAt)
|
||||
return err
|
||||
}
|
||||
func (s *AttachmentStore) GetForMember(ctx context.Context, id, userID string) (Attachment, error) {
|
||||
ctx, cancel := boundedQuery(ctx)
|
||||
defer cancel()
|
||||
a := Attachment{Variants: []any{}}
|
||||
var variants []byte
|
||||
err := s.db.QueryRow(ctx, `SELECT a.id,a.chat_id,a.owner_id,a.filename,a.object_key,a.content_type,a.size_bytes,a.sha256,a.created_at,a.updated_at,a.status,a.variants,a.error_code,a.published_pipeline_version FROM attachments a JOIN chat_members cm ON cm.chat_id=a.chat_id WHERE a.id=$1 AND cm.user_id=$2`, id, userID).Scan(&a.ID, &a.ChatID, &a.OwnerID, &a.Filename, &a.ObjectKey, &a.ContentType, &a.SizeBytes, &a.SHA256, &a.CreatedAt, &a.UpdatedAt, &a.Status, &variants, &a.ErrorCode, &a.PipelineVersion)
|
||||
if err == nil {
|
||||
err = json.Unmarshal(variants, &a.Variants)
|
||||
}
|
||||
a.CreatedAt = a.CreatedAt.UTC()
|
||||
a.UpdatedAt = a.UpdatedAt.UTC()
|
||||
return a, err
|
||||
}
|
||||
@@ -0,0 +1,94 @@
|
||||
package repositories
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/tls"
|
||||
"os"
|
||||
|
||||
"errors"
|
||||
"messenger/src/backend/telemetry"
|
||||
"net"
|
||||
"time"
|
||||
|
||||
amqp "github.com/rabbitmq/amqp091-go"
|
||||
)
|
||||
|
||||
const jobQueue = "attachments.v1"
|
||||
|
||||
type JobBroker struct {
|
||||
connection *amqp.Connection
|
||||
channel *amqp.Channel
|
||||
returned chan amqp.Return
|
||||
}
|
||||
|
||||
func NewJobBroker(ctx context.Context, address string) (*JobBroker, error) {
|
||||
config := amqp.Config{
|
||||
Heartbeat: 10 * time.Second,
|
||||
Dial: func(network, address string) (net.Conn, error) {
|
||||
return (&net.Dialer{Timeout: 5 * time.Second}).DialContext(ctx, network, address)
|
||||
},
|
||||
}
|
||||
if os.Getenv("BROKER_MTLS") == "1" {
|
||||
cert, err := tls.LoadX509KeyPair("/run/broker/client.crt", "/run/broker/client.key")
|
||||
if err != nil {
|
||||
return nil, errors.New("RabbitMQ client certificate unavailable")
|
||||
}
|
||||
config.TLSClientConfig = &tls.Config{MinVersion: tls.VersionTLS12, Certificates: []tls.Certificate{cert}}
|
||||
config.SASL = []amqp.Authentication{&amqp.ExternalAuth{}}
|
||||
}
|
||||
connection, err := amqp.DialConfig(address, config)
|
||||
if err != nil {
|
||||
return nil, errors.New("RabbitMQ connection unavailable")
|
||||
}
|
||||
channel, err := connection.Channel()
|
||||
if err != nil {
|
||||
connection.Close()
|
||||
return nil, errors.New("RabbitMQ channel unavailable")
|
||||
}
|
||||
b := &JobBroker{connection: connection, channel: channel, returned: make(chan amqp.Return, 1)}
|
||||
// Queue metadata and message bodies both survive a broker restart.
|
||||
_, err = channel.QueueDeclare(jobQueue, true, false, false, false, nil)
|
||||
if err == nil {
|
||||
err = channel.Confirm(false)
|
||||
}
|
||||
if err == nil {
|
||||
err = channel.Qos(1, 0, false)
|
||||
}
|
||||
if err != nil {
|
||||
b.Close()
|
||||
return nil, errors.New("RabbitMQ queue initialization failed")
|
||||
}
|
||||
channel.NotifyReturn(b.returned)
|
||||
return b, nil
|
||||
}
|
||||
|
||||
func (b *JobBroker) Close() { b.channel.Close(); b.connection.Close() }
|
||||
|
||||
func (b *JobBroker) Publish(ctx context.Context, body []byte) error {
|
||||
// mandatory catches an unroutable message; confirms alone do not prove routing.
|
||||
headers := amqp.Table{}
|
||||
for key, value := range telemetry.TraceHeaders(ctx) {
|
||||
headers[key] = value
|
||||
}
|
||||
confirmation, err := b.channel.PublishWithDeferredConfirmWithContext(ctx, "", jobQueue, true, false, amqp.Publishing{
|
||||
DeliveryMode: amqp.Persistent, ContentType: "application/json", Body: body, Headers: headers,
|
||||
})
|
||||
if err != nil {
|
||||
return errors.New("RabbitMQ publication failed")
|
||||
}
|
||||
confirmed, err := confirmation.WaitContext(ctx)
|
||||
if err != nil || !confirmed {
|
||||
return errors.New("RabbitMQ publication not confirmed")
|
||||
}
|
||||
// basic.return precedes basic.ack on the channel; NotifyReturn is buffered.
|
||||
select {
|
||||
case <-b.returned:
|
||||
return errors.New("RabbitMQ message was not routed")
|
||||
default:
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
func (b *JobBroker) Consume() (<-chan amqp.Delivery, error) {
|
||||
return b.channel.Consume(jobQueue, "", false, false, false, false, nil)
|
||||
}
|
||||
@@ -0,0 +1,129 @@
|
||||
package repositories
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
)
|
||||
|
||||
type Chat struct {
|
||||
ID string `json:"id"`
|
||||
Title string `json:"title"`
|
||||
MemberIDs []string `json:"member_ids"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
}
|
||||
|
||||
type ChatStore struct {
|
||||
db *pgxpool.Pool
|
||||
}
|
||||
|
||||
func NewChatStore(db *pgxpool.Pool) *ChatStore {
|
||||
return &ChatStore{db: db}
|
||||
}
|
||||
|
||||
var ErrUnknownMember = errors.New("unknown member")
|
||||
|
||||
func (s *ChatStore) Create(ctx context.Context, title string, memberIDs []string) (Chat, error) {
|
||||
ctx, cancel := boundedQuery(ctx)
|
||||
defer cancel()
|
||||
membersCopy := append([]string(nil), memberIDs...)
|
||||
chat := Chat{
|
||||
ID: uuid.New().String(),
|
||||
Title: title,
|
||||
MemberIDs: membersCopy,
|
||||
CreatedAt: time.Now().UTC().Truncate(time.Microsecond),
|
||||
}
|
||||
|
||||
tx, err := s.db.Begin(ctx)
|
||||
if err != nil {
|
||||
return Chat{}, err
|
||||
}
|
||||
defer tx.Rollback(ctx)
|
||||
for _, id := range membersCopy {
|
||||
var exists bool
|
||||
if err = tx.QueryRow(ctx, "SELECT EXISTS(SELECT 1 FROM users WHERE id=$1)", id).Scan(&exists); err != nil {
|
||||
return Chat{}, err
|
||||
}
|
||||
if !exists {
|
||||
return Chat{}, ErrUnknownMember
|
||||
}
|
||||
}
|
||||
if _, err = tx.Exec(ctx, "INSERT INTO chats(id,title,created_at) VALUES($1,$2,$3)", chat.ID, title, chat.CreatedAt); err != nil {
|
||||
return Chat{}, err
|
||||
}
|
||||
for _, id := range membersCopy {
|
||||
if _, err = tx.Exec(ctx, "INSERT INTO chat_members(chat_id,user_id) VALUES($1,$2)", chat.ID, id); err != nil {
|
||||
return Chat{}, err
|
||||
}
|
||||
}
|
||||
if err = tx.Commit(ctx); err != nil {
|
||||
return Chat{}, err
|
||||
}
|
||||
return chat, nil
|
||||
}
|
||||
|
||||
func (s *ChatStore) GetByID(ctx context.Context, id string) (Chat, error) {
|
||||
ctx, cancel := boundedQuery(ctx)
|
||||
defer cancel()
|
||||
var chat Chat
|
||||
err := s.db.QueryRow(ctx, "SELECT id,title,created_at FROM chats WHERE id=$1", id).Scan(&chat.ID, &chat.Title, &chat.CreatedAt)
|
||||
if err != nil {
|
||||
return Chat{}, err
|
||||
}
|
||||
rows, err := s.db.Query(ctx, "SELECT user_id FROM chat_members WHERE chat_id=$1 ORDER BY user_id", id)
|
||||
if err != nil {
|
||||
return Chat{}, err
|
||||
}
|
||||
defer rows.Close()
|
||||
chat.MemberIDs = []string{}
|
||||
for rows.Next() {
|
||||
var member string
|
||||
if err = rows.Scan(&member); err != nil {
|
||||
return Chat{}, err
|
||||
}
|
||||
chat.MemberIDs = append(chat.MemberIDs, member)
|
||||
}
|
||||
return chat, rows.Err()
|
||||
}
|
||||
|
||||
func (s *ChatStore) ListPageForUser(ctx context.Context, userID string, cursor *ChatCursor, limit int) ([]Chat, bool, error) {
|
||||
ctx, cancel := boundedQuery(ctx)
|
||||
defer cancel()
|
||||
afterTime := time.Time{}
|
||||
afterID := "00000000-0000-0000-0000-000000000000"
|
||||
if cursor != nil {
|
||||
afterTime = cursor.CreatedAt
|
||||
afterID = cursor.ChatID
|
||||
}
|
||||
rows, err := s.db.Query(ctx, `SELECT c.id,c.title,c.created_at FROM chats c JOIN chat_members m ON m.chat_id=c.id WHERE m.user_id=$1 AND (c.created_at,c.id)>($2,$3::uuid) ORDER BY c.created_at,c.id LIMIT $4`, userID, afterTime, afterID, limit+1)
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
defer rows.Close()
|
||||
chats := []Chat{}
|
||||
for rows.Next() {
|
||||
var c Chat
|
||||
if err = rows.Scan(&c.ID, &c.Title, &c.CreatedAt); err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
chats = append(chats, c)
|
||||
}
|
||||
if err = rows.Err(); err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
more := len(chats) > limit
|
||||
if more {
|
||||
chats = chats[:limit]
|
||||
}
|
||||
for i := range chats {
|
||||
chat, err := s.GetByID(ctx, chats[i].ID)
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
chats[i] = chat
|
||||
}
|
||||
return chats, more, nil
|
||||
}
|
||||
@@ -0,0 +1,20 @@
|
||||
package repositories
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"os"
|
||||
"strconv"
|
||||
"time"
|
||||
)
|
||||
|
||||
func UploadCleanupGrace() (time.Duration, error) {
|
||||
value := os.Getenv("UPLOAD_CLEANUP_GRACE_SECONDS")
|
||||
if value == "" {
|
||||
value = "600"
|
||||
}
|
||||
seconds, err := strconv.Atoi(value)
|
||||
if err != nil || seconds < 120 || seconds > 86400 {
|
||||
return 0, errors.New("UPLOAD_CLEANUP_GRACE_SECONDS must be between 120 and 86400")
|
||||
}
|
||||
return time.Duration(seconds) * time.Second, nil
|
||||
}
|
||||
@@ -0,0 +1,47 @@
|
||||
package repositories
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/redis/go-redis/v9"
|
||||
)
|
||||
|
||||
func RedisClientFromEnvironment() (*redis.Client, error) {
|
||||
if path := os.Getenv("CACHE_URL_FILE"); path != "" {
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
options, err := redis.ParseURL(strings.TrimSpace(string(data)))
|
||||
if err != nil {
|
||||
return nil, errors.New("invalid Redis URL in CACHE_URL_FILE")
|
||||
}
|
||||
options.PoolSize = 10
|
||||
options.MaxRetries = -1
|
||||
options.DialTimeout = time.Second
|
||||
options.ReadTimeout = time.Second
|
||||
options.WriteTimeout = time.Second
|
||||
return redis.NewClient(options), nil
|
||||
}
|
||||
address := os.Getenv("REDIS_ADDR")
|
||||
if address == "" {
|
||||
address = "redis:6379"
|
||||
}
|
||||
return NewRedisClient(address), nil
|
||||
}
|
||||
|
||||
func IdempotencyTTL() (time.Duration, error) {
|
||||
value := os.Getenv("IDEMPOTENCY_TTL_SECONDS")
|
||||
if value == "" {
|
||||
value = "86400"
|
||||
}
|
||||
seconds, err := strconv.ParseInt(value, 10, 64)
|
||||
if err != nil || seconds < 1 || seconds > 31536000 {
|
||||
return 0, errors.New("IDEMPOTENCY_TTL_SECONDS must be between 1 and 31536000")
|
||||
}
|
||||
return time.Duration(seconds) * time.Second, nil
|
||||
}
|
||||
@@ -0,0 +1,15 @@
|
||||
package repositories
|
||||
|
||||
import "time"
|
||||
|
||||
type ChatCursor struct {
|
||||
UserID string `json:"user_id"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
ChatID string `json:"chat_id"`
|
||||
}
|
||||
|
||||
type MessageCursor struct {
|
||||
ChatID string `json:"chat_id"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
MessageID string `json:"message_id"`
|
||||
}
|
||||
@@ -0,0 +1,100 @@
|
||||
package repositories
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"embed"
|
||||
"errors"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"messenger/src/backend/telemetry"
|
||||
|
||||
"github.com/golang-migrate/migrate/v4"
|
||||
pgxmigrate "github.com/golang-migrate/migrate/v4/database/pgx/v5"
|
||||
"github.com/golang-migrate/migrate/v4/source/iofs"
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
_ "github.com/jackc/pgx/v5/stdlib"
|
||||
)
|
||||
|
||||
//go:embed migrations/*.sql
|
||||
var migrationFiles embed.FS
|
||||
|
||||
func DatabaseURL() (string, error) {
|
||||
if path := os.Getenv("DATABASE_URL_FILE"); path != "" {
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return strings.TrimSpace(string(data)), nil
|
||||
}
|
||||
if url := os.Getenv("DATABASE_URL"); url != "" {
|
||||
return url, nil
|
||||
}
|
||||
return "", errors.New("DATABASE_URL_FILE or DATABASE_URL is required")
|
||||
}
|
||||
|
||||
func OpenDatabase(ctx context.Context) (*pgxpool.Pool, error) {
|
||||
url, err := DatabaseURL()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
cfg, err := pgxpool.ParseConfig(url)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
cfg.ConnConfig.Tracer = telemetry.PostgresTracer{}
|
||||
cfg.MaxConns = 10
|
||||
if value := os.Getenv("DATABASE_MAX_CONNS"); value != "" {
|
||||
maxConns, parseErr := strconv.Atoi(value)
|
||||
if parseErr != nil || maxConns < 1 || maxConns > 100 {
|
||||
return nil, errors.New("DATABASE_MAX_CONNS must be between 1 and 100")
|
||||
}
|
||||
cfg.MaxConns = int32(maxConns)
|
||||
}
|
||||
cfg.ConnConfig.ConnectTimeout = 5 * time.Second
|
||||
db, err := pgxpool.NewWithConfig(ctx, cfg)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err = db.Ping(ctx); err != nil {
|
||||
db.Close()
|
||||
return nil, err
|
||||
}
|
||||
return db, nil
|
||||
}
|
||||
|
||||
// RunMigrations uses versioned up/down SQL embedded in the API binary.
|
||||
// "down" rolls back exactly one version and is an explicit operator command.
|
||||
func RunMigrations(url, direction string) error {
|
||||
source, err := iofs.New(migrationFiles, "migrations")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
sqlDB, err := sql.Open("pgx", url)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
driver, err := pgxmigrate.WithInstance(sqlDB, &pgxmigrate.Config{})
|
||||
if err != nil {
|
||||
sqlDB.Close()
|
||||
return err
|
||||
}
|
||||
m, err := migrate.NewWithInstance("iofs", source, "pgx5", driver)
|
||||
if err != nil {
|
||||
sqlDB.Close()
|
||||
return err
|
||||
}
|
||||
defer m.Close()
|
||||
if direction == "down" {
|
||||
err = m.Steps(-1)
|
||||
} else {
|
||||
err = m.Up()
|
||||
}
|
||||
if errors.Is(err, migrate.ErrNoChange) {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
@@ -0,0 +1,106 @@
|
||||
package repositories
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/jackc/pgx/v5"
|
||||
)
|
||||
|
||||
func (s *AttachmentStore) CleanupDirectUploads(ctx context.Context, dryRun bool, remove func(context.Context, string) error) (CleanupResult, error) {
|
||||
result := CleanupResult{DryRun: dryRun}
|
||||
rows, err := s.db.Query(ctx, `SELECT id::text FROM direct_uploads WHERE cleanup_after<=clock_timestamp() ORDER BY cleanup_after,id LIMIT 100`)
|
||||
if err != nil {
|
||||
return result, err
|
||||
}
|
||||
ids := []string{}
|
||||
for rows.Next() {
|
||||
var id string
|
||||
if err = rows.Scan(&id); err != nil {
|
||||
rows.Close()
|
||||
return result, err
|
||||
}
|
||||
ids = append(ids, id)
|
||||
}
|
||||
err = rows.Err()
|
||||
rows.Close()
|
||||
if err != nil {
|
||||
return result, err
|
||||
}
|
||||
result.Candidates = len(ids)
|
||||
for _, id := range ids {
|
||||
outcome, err := s.cleanupDirectUpload(ctx, id, dryRun, remove)
|
||||
if err != nil {
|
||||
result.Failed++
|
||||
continue
|
||||
}
|
||||
switch outcome {
|
||||
case "deleted":
|
||||
result.Deleted++
|
||||
case "eligible":
|
||||
result.Eligible++
|
||||
default:
|
||||
result.Skipped++
|
||||
}
|
||||
}
|
||||
if result.Failed > 0 {
|
||||
return result, fmt.Errorf("%d direct upload cleanup operations failed", result.Failed)
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func (s *AttachmentStore) cleanupDirectUpload(ctx context.Context, id string, dryRun bool, remove func(context.Context, string) error) (string, error) {
|
||||
ctx, cancel := context.WithTimeout(ctx, 10*time.Second)
|
||||
defer cancel()
|
||||
tx, err := s.db.Begin(ctx)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer tx.Rollback(context.Background())
|
||||
var key, state string
|
||||
var barrier, complete, purge, now time.Time
|
||||
err = tx.QueryRow(ctx, `SELECT staging_key,state,complete_before+interval '600 seconds',complete_before,purge_after,clock_timestamp() FROM direct_uploads WHERE id=$1 AND cleanup_after<=clock_timestamp() FOR UPDATE SKIP LOCKED`, id).Scan(&key, &state, &barrier, &complete, &purge, &now)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return "skipped", nil
|
||||
}
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
parsed, err := uuid.Parse(id)
|
||||
if err != nil || key != "staging/direct/"+parsed.String() {
|
||||
return "", errors.New("invalid direct staging key")
|
||||
}
|
||||
if state == "pending" && now.Before(barrier) {
|
||||
return "skipped", nil
|
||||
}
|
||||
if dryRun {
|
||||
return "eligible", nil
|
||||
}
|
||||
if err = remove(ctx, key); err != nil {
|
||||
return "", err
|
||||
}
|
||||
if now.After(purge) && !now.Before(barrier) {
|
||||
_, err = tx.Exec(ctx, `DELETE FROM direct_uploads WHERE id=$1`, id)
|
||||
} else {
|
||||
if state == "pending" && !now.Before(complete) {
|
||||
state = "expired"
|
||||
}
|
||||
// An early cleanup is not final: a still valid POST can recreate staging.
|
||||
// Keep a second cleanup after the safety barrier, then retain the receipt 24h.
|
||||
next := barrier
|
||||
if !now.Before(barrier) {
|
||||
next = purge
|
||||
}
|
||||
_, err = tx.Exec(ctx, `UPDATE direct_uploads SET state=$2,cleanup_after=$3 WHERE id=$1`, id, state, next)
|
||||
}
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if err = tx.Commit(ctx); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return "deleted", nil
|
||||
}
|
||||
@@ -0,0 +1,150 @@
|
||||
package repositories
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
)
|
||||
|
||||
// UploadError is also persisted for terminal validation failures, so retries agree.
|
||||
type UploadError struct {
|
||||
Status int
|
||||
Code, Message string
|
||||
}
|
||||
|
||||
func (e UploadError) Error() string { return e.Message }
|
||||
|
||||
func DirectError(code string) UploadError {
|
||||
switch code {
|
||||
case "upload_not_uploaded":
|
||||
return UploadError{409, code, "file has not been uploaded"}
|
||||
case "upload_expired":
|
||||
return UploadError{409, code, "upload confirmation period has expired"}
|
||||
case "upload_size_mismatch":
|
||||
return UploadError{400, code, "uploaded size differs from declared size"}
|
||||
case "upload_checksum_mismatch":
|
||||
return UploadError{400, code, "uploaded SHA-256 differs from declared checksum"}
|
||||
case "payload_too_large":
|
||||
return UploadError{413, code, "image exceeds upload limits"}
|
||||
default:
|
||||
return UploadError{415, "unsupported_media_type", "a valid PNG or JPEG matching Content-Type is required"}
|
||||
}
|
||||
}
|
||||
|
||||
type DirectUpload struct {
|
||||
ID string `json:"upload_id"`
|
||||
AttachmentID string `json:"-"`
|
||||
OwnerID string `json:"-"`
|
||||
ChatID string `json:"-"`
|
||||
Filename string `json:"-"`
|
||||
ContentType string `json:"-"`
|
||||
SizeBytes int64 `json:"-"`
|
||||
SHA256 string `json:"-"`
|
||||
StagingKey string `json:"-"`
|
||||
FinalKey string `json:"-"`
|
||||
State string `json:"-"`
|
||||
ErrorCode *string `json:"-"`
|
||||
ExpiresAt time.Time `json:"expires_at"`
|
||||
CompleteBefore time.Time `json:"complete_before"`
|
||||
}
|
||||
|
||||
const directColumns = `d.id,d.attachment_id,d.owner_id,d.chat_id,d.filename,d.content_type,d.size_bytes,d.sha256,d.staging_key,d.final_key,d.state,d.error_code,d.expires_at,d.complete_before`
|
||||
|
||||
func scanDirect(row pgx.Row) (DirectUpload, error) {
|
||||
var u DirectUpload
|
||||
err := row.Scan(&u.ID, &u.AttachmentID, &u.OwnerID, &u.ChatID, &u.Filename, &u.ContentType, &u.SizeBytes, &u.SHA256, &u.StagingKey, &u.FinalKey, &u.State, &u.ErrorCode, &u.ExpiresAt, &u.CompleteBefore)
|
||||
return u, err
|
||||
}
|
||||
|
||||
func (s *AttachmentStore) CreateDirect(ctx context.Context, u DirectUpload) (DirectUpload, error) {
|
||||
ctx, cancel := boundedQuery(ctx)
|
||||
defer cancel()
|
||||
return scanDirect(s.db.QueryRow(ctx, `INSERT INTO direct_uploads AS d(id,attachment_id,owner_id,chat_id,filename,content_type,size_bytes,sha256,staging_key,final_key) VALUES($1,$2,$3,$4,$5,$6,$7,$8,$9,$10) RETURNING `+directColumns, u.ID, u.AttachmentID, u.OwnerID, u.ChatID, u.Filename, u.ContentType, u.SizeBytes, u.SHA256, u.StagingKey, u.FinalKey))
|
||||
}
|
||||
|
||||
func (s *AttachmentStore) GetDirect(ctx context.Context, id, chat, user string) (DirectUpload, error) {
|
||||
ctx, cancel := boundedQuery(ctx)
|
||||
defer cancel()
|
||||
return scanDirect(s.db.QueryRow(ctx, `SELECT `+directColumns+` FROM direct_uploads d JOIN chat_members cm ON cm.chat_id=d.chat_id AND cm.user_id=$3 WHERE d.id=$1 AND d.chat_id=$2 AND d.owner_id=$3`, id, chat, user))
|
||||
}
|
||||
|
||||
// The session lock serializes complete across replicas. A separately committed
|
||||
// reservation survives rollback/crash, while its row lock protects the final key.
|
||||
func (s *AttachmentStore) CompleteDirect(ctx context.Context, id, chat, user string, validate func(context.Context, DirectUpload) error) (Attachment, error) {
|
||||
var a Attachment
|
||||
tx, err := s.db.Begin(ctx)
|
||||
if err != nil {
|
||||
return a, err
|
||||
}
|
||||
defer tx.Rollback(context.Background())
|
||||
u, err := scanDirect(tx.QueryRow(ctx, `SELECT `+directColumns+` FROM direct_uploads d JOIN chat_members cm ON cm.chat_id=d.chat_id AND cm.user_id=$3 WHERE d.id=$1 AND d.chat_id=$2 AND d.owner_id=$3 FOR UPDATE OF d`, id, chat, user))
|
||||
if err != nil {
|
||||
return a, err
|
||||
}
|
||||
if u.StagingKey != "staging/direct/"+u.ID || u.FinalKey != "originals/"+u.AttachmentID {
|
||||
return a, errors.New("invalid direct upload object keys")
|
||||
}
|
||||
if u.State == "ready" {
|
||||
return s.GetForMember(ctx, u.AttachmentID, user)
|
||||
}
|
||||
if u.State == "rejected" && u.ErrorCode != nil {
|
||||
return a, DirectError(*u.ErrorCode)
|
||||
}
|
||||
var expired bool
|
||||
if err = tx.QueryRow(ctx, `SELECT clock_timestamp()>=$1`, u.CompleteBefore).Scan(&expired); err != nil {
|
||||
return a, err
|
||||
}
|
||||
if expired || u.State == "expired" {
|
||||
return a, DirectError("upload_expired")
|
||||
}
|
||||
grace, err := UploadCleanupGrace()
|
||||
if err != nil {
|
||||
return a, err
|
||||
}
|
||||
// ON CONFLICT handles retry after an interrupted COPY without losing tracking.
|
||||
_, err = s.db.Exec(ctx, `INSERT INTO upload_attempts(id,object_key,cleanup_after) VALUES($1,$2,clock_timestamp()+$3*interval '1 second') ON CONFLICT(id) DO UPDATE SET cleanup_after=GREATEST(upload_attempts.cleanup_after,EXCLUDED.cleanup_after) WHERE upload_attempts.object_key=EXCLUDED.object_key`, u.AttachmentID, u.FinalKey, grace.Seconds())
|
||||
if err != nil {
|
||||
return a, err
|
||||
}
|
||||
var key string
|
||||
if err = tx.QueryRow(ctx, `SELECT object_key FROM upload_attempts WHERE id=$1 AND cleanup_after>clock_timestamp() FOR UPDATE`, u.AttachmentID).Scan(&key); err != nil {
|
||||
return a, err
|
||||
}
|
||||
if key != u.FinalKey {
|
||||
return a, errors.New("direct upload reservation mismatch")
|
||||
}
|
||||
if err = validate(ctx, u); err != nil {
|
||||
var invalid UploadError
|
||||
if errors.As(err, &invalid) && (invalid.Status == 400 || invalid.Status == 413 || invalid.Status == 415) {
|
||||
_, saveErr := tx.Exec(ctx, `UPDATE direct_uploads SET state='rejected',error_code=$2,cleanup_after=clock_timestamp(),purge_after=clock_timestamp()+interval '24 hours' WHERE id=$1`, id, invalid.Code)
|
||||
if saveErr != nil {
|
||||
return a, saveErr
|
||||
}
|
||||
if saveErr = tx.Commit(ctx); saveErr != nil {
|
||||
return a, saveErr
|
||||
}
|
||||
}
|
||||
return a, err
|
||||
}
|
||||
now := time.Now().UTC().Truncate(time.Microsecond)
|
||||
a = Attachment{ID: u.AttachmentID, ChatID: u.ChatID, OwnerID: u.OwnerID, Filename: u.Filename, ContentType: u.ContentType, SizeBytes: u.SizeBytes, SHA256: u.SHA256, ObjectKey: u.FinalKey, Status: "queued", Variants: []any{}, CreatedAt: now, UpdatedAt: now}
|
||||
_, err = tx.Exec(ctx, `INSERT INTO attachments(id,chat_id,owner_id,filename,object_key,content_type,size_bytes,sha256,created_at,updated_at) VALUES($1,$2,$3,$4,$5,$6,$7,$8,$9,$10)`, a.ID, a.ChatID, a.OwnerID, a.Filename, a.ObjectKey, a.ContentType, a.SizeBytes, a.SHA256, a.CreatedAt, a.UpdatedAt)
|
||||
if err != nil {
|
||||
return Attachment{}, err
|
||||
}
|
||||
if err = enqueueJob(ctx, tx, a, a.ObjectKey); err != nil {
|
||||
return Attachment{}, err
|
||||
}
|
||||
if _, err = tx.Exec(ctx, `UPDATE direct_uploads SET state='ready',cleanup_after=clock_timestamp(),purge_after=clock_timestamp()+interval '24 hours' WHERE id=$1`, id); err != nil {
|
||||
return Attachment{}, err
|
||||
}
|
||||
if _, err = tx.Exec(ctx, `DELETE FROM upload_attempts WHERE id=$1`, a.ID); err != nil {
|
||||
return Attachment{}, err
|
||||
}
|
||||
if err = tx.Commit(ctx); err != nil {
|
||||
return Attachment{}, err
|
||||
}
|
||||
return a, nil
|
||||
}
|
||||
@@ -0,0 +1,21 @@
|
||||
package repositories
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"github.com/jackc/pgx/v5"
|
||||
"github.com/jackc/pgx/v5/pgconn"
|
||||
)
|
||||
|
||||
func NotFound(err error) bool {
|
||||
return errors.Is(err, pgx.ErrNoRows)
|
||||
}
|
||||
|
||||
func CheckViolation(err error) bool {
|
||||
var pgerr *pgconn.PgError
|
||||
return errors.As(err, &pgerr) && pgerr.Code == "23514"
|
||||
}
|
||||
|
||||
func UniqueViolation(err error) bool {
|
||||
var pgerr *pgconn.PgError
|
||||
return errors.As(err, &pgerr) && pgerr.Code == "23505"
|
||||
}
|
||||
@@ -0,0 +1,21 @@
|
||||
package repositories
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
)
|
||||
|
||||
type HealthStore struct {
|
||||
db *pgxpool.Pool
|
||||
}
|
||||
|
||||
func NewHealthStore(db *pgxpool.Pool) *HealthStore {
|
||||
return &HealthStore{db: db}
|
||||
}
|
||||
|
||||
func (s *HealthStore) Ping(ctx context.Context) error {
|
||||
ctx, cancel := boundedQuery(ctx)
|
||||
defer cancel()
|
||||
return s.db.Ping(ctx)
|
||||
}
|
||||
@@ -0,0 +1,95 @@
|
||||
package repositories
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"reflect"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/jackc/pgx/v5"
|
||||
)
|
||||
|
||||
var ErrAttachmentNotFound = errors.New("attachment not found")
|
||||
var ErrAttachmentFailed = errors.New("attachment failed")
|
||||
|
||||
var ErrIdempotencyConflict = errors.New("idempotency key belongs to a different message")
|
||||
|
||||
// CreateIdempotent commits the message and its replay record together.
|
||||
func (s *MessageStore) CreateIdempotent(ctx context.Context, chatID, senderID, text, key string, attachments ...[]string) (Message, error) {
|
||||
ids := []string{}
|
||||
if len(attachments) > 0 && attachments[0] != nil {
|
||||
ids = attachments[0]
|
||||
}
|
||||
ctx, cancel := boundedQuery(ctx)
|
||||
defer cancel()
|
||||
tx, err := s.db.Begin(ctx)
|
||||
if err != nil {
|
||||
return Message{}, err
|
||||
}
|
||||
defer tx.Rollback(context.Background())
|
||||
|
||||
// The transaction-scoped lock is shared by all API replicas. Hash collisions
|
||||
// only serialize unrelated requests; the full scope is checked by the query.
|
||||
scope := senderID + ":POST:/api/v1/chats/" + chatID + "/messages:" + key
|
||||
if _, err = tx.Exec(ctx, "SELECT pg_advisory_xact_lock(hashtextextended($1, 0))", scope); err != nil {
|
||||
return Message{}, err
|
||||
}
|
||||
// A shared row lock prevents failure publication racing with a new link.
|
||||
hasFailed := false
|
||||
for _, id := range ids {
|
||||
var status string
|
||||
err = tx.QueryRow(ctx, "SELECT status FROM attachments WHERE id=$1 AND chat_id=$2 FOR SHARE", id, chatID).Scan(&status)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return Message{}, ErrAttachmentNotFound
|
||||
}
|
||||
if err != nil {
|
||||
return Message{}, err
|
||||
}
|
||||
hasFailed = hasFailed || status == "failed"
|
||||
}
|
||||
|
||||
var message Message
|
||||
err = tx.QueryRow(ctx, `
|
||||
SELECT m.id, m.chat_id, m.sender_id, m.text, m.created_at, ARRAY(SELECT attachment_id::text FROM message_attachments ma WHERE ma.message_id=m.id ORDER BY position)
|
||||
FROM message_idempotency i JOIN messages m ON m.id = i.message_id
|
||||
WHERE i.sender_id=$1 AND i.chat_id=$2 AND i.key=$3
|
||||
AND i.expires_at > clock_timestamp()`, senderID, chatID, key).
|
||||
Scan(&message.ID, &message.ChatID, &message.SenderID, &message.Text, &message.CreatedAt, &message.AttachmentIDs)
|
||||
if err == nil {
|
||||
message.CreatedAt = message.CreatedAt.UTC()
|
||||
if message.Text != text || !reflect.DeepEqual(message.AttachmentIDs, ids) {
|
||||
return Message{}, ErrIdempotencyConflict
|
||||
}
|
||||
return message, tx.Commit(ctx)
|
||||
}
|
||||
if !errors.Is(err, pgx.ErrNoRows) {
|
||||
return Message{}, err
|
||||
}
|
||||
|
||||
if hasFailed {
|
||||
return Message{}, ErrAttachmentFailed
|
||||
}
|
||||
|
||||
// Expiry permits key reuse; it never deletes the original message.
|
||||
if _, err = tx.Exec(ctx, "DELETE FROM message_idempotency WHERE sender_id=$1 AND chat_id=$2 AND key=$3", senderID, chatID, key); err != nil {
|
||||
return Message{}, err
|
||||
}
|
||||
message = Message{
|
||||
AttachmentIDs: ids, ID: uuid.NewString(), ChatID: chatID, SenderID: senderID, Text: text,
|
||||
CreatedAt: time.Now().UTC().Truncate(time.Microsecond),
|
||||
}
|
||||
if _, err = tx.Exec(ctx, "INSERT INTO messages(id,chat_id,sender_id,text,created_at) VALUES($1,$2,$3,$4,$5)", message.ID, chatID, senderID, text, message.CreatedAt); err != nil {
|
||||
return Message{}, err
|
||||
}
|
||||
for position, id := range ids {
|
||||
if _, err = tx.Exec(ctx, "INSERT INTO message_attachments(message_id,attachment_id,position) VALUES($1,$2,$3)", message.ID, id, position); err != nil {
|
||||
return Message{}, err
|
||||
}
|
||||
}
|
||||
if _, err = tx.Exec(ctx, `INSERT INTO message_idempotency(sender_id,chat_id,key,message_id,expires_at)
|
||||
VALUES($1,$2,$3,$4,clock_timestamp() + $5 * interval '1 second')`, senderID, chatID, key, message.ID, s.idempotencyTTL.Seconds()); err != nil {
|
||||
return Message{}, err
|
||||
}
|
||||
return message, tx.Commit(ctx)
|
||||
}
|
||||
@@ -0,0 +1,142 @@
|
||||
package repositories
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"github.com/jackc/pgx/v5"
|
||||
"strings"
|
||||
"time"
|
||||
"unicode/utf8"
|
||||
)
|
||||
|
||||
type FailedJob struct {
|
||||
ID string `json:"job_id"`
|
||||
AttachmentID string `json:"attachment_id"`
|
||||
Code string `json:"error_code"`
|
||||
Attempts int `json:"attempts"`
|
||||
FailedAt time.Time `json:"failed_at"`
|
||||
}
|
||||
|
||||
func (s *AttachmentStore) FailedJobs(ctx context.Context) ([]FailedJob, error) {
|
||||
rows, err := s.db.Query(ctx, `SELECT j.id,j.attachment_id,f.error_code,f.attempts,f.failed_at FROM attachment_job_failures f JOIN attachment_jobs j ON j.id=f.job_id JOIN attachments a ON a.id=j.attachment_id WHERE a.status='failed' AND j.pipeline_version=a.pipeline_version AND j.superseded_at IS NULL ORDER BY f.failed_at DESC LIMIT 100`)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
result := []FailedJob{}
|
||||
for rows.Next() {
|
||||
var j FailedJob
|
||||
if err = rows.Scan(&j.ID, &j.AttachmentID, &j.Code, &j.Attempts, &j.FailedAt); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
result = append(result, j)
|
||||
}
|
||||
return result, rows.Err()
|
||||
}
|
||||
|
||||
func (s *AttachmentStore) ReplayJob(ctx context.Context, id, reason string, exists func(context.Context, string) error) (bool, error) {
|
||||
reason = strings.TrimSpace(reason)
|
||||
if reason == "" || utf8.RuneCountInString(reason) > 200 {
|
||||
return false, errors.New("replay reason must contain 1-200 characters")
|
||||
}
|
||||
tx, err := s.db.Begin(ctx)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
defer tx.Rollback(context.Background())
|
||||
var status, key, attachment string
|
||||
var deleted, obsolete bool
|
||||
err = tx.QueryRow(ctx, `SELECT a.status,j.staging_key,a.id,j.staging_deleted_at IS NOT NULL,j.pipeline_version<>a.pipeline_version OR j.superseded_at IS NOT NULL FROM attachment_jobs j JOIN attachments a ON a.id=j.attachment_id WHERE j.id=$1 FOR UPDATE OF j,a`, id).Scan(&status, &key, &attachment, &deleted, &obsolete)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
if obsolete || status != "failed" {
|
||||
return false, nil
|
||||
}
|
||||
if deleted {
|
||||
return false, errors.New("staging has expired; replay unavailable")
|
||||
}
|
||||
if key != "staging/jobs/"+attachment && key != "originals/"+attachment {
|
||||
return false, errors.New("invalid staging key")
|
||||
}
|
||||
if err = exists(ctx, key); err != nil {
|
||||
return false, errors.New("staging unavailable; replay not scheduled")
|
||||
}
|
||||
if _, err = tx.Exec(ctx, `INSERT INTO attachment_job_replays(job_id,reason) VALUES($1,$2)`, id, reason); err != nil {
|
||||
return false, err
|
||||
}
|
||||
if _, err = tx.Exec(ctx, `UPDATE attachment_jobs SET attempts=0,generation=generation+1,next_attempt_at=now(),lease_until=NULL,lease_token=NULL,published_at=NULL WHERE id=$1`, id); err != nil {
|
||||
return false, err
|
||||
}
|
||||
if _, err = tx.Exec(ctx, `UPDATE attachments SET status='queued',error_code=NULL,updated_at=now() WHERE id=$1`, attachment); err != nil {
|
||||
return false, err
|
||||
}
|
||||
return true, tx.Commit(ctx)
|
||||
}
|
||||
|
||||
// Replay and cleanup lock the same rows. A ready source is removed only after
|
||||
// final publication, failed sources expire after retention; queued work is safe.
|
||||
func (s *AttachmentStore) CleanupJobStaging(ctx context.Context, id string, retention time.Duration, dryRun bool, remove func(context.Context, string) error) (bool, error) {
|
||||
tx, err := s.db.Begin(ctx)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
defer tx.Rollback(context.Background())
|
||||
var key, attachment, status string
|
||||
var eligible bool
|
||||
err = tx.QueryRow(ctx, `SELECT j.staging_key,a.id,a.status,j.staging_deleted_at IS NULL AND (a.status='ready' AND jsonb_array_length(a.variants)>=3 OR a.status='failed' AND j.pipeline_version=a.pipeline_version AND COALESCE(f.failed_at,a.updated_at)<now()-$2*interval '1 second') FROM attachment_jobs j JOIN attachments a ON a.id=j.attachment_id LEFT JOIN attachment_job_failures f ON f.job_id=j.id WHERE j.id=$1 FOR UPDATE OF j,a SKIP LOCKED`, id, retention.Seconds()).Scan(&key, &attachment, &status, &eligible)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return false, nil
|
||||
}
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
if !eligible || key == "originals/"+attachment {
|
||||
return false, nil
|
||||
}
|
||||
if key != "staging/jobs/"+attachment {
|
||||
return false, errors.New("invalid staging key")
|
||||
}
|
||||
if dryRun {
|
||||
return true, nil
|
||||
}
|
||||
if err = remove(ctx, key); err != nil {
|
||||
return false, err
|
||||
}
|
||||
if _, err = tx.Exec(ctx, `UPDATE attachment_jobs SET staging_deleted_at=now() WHERE id=$1`, id); err != nil {
|
||||
return false, err
|
||||
}
|
||||
return true, tx.Commit(ctx)
|
||||
}
|
||||
|
||||
func (s *AttachmentStore) CleanupJobs(ctx context.Context, retention time.Duration, dryRun bool, remove func(context.Context, string) error) (int, error) {
|
||||
rows, err := s.db.Query(ctx, `SELECT j.id FROM attachment_jobs j JOIN attachments a ON a.id=j.attachment_id LEFT JOIN attachment_job_failures f ON f.job_id=j.id WHERE j.staging_deleted_at IS NULL AND j.staging_key='staging/jobs/'||j.attachment_id::text AND (a.status='ready' AND jsonb_array_length(a.variants)>=3 OR a.status='failed' AND j.pipeline_version=a.pipeline_version AND COALESCE(f.failed_at,a.updated_at)<now()-$1*interval '1 second') ORDER BY j.created_at LIMIT 100`, retention.Seconds())
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
ids := []string{}
|
||||
for rows.Next() {
|
||||
var id string
|
||||
if err = rows.Scan(&id); err != nil {
|
||||
rows.Close()
|
||||
return 0, err
|
||||
}
|
||||
ids = append(ids, id)
|
||||
}
|
||||
err = rows.Err()
|
||||
rows.Close()
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
count := 0
|
||||
for _, id := range ids {
|
||||
done, e := s.CleanupJobStaging(ctx, id, retention, dryRun, remove)
|
||||
if e != nil {
|
||||
return count, e
|
||||
}
|
||||
if done {
|
||||
count++
|
||||
}
|
||||
}
|
||||
return count, nil
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user