199 changed files with 16499 additions and 60 deletions
+1
View File
@@ -1,4 +1,5 @@
# Local/private state
/.memory/
.env
.env.*
!.env.example
+89
View File
@@ -4,10 +4,26 @@ CA_FILE ?=
ACTION_SCRIPT ?=
TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",)
CLEANUP_ARGS ?= --dry-run
SERVICES ?=
JOB_ID ?=
ATTACHMENT_ID ?=
REASON ?=
export JOB_ID ATTACHMENT_ID REASON
COMPOSE = docker compose --project-name highload-lab7 --env-file .env -f compose.yml
.PHONY: app-init app-build app-up app-start app-stop app-down app-restart app-logs app-status
.PHONY: app-migrate app-s3-bootstrap worker-start worker-stop queue-status attachments-cleanup
.PHONY: jobs-failed jobs-replay jobs-upgrade jobs-cleanup-staging
.PHONY: test-backend test-pipeline app-provision
.PHONY: help check validate test load test-persistence test-failover test-security
help:
@echo "check: offline structure; validate: full OpenAPI (requirements-dev.txt); test: running API; load: synthetic writes"
@echo "app-provision: prepare dependencies and OpenBao before first startup"
@echo "app-up: build and start the stand; app-status: inspect it; app-down: stop it"
check:
$(PYTHON) scripts/check_contract.py
@@ -31,3 +47,76 @@ test-failover:
test-security:
$(PYTHON) tests/security.py --base-url "$(BASE_URL)" $(TLS_ARGS)
app-init:
$(PYTHON) scripts/init_env.py
./scripts/bootstrap_tls.sh
app-provision: app-build
$(COMPOSE) up -d openbao
./scripts/provision_dependencies.sh
app-build: app-init
$(COMPOSE) build
app-up: app-init
$(COMPOSE) up -d --build --wait --wait-timeout 120
./scripts/wait_ingress.sh
app-start: app-init
$(COMPOSE) up -d --no-build --wait --wait-timeout 120
./scripts/wait_ingress.sh
app-stop:
$(COMPOSE) stop
app-down:
$(COMPOSE) down
app-restart: app-init
./scripts/restart.sh
app-logs:
$(COMPOSE) logs -f --tail=100 $(SERVICES)
app-status:
$(COMPOSE) ps --all
app-migrate: app-init
$(COMPOSE) run --rm migrate
app-s3-bootstrap: app-init
$(COMPOSE) run --rm s3-bootstrap
worker-start:
$(COMPOSE) start worker
worker-stop:
$(COMPOSE) stop worker
queue-status:
$(COMPOSE) exec -T broker rabbitmqctl -q list_queues name messages_ready messages_unacknowledged --formatter json
attachments-cleanup:
$(COMPOSE) run --rm --no-deps upload-cleanup ./messenger attachments-cleanup $(CLEANUP_ARGS)
jobs-failed:
$(COMPOSE) run --rm -T job-admin ./messenger jobs list-failed
jobs-replay:
@test -n "$$JOB_ID" && test -n "$$REASON" || { echo "Set JOB_ID=... REASON='...'"; exit 2; }
$(COMPOSE) run --rm -T job-admin ./messenger jobs replay "$$JOB_ID" "$$REASON"
jobs-upgrade:
@test -n "$$ATTACHMENT_ID" && test -n "$$REASON" || { echo "Set ATTACHMENT_ID=... REASON='...'"; exit 2; }
$(COMPOSE) run --rm -T job-admin ./messenger jobs upgrade "$$ATTACHMENT_ID" 2 "$$REASON"
jobs-cleanup-staging:
@test -n "$$JOB_ID" || { echo "Set JOB_ID=..."; exit 2; }
$(COMPOSE) run --rm -T job-admin ./messenger jobs cleanup-staging "$$JOB_ID"
test-backend: app-init
$(COMPOSE) --profile test run --rm --build backend-tests
test-pipeline: app-init
$(COMPOSE) --profile test run --rm --build backend-tests go test -race -count=1 -v ./backend/tests ./backend/workers -run 'TestPipeline|TestVersionedRendering|TestThumbnail|TestOptimized'
+50 -19
View File
@@ -1,37 +1,68 @@
# Отчёт — лабораторная 7
# Лабораторная 7
- Автор, группа: TODO
- Целевая оценка; выбранный трек 5, если нужен: TODO
- Commit/PR: TODO после публикации
- Использованные источники, библиотеки и помощь ИИ: TODO
- Автор — Гришко Дмитрий Михайлович, БВТ2303.
- Целевая оценка — 4.
- PR: https://gitea.privcloud.tech/mtusi2026/mtusi2026lab7/pulls/16.
- Источники — задание курса, документация [RabbitMQ](https://www.rabbitmq.com/docs/ssl) и [OpenBao](https://openbao.org/docs/auth/cert/). Библиотеки указаны в `src/go.mod`. Codex использовался при переносе, реализации и проверках.
## Запуск из чистого clone
## Запуск из чистого клона
TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных.
Нужны Linux, Git, Docker Compose, OpenSSL, Python 3.9+ и curl. Приложение собирается на Go 1.22.2. Версии образов указаны в [Dockerfile](src/Dockerfile) и [compose.yml](compose.yml).
```sh
git clone --branch solution/Grishko-BVT2303 --single-branch https://gitea.privcloud.tech/TAskMAster339/mtusi2026lab7-fork.git
cd mtusi2026lab7-fork
make app-provision
python3 scripts/bao_operator.py init
python3 scripts/bao_operator.py unseal
python3 scripts/configure_bao.py --operator-file secrets/bao-operator.json
make app-up
make test test-security CA_FILE=secrets/tls/lab7-ingress-ca.crt
```
`app-provision` создаёт `.env` и локальные ключи и пароли, поднимает зависимости, применяет миграции и настраивает права. `init` выполняют один раз для нового тома OpenBao. После перезапуска OpenBao достаточно повторить `unseal` и запустить приложение. Файл оператора хранится на хосте с правами 0600. `make app-up` ждёт ответа HTTPS на `/health/ready`.
`make app-stop` останавливает контейнеры с сохранением данных. Для намеренного удаления учебных томов используется `docker compose down -v`.
## Архитектура и решения
TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab6, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации.
Из ЛР 6 перенесены API мессенджера, обработка вложений и сбор телеметрии. В ЛР 7 добавлены TLS между сервисами, отдельные сервисные роли и выдача учётных данных через OpenBao.
**Вход в API.** Клиент подключается к HAProxy на `localhost:8443`. Балансировщик проверяет имя в сертификате каждой реплики API и проверяет готовность реплик тоже по mTLS. Реплики принимают клиентский сертификат с URI SAN балансировщика. Порты реплик доступны внутри Docker. Доступ к сообщениям и вложениям API проверяет по Bearer-сессии и членству в чате. Обе реплики используют Redis для сессий и общего счётчика попыток входа по IP.
**Обработка вложений.** API сохраняет исходный файл в закрытом staging-префиксе S3. Метаданные вложения и запись о задании появляются в одной транзакции PostgreSQL. Диспетчер читает запись из БД и публикует задание в RabbitMQ. Воркер забирает файл из staging, сохраняет оригинал и производные изображения в S3 и фиксирует результат в PostgreSQL. Подтверждение RabbitMQ отправляется после записи результата или решения о повторе. После обрыва соединения неподтверждённое задание может поступить повторно. Воркер сверяет версию задания и его состояние в БД перед обработкой.
**Идентичность и секреты.** В RabbitMQ сертификаты диспетчера и воркера сопоставляются с разными учётными записями через SASL EXTERNAL. Диспетчер может публиковать, воркер может читать очередь. Collector принимает только зарегистрированные сертификаты API, диспетчера и воркера. При старте каждый прикладной процесс получает в OpenBao токен на чтение своего пути KV. Пароли зависимостей записываются в `tmpfs` и используются до перезапуска процесса или отзыва в самой зависимости. Сертификаты и операторские ключи создаются локально.
`API_HTTPS_PORT`, `API_REPLICAS` и `SESSION_TTL_SECONDS` задают внешний порт, число реплик и срок сессии. Адреса TLS-сервисов заданы в Compose. Файлы подключения приложений заполняются из OpenBao при запуске.
[Матрица доверия](contracts/trust-matrix.md) · [Модель угроз](security/THREAT_MODEL.md)
## Доказательства критериев
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
| --- | --- | --- | --- |
| Публичный smoke | make test | TODO | TODO |
| Остальные пункты уровня 3 | TODO | TODO | TODO |
| Все пункты уровня 4, если заявлен | TODO | TODO | TODO |
| Выбранный трек 5, если заявлен | TODO | TODO | TODO |
TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены.
| Публичный smoke по HTTPS | `make test test-security CA_FILE=secrets/tls/lab7-ingress-ca.crt` | API прошёл smoke. Соединение без доверия к CA отклонено. | [HTTPS](evidence/grade3-https.md) |
| Argon2id | Бенчмарк `BenchmarkPasswordVerification` при 0,5 CPU | 64 МиБ, 3 прохода, p=2. В трёх сериях по три операции среднее время проверки составило 177–196 мс. | [Замер](evidence/grade3-argon2.md) |
| mTLS HAProxy → API | `python3 scripts/check_ingress_mtls.py` | Обе реплики приняли сертификат HAProxy. Отказ без сертификата, с чужим CA и с чужим SAN. | [mTLS](evidence/grade3-mtls.md) |
| TLS зависимостей | `python3 scripts/check_grade3_tls.py`, `./scripts/check_dependency_roles.sh` | Хранилища, брокер и телеметрия приняли соединения с проверкой имени сервера и отклонили чужой CA. Проверены роли PostgreSQL, Redis и RabbitMQ. | [TLS и роли](evidence/grade3-dependencies.md) |
| Доступ к секретам | `python3 scripts/check_bao_roles.py` | Каждая роль прочитала свой путь KV. Попытки прочитать чужой путь или выполнить запись отклонены. | [OpenBao](evidence/grade3-openbao.md) |
| Изоляция и бюджет воркера | `python3 scripts/check_grade3_runtime.py` | Проверены локальные публикации портов, read-only root, сброс capabilities и tmpfs. Для воркера заданы 1 CPU, 512 МиБ и 60 с на обработку по умолчанию. | [Изоляция](evidence/grade4-hardening.md) |
| Вложения, сессии и лимит входа | `check_grade3_http.py`, `check_session_expiry.py`, `check_login_rate.py`, smoke | API отклонил чужое вложение, подмену пользователя, неверный MIME и превышение размера. Токен после выхода или истечения отклонён. На 13-й попытке входа получен 429. | [ACL](evidence/grade3-http-security.md), [сессии](evidence/grade3-expiry.md), [лимит](evidence/grade3-login.md) |
| mTLS брокера и Collector | `python3 scripts/collect_grade4.py --only grade4-mtls` | Сертификат чужой роли от того же CA отклонён. RabbitMQ запретил публикацию воркером и чтение диспетчером. Телеметрия поступила. | [Роли и mTLS](evidence/grade4-mtls.md) |
| Обновление хеша и нагрузка на вход | `python3 scripts/check_grade4_login.py` | Хеш обновился в PostgreSQL после успешного входа. Из 24 параллельных запросов 8 получили 201 и 16 получили 429. Медиана 548 мс, максимум 1156 мс. | [Вход под нагрузкой](evidence/grade4-login.md) |
| Ротация и запечатывание OpenBao | `python3 scripts/check_grade4_lifecycle.py` | Пароль БД и сертификат OpenBao заменены без пересборки. Старые данные и токен после истечения отклонены. При seal запущенный API оставался готов. | [Ротация](evidence/grade4-lifecycle.md) |
| Модель угроз и аудит | `python3 scripts/collect_grade4.py --only grade4-audit` | Повторены проверки TLS и ACL. Синтетический секрет не найден в связанном логе и трейсе, метриках и логах контейнеров. | [Аудит](evidence/grade4-audit.md) |
| Сканирование | `./scripts/scan_grade4.sh` | Trivy проверил зависимости и образы. Для MinIO использован текущий собранный образ, поскольку прежний ID Docker не смог экспортировать. | [Сканирование](evidence/grade4-scan.md), [применимость находок](evidence/grade4-scan-triage.md) |
## Эксперименты и отказы
TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения.
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
| --- | --- | --- | --- |
| TODO | TODO | TODO | TODO |
| Перезапуск PostgreSQL, Redis, S3, RabbitMQ и двух API | Данные и сессия сохраняются | Повторный запрос с прежним токеном прошёл. Время старта контейнеров изменилось. | Подтверждённые данные сохранились. [Результат](evidence/grade3-resilience.md) |
| Остановка первой реплики API | Вторая реплика продолжит принимать запросы | В двух прогонах прошли 124 и 122 записи. | Запись восстановилась за 1,454 и 1,580 с. Подтверждённые сообщения сохранились. [Результат](evidence/grade3-resilience.md) |
| Остановка второй реплики API | Первая реплика обслужит бизнес-сценарий | HTTPS smoke прошёл. | Время восстановления отдельно не замерялось. [Результат](evidence/grade3-other-replica.md) |
## Ограничения и следующие шаги
## Ограничения
TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза.
Стенд работает на одном хосте. Эксперименты покрывают перезапуск хранилищ и остановку одной API-реплики. OpenBao разблокируется вручную. Истечение токена OpenBao прекращает чтение KV, а уже выданный статический пароль действует до смены в целевом сервисе.
+557
View File
@@ -0,0 +1,557 @@
x-bao-environment: &bao-environment
BAO_ADDR: https://openbao:8200
SSL_CERT_FILE: /run/tls/ca.crt
x-secret-tmpfs: &secret-tmpfs /run/runtime-secrets:rw,noexec,nosuid,size=1m,mode=1777
name: highload-lab7
services:
redis:
cpus: 0.25
mem_limit: 128m
networks: [data]
image: redis:7.4.11-bookworm
command:
[
"redis-server",
"--port", "0",
"--tls-port", "6379",
"--tls-cert-file", "/run/tls/redis.crt",
"--tls-key-file", "/run/tls/redis.key",
"--tls-ca-cert-file", "/run/tls/ca.crt",
"--tls-auth-clients", "no",
"--aclfile", "/run/secrets/redis_acl",
"--appendonly",
"yes",
"--appendfsync",
"everysec",
"--dir",
"/data",
]
volumes:
- redis_data:/data
- ./secrets/tls/lab7-ca.crt:/run/tls/ca.crt:ro
- ./secrets/tls/redis.crt:/run/tls/redis.crt:ro
- ./secrets/tls/redis.key:/run/tls/redis.key:ro
- ./secrets/redis_acl:/run/secrets/redis_acl:ro
- ./secrets/redis_password:/run/secrets/redis_password:ro
healthcheck:
test: ["CMD-SHELL", "REDISCLI_AUTH=$$(cat /run/secrets/redis_password) redis-cli --tls --cacert /run/tls/ca.crt -h redis --user messenger ping | grep -qx PONG"]
interval: 3s
timeout: 3s
retries: 20
db:
cpus: 1.0
mem_limit: 512m
networks: [data]
image: postgres:16.10-bookworm
environment:
POSTGRES_DB: ${POSTGRES_DB}
POSTGRES_USER: ${POSTGRES_USER}
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD}
volumes:
- postgres_data:/var/lib/postgresql/data
- ./secrets/tls/lab7-ca.crt:/run/tls/ca.crt:ro
- ./secrets/tls/db.crt:/run/tls/db.crt:ro
- ./secrets/tls/db.key:/run/tls/db.key:ro
- ./secrets/database_api_password:/run/secrets/database_api_password:ro
- ./secrets/database_worker_password:/run/secrets/database_worker_password:ro
- ./secrets/database_dispatcher_password:/run/secrets/database_dispatcher_password:ro
- ./secrets/database_cleanup_password:/run/secrets/database_cleanup_password:ro
- ./infra/postgres-hba.conf:/run/tls/pg_hba.conf:ro
entrypoint: ["/bin/sh", "-ec", "install -o postgres -g postgres -m 600 /run/tls/db.key /tmp/db.key; exec docker-entrypoint.sh postgres -c ssl=on -c ssl_cert_file=/run/tls/db.crt -c ssl_key_file=/tmp/db.key -c hba_file=/run/tls/pg_hba.conf"]
healthcheck:
test: ["CMD-SHELL", "pg_isready -U $$POSTGRES_USER -d $$POSTGRES_DB"]
interval: 3s
timeout: 3s
retries: 20
openbao:
image: openbao/openbao:2.4.4
user: openbao
networks: [bao, edge]
ports: ["127.0.0.1:8200:8200"]
cap_drop: [ALL]
security_opt: ["no-new-privileges:true"]
mem_limit: 256m
cpus: 0.5
entrypoint: ["bao"]
command: ["server", "-config=/openbao/config/server.hcl"]
volumes:
- openbao_data:/openbao/file
- ./infra/openbao/server.hcl:/openbao/config/server.hcl:ro
- ./secrets/tls/openbao.crt:/run/tls/bao.crt:ro
- ./secrets/tls/openbao.key:/run/tls/bao.key:ro
migrate:
cpus: 0.5
mem_limit: 128m
networks: [data]
build: ./src
environment:
DATABASE_URL_FILE: ${DATABASE_URL_FILE:-/run/secrets/database_url}
volumes:
- ./secrets/tls/lab7-ca.crt:/run/tls/ca.crt:ro
- ./secrets/database_tls_url:${DATABASE_URL_FILE:-/run/secrets/database_url}:ro
command: ["./messenger", "migrate", "up"]
depends_on:
db:
condition: service_healthy
s3:
cpus: 1.0
mem_limit: 512m
networks: [edge, data]
ports:
- "127.0.0.1:9000:9000"
build:
context: ./infra
dockerfile: minio.Dockerfile
environment:
MINIO_ROOT_USER_FILE: /run/secrets/s3_access_key_id
MINIO_ROOT_PASSWORD_FILE: /run/secrets/s3_secret_access_key
MINIO_BROWSER: "off"
MINIO_API_CORS_ALLOW_ORIGIN: ${S3_CORS_ORIGINS:-http://127.0.0.1:5173,http://localhost:5173}
volumes:
- s3_data:/data
- ./secrets/tls/s3.crt:/run/minio-certs/public.crt:ro
- ./secrets/tls/s3.key:/run/minio-certs/private.key:ro
- ./secrets/tls/lab7-ca.crt:/run/tls/ca.crt:ro
- ./secrets/s3_access_key_id:/run/secrets/s3_access_key_id:ro
- ./secrets/s3_secret_access_key:/run/secrets/s3_secret_access_key:ro
command: ["server", "/data", "--certs-dir", "/run/minio-certs", "--console-address", "127.0.0.1:9001"]
healthcheck:
test: ["CMD", "curl", "--cacert", "/run/tls/ca.crt", "-f", "https://s3:9000/minio/health/ready"]
interval: 3s
timeout: 2s
retries: 30
s3-bootstrap:
cpus: 0.5
mem_limit: 128m
networks: [data]
build: ./src
environment: &s3-environment
SSL_CERT_FILE: /run/tls/ca.crt
S3_CLEANUP_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_cleanup_secret_access_key
S3_CLEANUP_ACCESS_KEY_ID_FILE: /run/secrets/s3_cleanup_access_key_id
S3_WORKER_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_worker_secret_access_key
S3_WORKER_ACCESS_KEY_ID_FILE: /run/secrets/s3_worker_access_key_id
S3_API_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_api_secret_access_key
S3_API_ACCESS_KEY_ID_FILE: /run/secrets/s3_api_access_key_id
S3_ROOT_ACCESS_KEY_ID_FILE: /run/secrets/s3_root_access_key_id
S3_ROOT_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_root_secret_access_key
S3_ENDPOINT: https://s3:9000
S3_PUBLIC_ENDPOINT: https://localhost:9000
S3_REGION: ${S3_REGION:-us-east-1}
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
S3_ACCESS_KEY_ID_FILE: ${S3_ACCESS_KEY_ID_FILE:-/run/secrets/s3_access_key_id}
S3_SECRET_ACCESS_KEY_FILE: ${S3_SECRET_ACCESS_KEY_FILE:-/run/secrets/s3_secret_access_key}
volumes:
- ./secrets/s3_cleanup_secret_access_key:/run/secrets/s3_cleanup_secret_access_key:ro
- ./secrets/s3_cleanup_access_key_id:/run/secrets/s3_cleanup_access_key_id:ro
- ./secrets/s3_worker_secret_access_key:/run/secrets/s3_worker_secret_access_key:ro
- ./secrets/s3_worker_access_key_id:/run/secrets/s3_worker_access_key_id:ro
- ./secrets/s3_api_secret_access_key:/run/secrets/s3_api_secret_access_key:ro
- ./secrets/s3_api_access_key_id:/run/secrets/s3_api_access_key_id:ro
- ./secrets/tls/lab7-ca.crt:/run/tls/ca.crt:ro
- ./secrets/s3_access_key_id:/run/secrets/s3_root_access_key_id:ro
- ./secrets/s3_secret_access_key:/run/secrets/s3_root_secret_access_key:ro
- ./secrets/s3_api_access_key_id:${S3_ACCESS_KEY_ID_FILE:-/run/secrets/s3_access_key_id}:ro
- ./secrets/s3_api_secret_access_key:${S3_SECRET_ACCESS_KEY_FILE:-/run/secrets/s3_secret_access_key}:ro
command: ["./messenger", "s3-bootstrap"]
depends_on:
s3:
condition: service_healthy
api:
cap_drop: [ALL]
security_opt: ["no-new-privileges:true"]
read_only: true
cpus: 0.5
mem_limit: 512m
networks: [proxy, data, ingest, bao]
build: ./src
deploy:
replicas: ${API_REPLICAS:-2}
stop_grace_period: 15s
environment: &api-environment
<<: [*s3-environment, *bao-environment]
OTEL_EXPORTER_OTLP_ENDPOINT: https://otel-collector:4318
OTEL_EXPORTER_OTLP_CLIENT_CERTIFICATE: /run/otel/client.crt
OTEL_EXPORTER_OTLP_CLIENT_KEY: /run/otel/client.key
S3_ROOT_ACCESS_KEY_ID_FILE: ""
S3_ROOT_SECRET_ACCESS_KEY_FILE: ""
JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5}
JOB_TIMEOUT_SECONDS: ${JOB_TIMEOUT_SECONDS:-60}
WORKER_MAX_PIXELS: ${WORKER_MAX_PIXELS:-8000000}
UPLOAD_TMP_DIR: /tmp/uploads
UPLOAD_CLEANUP_GRACE_SECONDS: ${UPLOAD_CLEANUP_GRACE_SECONDS:-600}
# Go 1.22 sees host CPUs; keep parallelism suitable for this 0.5 CPU limit.
GOMAXPROCS: ${API_GOMAXPROCS:-1}
GOMEMLIMIT: 384MiB
BAO_ROLE: api
TLS_CA_FILE: /run/tls/ca.crt
TLS_CERT_FILE: /run/tls/api.crt
TLS_KEY_FILE: /run/tls/api.key
APP_HOST: ${APP_HOST:-0.0.0.0}
APP_PORT: ${APP_PORT:-8080}
APP_ENV: ${APP_ENV:-lab}
IDEMPOTENCY_TTL_SECONDS: ${IDEMPOTENCY_TTL_SECONDS:-86400}
DATABASE_URL_FILE: ${DATABASE_URL_FILE:-/run/secrets/database_url}
CACHE_URL_FILE: ${CACHE_URL_FILE:-/run/secrets/cache_url}
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600}
REDIS_ADDR: redis:6379
tmpfs:
- *secret-tmpfs
- /tmp/uploads:rw,noexec,nosuid,size=16m,mode=1777
volumes:
- &ca ./secrets/tls/lab7-ca.crt:/run/tls/ca.crt:ro
- ./secrets/tls/api.crt:/run/tls/api.crt:ro
- ./secrets/tls/api.key:/run/tls/api.key:ro
- ./secrets/tls/bao-api.crt:/run/bao/client.crt:ro
- ./secrets/tls/bao-api.key:/run/bao/client.key:ro
- ./secrets/tls/otel-api.crt:/run/otel/client.crt:ro
- ./secrets/tls/otel-api.key:/run/otel/client.key:ro
depends_on:
s3-bootstrap:
condition: service_completed_successfully
migrate:
condition: service_completed_successfully
db:
condition: service_healthy
redis:
condition: service_healthy
healthcheck:
disable: true
broker:
image: rabbitmq:4.2.0-management
hostname: broker
environment:
RABBITMQ_SERVER_ADDITIONAL_ERL_ARGS: "+S 2:2"
RABBITMQ_CTL_ERL_ARGS: "+S 1:1"
cpus: 0.5
mem_limit: 512m
networks: [data]
command:
- sh
- -ec
- |
export RABBITMQ_DEFAULT_USER="$$(cat /run/secrets/broker_user)"
export RABBITMQ_DEFAULT_PASS="$$(cat /run/secrets/broker_password)"
exec docker-entrypoint.sh rabbitmq-server
volumes:
- ./infra/rabbitmq/bootstrap.sh:/bootstrap.sh:ro
- rabbitmq_data:/var/lib/rabbitmq
- ./infra/rabbitmq/enabled_plugins:/etc/rabbitmq/enabled_plugins:ro
- ./secrets/tls/broker-client-ca.crt:/run/tls/broker-client-ca.crt:ro
- ./infra/rabbitmq/rabbitmq.conf:/etc/rabbitmq/rabbitmq.conf:ro
- ./secrets/tls/lab7-ca.crt:/run/tls/ca.crt:ro
- ./secrets/tls/broker.crt:/run/tls/broker.crt:ro
- ./secrets/tls/broker.key:/run/tls/broker.key:ro
- ./secrets/broker_user:/run/secrets/broker_user:ro
- ./secrets/broker_password:/run/secrets/broker_password:ro
healthcheck:
test: ["CMD-SHELL", "rabbitmq-diagnostics -q check_running"]
interval: 5s
timeout: 10s
start_period: 20s
retries: 30
worker:
cap_drop: [ALL]
security_opt: ["no-new-privileges:true"]
read_only: true
tmpfs:
- *secret-tmpfs
- /tmp:rw,noexec,nosuid,size=64m,mode=1777
build: ./src
restart: unless-stopped
cpus: 1.0
mem_limit: 512m
networks: [data, ingest, bao]
environment:
<<: *api-environment
BAO_ROLE: worker
BROKER_MTLS: "1"
CACHE_URL_FILE: ""
volumes:
- *ca
- ./secrets/tls/bao-worker.crt:/run/bao/client.crt:ro
- ./secrets/tls/bao-worker.key:/run/bao/client.key:ro
- ./secrets/tls/messenger_worker.crt:/run/broker/client.crt:ro
- ./secrets/tls/messenger_worker.key:/run/broker/client.key:ro
- ./secrets/tls/otel-worker.crt:/run/otel/client.crt:ro
- ./secrets/tls/otel-worker.key:/run/otel/client.key:ro
command: ["./messenger", "worker"]
healthcheck:
disable: true
depends_on:
migrate:
condition: service_completed_successfully
s3-bootstrap:
condition: service_completed_successfully
broker:
condition: service_healthy
dispatcher:
cap_drop: [ALL]
security_opt: ["no-new-privileges:true"]
read_only: true
tmpfs: [*secret-tmpfs]
build: ./src
restart: unless-stopped
cpus: 0.25
mem_limit: 128m
networks: [data, ingest, bao]
environment:
<<: *api-environment
BAO_ROLE: dispatcher
BROKER_MTLS: "1"
CACHE_URL_FILE: ""
S3_ACCESS_KEY_ID_FILE: ""
S3_SECRET_ACCESS_KEY_FILE: ""
volumes:
- *ca
- ./secrets/tls/bao-dispatcher.crt:/run/bao/client.crt:ro
- ./secrets/tls/bao-dispatcher.key:/run/bao/client.key:ro
- ./secrets/tls/messenger_dispatcher.crt:/run/broker/client.crt:ro
- ./secrets/tls/messenger_dispatcher.key:/run/broker/client.key:ro
- ./secrets/tls/otel-dispatcher.crt:/run/otel/client.crt:ro
- ./secrets/tls/otel-dispatcher.key:/run/otel/client.key:ro
command: ["./messenger", "dispatcher"]
healthcheck:
disable: true
depends_on:
migrate:
condition: service_completed_successfully
broker:
condition: service_healthy
upload-cleanup:
cap_drop: [ALL]
security_opt: ["no-new-privileges:true"]
read_only: true
tmpfs: [*secret-tmpfs]
restart: unless-stopped
cpus: 0.25
mem_limit: 128m
networks: [data, bao]
build: ./src
command: ["./messenger", "attachments-cleanup", "--loop"]
environment:
<<: [*s3-environment, *bao-environment]
BAO_ROLE: cleanup
S3_ROOT_ACCESS_KEY_ID_FILE: ""
S3_ROOT_SECRET_ACCESS_KEY_FILE: ""
DATABASE_URL_FILE: ${DATABASE_URL_FILE:-/run/secrets/database_url}
volumes:
- *ca
- ./secrets/tls/bao-cleanup.crt:/run/bao/client.crt:ro
- ./secrets/tls/bao-cleanup.key:/run/bao/client.key:ro
healthcheck:
disable: true
depends_on:
migrate:
condition: service_completed_successfully
s3-bootstrap:
condition: service_completed_successfully
haproxy:
cap_drop: [ALL]
security_opt: ["no-new-privileges:true"]
read_only: true
tmpfs: ["/tmp:rw,noexec,nosuid,size=16m,mode=1777"]
environment:
APP_PORT: ${APP_PORT:-8080}
API_REPLICAS: ${API_REPLICAS:-2}
cpus: 0.5
mem_limit: 128m
networks: [edge, proxy]
image: haproxy:3.2.25-alpine
ports:
- "127.0.0.1:${API_HTTPS_PORT:-8443}:8443"
volumes:
- ./infra/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro
- ./secrets/tls/lab7-ca.crt:/run/tls/ca.crt:ro
- ./secrets/tls/ingress-browser.pem:/run/tls/ingress.pem:ro
- ./secrets/tls/lb.pem:/run/tls/lb.pem:ro
depends_on:
api:
condition: service_started
job-admin:
profiles: ["operator"]
build: ./src
networks: [data]
cpus: 0.25
mem_limit: 128m
cap_drop: [ALL]
security_opt: ["no-new-privileges:true"]
environment:
<<: *s3-environment
DATABASE_URL_FILE: /run/secrets/database_url
S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id
S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key
volumes:
- *ca
- ./secrets/database_tls_url:/run/secrets/database_url:ro
- ./secrets/s3_access_key_id:/run/secrets/s3_access_key_id:ro
- ./secrets/s3_secret_access_key:/run/secrets/s3_secret_access_key:ro
backend-tests:
cpus: 2.0
mem_limit: 1g
networks: [data]
profiles: ["test"]
build:
context: ./src
target: builder
environment:
<<: *s3-environment
S3_PUBLIC_ENDPOINT: https://s3:9000
TEST_DATABASE_URL: postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@db:5432/${POSTGRES_DB}?sslmode=verify-full&sslrootcert=/run/tls/ca.crt
TEST_REDIS_ADDR: redis:6379
CACHE_URL_FILE: /run/secrets/cache_url
volumes:
- *ca
- ./secrets/cache_tls_url:/run/secrets/cache_url:ro
- ./secrets/s3_api_access_key_id:${S3_ACCESS_KEY_ID_FILE:-/run/secrets/s3_access_key_id}:ro
- ./secrets/s3_api_secret_access_key:${S3_SECRET_ACCESS_KEY_FILE:-/run/secrets/s3_secret_access_key}:ro
- ./secrets/s3_access_key_id:/run/secrets/s3_root_access_key_id:ro
- ./secrets/s3_secret_access_key:/run/secrets/s3_root_secret_access_key:ro
depends_on:
s3-bootstrap:
condition: service_completed_successfully
db:
condition: service_healthy
redis:
condition: service_healthy
command: ["go", "test", "-race", "-count=1", "-v", "./backend/..."]
otel-collector:
cap_drop: [ALL]
security_opt: ["no-new-privileges:true"]
read_only: true
tmpfs: ["/tmp:rw,noexec,nosuid,size=16m,mode=1777"]
cpus: 0.5
image: otel/opentelemetry-collector-contrib:0.120.0
networks: [telemetry, ingest]
mem_limit: 192m
command: ["--config=/etc/otelcol-contrib/config.yaml"]
volumes:
- ./secrets/tls/telemetry-export-ca.crt:/run/otel/export-ca.crt:ro
- ./secrets/tls/collector-exporter.crt:/run/otel/client.crt:ro
- ./secrets/tls/collector-exporter.key:/run/otel/client.key:ro
- *ca
- ./secrets/tls/otel-collector.crt:/run/otel/server.crt:ro
- ./secrets/tls/otel-collector.key:/run/otel/server.key:ro
- ./secrets/tls/telemetry-ingest-clients.crt:/run/otel/ingest-ca.crt:ro
- ./infra/otel/collector.yaml:/etc/otelcol-contrib/config.yaml:ro
prometheus:
cap_drop: [ALL]
security_opt: ["no-new-privileges:true"]
read_only: true
tmpfs: ["/tmp:rw,noexec,nosuid,size=16m,mode=1777"]
cpus: 0.5
mem_limit: 384m
image: prom/prometheus:v3.2.1
networks: [telemetry]
command: ["--config.file=/etc/prometheus/prometheus.yml", "--storage.tsdb.path=/prometheus", "--storage.tsdb.retention.time=7d", "--web.config.file=/etc/prometheus/web.yaml"]
volumes:
- ./secrets/tls/telemetry-read-ca.crt:/run/otel/read-ca.crt:ro
- ./secrets/tls/prometheus-scraper.crt:/run/otel/client.crt:ro
- ./secrets/tls/prometheus-scraper.key:/run/otel/client.key:ro
- *ca
- ./secrets/tls/prometheus.crt:/run/otel/server.crt:ro
- ./secrets/tls/prometheus.key:/run/otel/server.key:ro
- ./infra/otel/prometheus-web.yaml:/etc/prometheus/web.yaml:ro
- ./infra/otel/prometheus.yaml:/etc/prometheus/prometheus.yml:ro
- ./infra/otel/alerts.yaml:/etc/prometheus/alerts.yaml:ro
- prometheus_data:/prometheus
loki:
cap_drop: [ALL]
security_opt: ["no-new-privileges:true"]
read_only: true
tmpfs: ["/tmp:rw,noexec,nosuid,size=16m,mode=1777"]
cpus: 0.5
mem_limit: 384m
image: grafana/loki:3.4.2
networks: [telemetry]
command: ["-config.file=/etc/loki/lab.yaml"]
volumes:
- ./secrets/tls/loki-client-ca.crt:/run/otel/loki-client-ca.crt:ro
- *ca
- ./secrets/tls/loki.crt:/run/otel/server.crt:ro
- ./secrets/tls/loki.key:/run/otel/server.key:ro
- ./infra/otel/loki.yaml:/etc/loki/lab.yaml:ro
- loki_data:/loki
tempo:
cap_drop: [ALL]
security_opt: ["no-new-privileges:true"]
read_only: true
tmpfs: ["/tmp:rw,noexec,nosuid,size=16m,mode=1777"]
cpus: 0.5
mem_limit: 384m
image: grafana/tempo:2.7.1
networks: [telemetry]
command: ["-config.file=/etc/tempo/lab.yaml"]
volumes:
- ./secrets/tls/telemetry-read-ca.crt:/run/otel/read-ca.crt:ro
- ./secrets/tls/telemetry-export-ca.crt:/run/otel/export-ca.crt:ro
- *ca
- ./secrets/tls/tempo.crt:/run/otel/server.crt:ro
- ./secrets/tls/tempo.key:/run/otel/server.key:ro
- ./infra/otel/tempo.yaml:/etc/tempo/lab.yaml:ro
- tempo_data:/var/tempo
grafana:
cap_drop: [ALL]
security_opt: ["no-new-privileges:true"]
read_only: true
tmpfs: ["/tmp:rw,noexec,nosuid,size=16m,mode=1777"]
entrypoint:
- /bin/sh
- -ec
- |
export GRAFANA_READER_CERT="$$(cat /run/otel/client.crt)"
export GRAFANA_READER_KEY="$$(cat /run/otel/client.key)"
exec /run.sh
cpus: 0.5
mem_limit: 384m
image: grafana/grafana:11.5.2
networks: [telemetry, edge]
environment:
SSL_CERT_FILE: /run/tls/ca.crt
GF_AUTH_ANONYMOUS_ENABLED: "false"
GF_SERVER_PROTOCOL: https
GF_SERVER_CERT_FILE: /run/otel/server.crt
GF_SERVER_CERT_KEY: /run/otel/server.key
GF_SECURITY_ADMIN_PASSWORD__FILE: /run/secrets/grafana_admin_password
ports:
- "127.0.0.1:${GRAFANA_HTTP_PORT:-3000}:3000"
volumes:
- ./secrets/tls/grafana-reader.crt:/run/otel/client.crt:ro
- ./secrets/tls/grafana-reader.key:/run/otel/client.key:ro
- *ca
- ./secrets/tls/grafana.crt:/run/otel/server.crt:ro
- ./secrets/tls/grafana.key:/run/otel/server.key:ro
- ./infra/otel/grafana/provisioning:/etc/grafana/provisioning:ro
- ./infra/otel/grafana/dashboards:/var/lib/grafana/dashboards:ro
- ./secrets/grafana_admin_password:/run/secrets/grafana_admin_password:ro
- grafana_data:/var/lib/grafana
volumes:
openbao_data:
prometheus_data:
loki_data:
tempo_data:
grafana_data:
rabbitmq_data:
postgres_data:
redis_data:
s3_data:
networks:
bao:
internal: true
edge:
proxy:
internal: true
data:
internal: true
telemetry:
internal: true
ingest:
internal: true
+19 -32
View File
@@ -1,37 +1,24 @@
# Матрица доверия — заполнить в решении lab7
# Матрица доверия
Это обязательная матрица покрытия. Для каждой строки укажите реальный порт, endpoint, CA, service identity, проверку SAN/ACL, срок жизни credentials и команду положительного/отрицательного теста. TLS шифрует и подтверждает сервер; mTLS также проверяет клиента. При доверенном CA всё равно нужно ограничивать разрешённую identity. Plaintext hop допустим только внутри закрытого локального transport/proxy boundary, без опубликованного обхода; границу и rationale нужно доказать.
Внешний сертификат HAProxy использует ECDSA P-256 и отдельный CA `lab7-ingress-ca`. Остальные сетевые серверы предъявляют leaf со своим DNS SAN, подписанный локальным `lab7-ca`. Leaf действуют 30 дней, CA — 365 дней. Для доступа с хоста сертификаты HAProxy, MinIO, Grafana и OpenBao также содержат `localhost` и `127.0.0.1`. Пароли PostgreSQL, Redis, S3 и RabbitMQ статические, до смены оператором. Параметры действующего стенда приведены ниже.
| Связь | Минимум на 3 | Дополнительно на 4 | Доказательство / конфиг |
| --- | --- | --- | --- |
| Клиент → ingress/LB | TLS, CA + hostname, Bearer | Проверка expiry/revocation сессии и общего login rate limit | TODO |
| LB → API (все реплики) | mTLS + allowlist LB identity | Ротация leaf без обхода проверки | TODO |
| API/worker → PostgreSQL | TLS verify-full + раздельные ограниченные DB roles | Ротация credentials; mTLS — допустимое усиление | TODO |
| API/worker → S3 | TLS + разные scoped credentials/prefix rights | Проверка denied prefix и ротация | TODO |
| API/worker → broker | TLS + разные producer/consumer identities и ACL | mTLS + ограничение допустимых SAN/ролей | TODO |
| API/worker → Collector | TLS; сеть/аутентификация запрещает произвольный ingest | mTLS с раздельными identities | TODO |
| Collector → telemetry backends | TLS + scoped backend credentials или mTLS | Ротация и проверка least privilege | TODO |
| Grafana → data sources | TLS с проверкой сервера, ограниченная identity | Отрицательные проверки доступа | TODO |
| API/worker/agent → OpenBao | TLS + сервисный auth + scoped policy | Renewal/expiry/revocation | TODO |
| Оператор → OpenBao/Grafana/S3 admin | TLS + authentication, доступ локальный/через защищённый канал | Разделение operator/application roles | TODO |
| API → Redis/Valkey, если используется | TLS + ACL/пароль с минимальными правами | Ротация; mTLS по возможности | TODO |
| Replication/кластерные каналы, если добавлены | TLS/mTLS либо подтверждённый шифрованный transport | Проверка заявленной модели отказа | TODO |
| Связь | TLS, identity и права | Проверка |
| --- | --- | --- |
| Клиент → HAProxy | HTTPS `localhost:8443` с проверкой CA и SAN. Bearer-сессия по умолчанию 3600 с. | `make test test-security CA_FILE=secrets/tls/lab7-ingress-ca.crt`, `scripts/check_grade3_http.py`, `scripts/check_login_rate.py` и `scripts/check_session_expiry.py`. |
| HAProxy → API | mTLS `api:8080`. HAProxy проверяет CA и `api`. API проверяет clientAuth и URI SAN `urn:lab7:load-balancer`. Порт API не опубликован. | `python3 scripts/check_ingress_mtls.py` проверяет разрешённый LB и три отказа на каждой реплике. |
| API, worker, dispatcher, cleanup и migrate → PostgreSQL | `db:5432`, `sslmode=verify-full`, hostssl + SCRAM. Владелец запускает миграции, остальные используют отдельные login-роли и права на таблицы и последовательности из `infra/postgres-roles.sql`. | TLS probe с правильным и чужим CA, HTTPS smoke проверяет работу ролей API и worker. |
| API, worker и cleanup → S3 | `https://s3:9000`, приватный bucket. У bootstrap root. API принимает staging и читает опубликованные файлы, worker читает staging и публикует оригиналы и варианты, cleanup удаляет по разрешённым префиксам. Права находятся в `s3_admin.go`. Direct upload использует подписанную форму через локальный `https://localhost:9000`. | TLS probe и smoke с побайтной сверкой оригинала и thumbnail. |
| Dispatcher и worker → RabbitMQ | AMQPS `broker:5671`. Сервисные сертификаты от broker-client CA, RabbitMQ сопоставляет CN с учётной записью через SASL EXTERNAL. Dispatcher публикует через default exchange, worker читает `attachments.v1`. Обе роли могут объявлять только эту durable очередь. API пишет намерение в БД. | TLS probe, smoke queued → ready и `scripts/check_grade4_broker.py`: без cert и с чужим CA отказ TLS, незарегистрированная identity своего CA отклоняется при входе, обратные операции ролей запрещены. `infra/rabbitmq/bootstrap.sh` задаёт ACL. |
| API, worker и dispatcher → OTel Collector | HTTPS `otel-collector:4318`. Collector закрепляет три конкретных leaf API, worker и dispatcher. Остальные leaf того же CA отклоняются. | TLS probe и проверка доставки свежих сигналов. |
| Collector → Loki/Tempo, Prometheus → Collector | mTLS `loki:3100/otlp`, `tempo:4318`, scrape `otel-collector:8889`. Клиенты имеют сертификаты отдельного export CA. | `scripts/check_grade3_tls.py` и `scripts/check_grade3_signals.py`. |
| Grafana → источники | mTLS `prometheus:9090`, `loki:3100`, `tempo:3200`. Grafana имеет ключ reader CA. Loki принимает reader и exporter CA на общем HTTP listener, поэтому граница контроля там ограничена этими двумя группами. | TLS probe с reader cert и запросы к каждому хранилищу в `scripts/check_grade3_signals.py`. |
| API, worker, dispatcher, cleanup → OpenBao | `https://openbao:8200`. Cert auth закрепляет отдельный leaf каждой роли. Токен до 900 с читает только `messenger/data/<role>`. Сервисы получают credentials при запуске и держат их на tmpfs. | `python3 scripts/check_bao_roles.py`: свой путь разрешён, чужое чтение, чужой вход и запись запрещены. |
| Оператор → OpenBao, Grafana и S3 | Локальные HTTPS-порты 8200, 3000, 9000. OpenBao — операторский токен, Grafana — login, MinIO — отдельные root credentials для bootstrap. | Проверка публикаций контейнеров и TLS. |
| API → Redis | `rediss://redis:6379`, имя сервера проверяется. Отдельный пользователь messenger ограничен ключами `messenger:*` и операциями сессий и счётчиков. Plaintext listener отключён. | TLS probe, smoke, проверка expiry и общего ограничения входа. |
| Replication/кластерные каналы | Не применимо. Реплики API не обмениваются данными напрямую, хранилища одиночные. | Проверка восстановления после остановки API. |
Нет выбранного компонента — отметьте N/A с причиной, а не оставляйте пустую строку. Внутренний публичный health listener, если нужен, минимален и изолирован; через него нельзя обращаться к бизнес-API без mTLS. Отдельно задокументируйте scrape/health/bootstrap каналы: они тоже не должны создавать обход.
## Health, scrape и bootstrap
## Обязательные отрицательные проверки
HAProxy проверяет `GET /health/ready` каждой реплики через её mTLS listener. Скрипт `scripts/wait_ingress.sh` проверяет готовность через HTTPS с доверенным CA. Служебные проверки Postgres и RabbitMQ запускаются внутри контейнеров, Redis проверяется локальным CLI, MinIO — HTTPS-запросом с проверкой CA. Redis healthcheck также использует TLS и ACL. Связь Prometheus с Collector учитывается отдельно в строке про scrape.
1. HTTPS с правильным CA/hostname проходит; с пустым/чужим CA и неправильным hostname — ошибка верификации, не «сервер недоступен».
2. LB→API: без client cert, с cert чужого CA, с запрещённой identity от доверенного CA — отказ; с разрешённой LB identity — успех.
3. На 4 повторите набор для broker и Collector; ограничение на network policy не заменяет проверку identity.
4. Пользователь вне чата не читает metadata/content и не прикрепляет чужой файл; поддельный X-User-Id не меняет Bearer identity.
5. OpenBao app policy читает только свой secret/path; запись policy и чтение чужого path запрещены. Worker не получает admin credentials API/БД/S3.
6. На 4 отзовите secret/cert и дождитесь контролируемого эффекта с учётом действующих соединений, TLS resumption и cache/TTL. Покажите новое подключение, иначе тест отзыва может ничего не проверить.
## Bootstrap и эксплуатация
В репозитории только открытые конфиги, policy templates и scripts. CA private key, leaf keys, root token, unseal shares и SecretID не коммитятся, не запекаются в образ и не печатаются в отчёте. Для локального стенда оператор создаёт их вне Git и передаёт через files/secrets с ограниченными правами. Опишите, кто имеет доступ к CA/unseal, что нужно после рестарта OpenBao и как восстановить работу.
OpenBao не использует dev-mode. Выберите persistent storage и auth метод (например AppRole с короткоживущим SecretID/response wrapping, cert auth или иная обоснованная identity). Не подменяйте root token постоянной переменной BAO_TOKEN в приложении. Рабочие tokens имеют policy, TTL и renewal/re-auth сценарий. Статические KV secrets на 3 допустимы при реальном чтении через сервисную identity; динамические DB credentials — хорошее расширение, но не обязательное название технологии.
Разрешённый компромисс учебного стенда: ручной operator init/unseal, если он безопасно документирован и не нужен при каждом обычном рестарте API. Автоматическое хранение всех unseal shares рядом с OpenBao без защиты не считается решённым bootstrap.
Bootstrap миграций и S3 выполняется одноразовыми контейнерами. Оператор запускает их с учётными данными владельцев и проверкой TLS. Прикладные контейнеры монтируют свои ключи cert auth и получают ограниченные credentials из OpenBao. Закрытые ключи, unseal shares и административные токены остаются у оператора вне Git и образов. `scripts/check_grade4_lifecycle.py` показывает замену leaf OpenBao и пароля PostgreSQL на новых соединениях. При рестарте сервис заново аутентифицируется по зарегистрированному leaf. Оператор выпускает новый leaf до истечения срока и обновляет доверие в выбранном получателе.
+83
View File
@@ -0,0 +1,83 @@
# Стоимость проверки хеша Argon2id на лимитах API
Предпосылки: подготовленный, разблокированный учебный стенд; Docker Compose, Python и локальный CA.
`docker compose --profile test build backend-tests`
Код завершения: 0.
```text
Image highload-lab7-backend-tests Building
#1 [internal] load local bake definitions
#1 reading from stdin 1.66kB done
#1 DONE 0.0s
#2 [internal] load build definition from Dockerfile
#2 transferring dockerfile: 816B done
#2 DONE 0.0s
#3 [internal] load metadata for docker.io/library/golang:1.22.2-bookworm
#3 DONE 1.1s
#4 [internal] load .dockerignore
#4 transferring context: 327B done
#4 DONE 0.0s
#5 [builder 1/7] FROM docker.io/library/golang:1.22.2-bookworm@sha256:d0902bacefdde1cf45528c098d14e55d78c107def8a22d148eabd71582d7a99f
#5 resolve docker.io/library/golang:1.22.2-bookworm@sha256:d0902bacefdde1cf45528c098d14e55d78c107def8a22d148eabd71582d7a99f 0.0s done
#5 DONE 0.0s
#6 [internal] load build context
#6 transferring context: 13.70kB done
#6 DONE 0.0s
#7 [builder 4/7] COPY go.mod go.sum ./
#7 CACHED
#8 [builder 2/7] RUN apt-get update && apt-get install -y --no-install-recommends webp && rm -rf /var/lib/apt/lists/*
#8 CACHED
#9 [builder 5/7] RUN go mod download
#9 CACHED
#10 [builder 6/7] COPY backend ./backend
#10 CACHED
#11 [builder 3/7] WORKDIR /src
#11 CACHED
#12 [builder 7/7] RUN CGO_ENABLED=0 GOOS=linux go build -trimpath -ldflags="-s -w" -o /out/messenger ./backend
#12 CACHED
#13 exporting to image
#13 exporting layers
#13 exporting layers 6.7s done
#13 exporting manifest sha256:cc55619214f59b9b27ad25a2e23d1abce9f64566439be3b766526869522398f8 done
#13 exporting config sha256:60158d0274ef38fabce1b1b41aef1dce15861f92bfe46cdd75183cc143df20f5 done
#13 exporting attestation manifest sha256:facefdb8b90c9ec469b34788e90e8b0d663bf8c722e0e92d10cbc231bc2e6bc2 done
#13 exporting manifest list sha256:a0ef01b3efc503922fd45a76b39ec30f5d34f2085c9a228eeb3277e39df87eab done
#13 naming to docker.io/library/highload-lab7-backend-tests:latest done
#13 unpacking to docker.io/library/highload-lab7-backend-tests:latest
#13 unpacking to docker.io/library/highload-lab7-backend-tests:latest 1.2s done
#13 DONE 7.9s
#14 resolving provenance for metadata file
#14 DONE 0.0s
Image highload-lab7-backend-tests Built
```
`docker run --rm --network none --cpus 0.5 --memory 512m -e GOMAXPROCS=1 highload-lab7-backend-tests go test ./backend/services -run '^$' -bench '^BenchmarkPasswordVerification$' -benchtime=3x -count=3`
Код завершения: 0.
```text
goos: linux
goarch: amd64
pkg: messenger/src/backend/services
cpu: AMD Ryzen 9 PRO 7940HS w/ Radeon 780M Graphics
BenchmarkPasswordVerification 3 194744721 ns/op 67115832 B/op 74 allocs/op
BenchmarkPasswordVerification 3 176595711 ns/op 67115832 B/op 74 allocs/op
BenchmarkPasswordVerification 3 196042348 ns/op 67115832 B/op 74 allocs/op
PASS
ok messenger/src/backend/services 3.606s
```
+35
View File
@@ -0,0 +1,35 @@
# TLS хранилищ, брокера и телеметрии
Предпосылки: подготовленный, разблокированный учебный стенд; Docker Compose, Python и локальный CA.
`python3 scripts/check_grade3_tls.py`
Код завершения: 0.
```text
PASS s3:9000: verified service SAN; foreign CA rejected
PASS broker:5671: verified service SAN; foreign CA rejected; foreign client identity rejected
PASS redis:6379: verified service SAN; foreign CA rejected
PASS db:5432: verified service SAN; foreign CA rejected
PASS otel-collector:4318: verified service SAN; foreign CA rejected; foreign client identity rejected
PASS otel-collector:8889: verified service SAN; foreign CA rejected; foreign client identity rejected
PASS prometheus:9090: verified service SAN; foreign CA rejected; foreign client identity rejected
PASS loki:3100: verified service SAN; foreign CA rejected; foreign client identity rejected
PASS loki:3100: verified service SAN; foreign CA rejected; foreign client identity rejected
PASS tempo:4318: verified service SAN; foreign CA rejected; foreign client identity rejected
PASS tempo:3200: verified service SAN; foreign CA rejected; foreign client identity rejected
PASS grafana:3000: verified service SAN; foreign CA rejected
PASS openbao:8200: verified service SAN; foreign CA rejected
```
`./scripts/check_dependency_roles.sh`
Код завершения: 0.
```text
PASS PostgreSQL: worker can update attachments; user table and schema creation denied
PASS RabbitMQ: producer can publish and consumer can read attachments.v1
PASS Redis: application role cannot read server configuration
```
Итог: все команды завершились успешно.
+67
View File
@@ -0,0 +1,67 @@
# Истечение сессии на стенде с коротким TTL
Предпосылки: подготовленный, разблокированный учебный стенд; Docker Compose, Python и локальный CA.
`sleep 60`
Код завершения: 0.
```text
```
`env SESSION_TTL_SECONDS=5 docker compose up -d --no-deps api`
Код завершения: 0.
```text
Container highload-lab7-api-1 Recreate
Container highload-lab7-api-2 Recreate
Container highload-lab7-api-1 Recreated
Container highload-lab7-api-2 Recreated
Container highload-lab7-api-1 Starting
Container highload-lab7-api-1 Started
Container highload-lab7-api-2 Starting
Container highload-lab7-api-2 Started
```
`./scripts/wait_ingress.sh`
Код завершения: 0.
```text
```
`python3 scripts/check_session_expiry.py --ca-file secrets/tls/lab7-ca.crt`
Код завершения: 0.
```text
PASS: valid session accepted before expiry and rejected after expiry
```
`docker compose up -d --no-deps api`
Код завершения: 0.
```text
Container highload-lab7-api-2 Recreate
Container highload-lab7-api-1 Recreate
Container highload-lab7-api-2 Recreated
Container highload-lab7-api-1 Recreated
Container highload-lab7-api-1 Starting
Container highload-lab7-api-1 Started
Container highload-lab7-api-2 Starting
Container highload-lab7-api-2 Started
```
`./scripts/wait_ingress.sh`
Код завершения: 0.
```text
```
Итог: все команды завершились успешно.
+13
View File
@@ -0,0 +1,13 @@
# Лимиты, MIME, чужие вложения и неверный TLS hostname
Предпосылки: подготовленный, разблокированный учебный стенд; Docker Compose, Python и локальный CA.
`python3 scripts/check_grade3_http.py --ca-file secrets/tls/lab7-ca.crt`
Код завершения: 0.
```text
PASS: upload size/MIME, forged identity, cross-chat attachment, wrong TLS hostname
```
Итог: все команды завершились успешно.
+25
View File
@@ -0,0 +1,25 @@
# Бизнес-API, проверка CA, logout и вложения по HTTPS
Предпосылки: подготовленный, разблокированный учебный стенд; Docker Compose, Python и локальный CA.
`make check`
Код завершения: 0.
```text
python3 scripts/check_contract.py
PASS: offline skeleton structure, local refs and Python syntax; application NOT tested.
```
`make test test-security CA_FILE=secrets/tls/lab7-ingress-ca.crt`
Код завершения: 0.
```text
python3 tests/smoke.py --base-url "https://localhost:8443" --ca-file "secrets/tls/lab7-ingress-ca.crt"
PASS: lab7 public API scenario; infrastructure and grading evidence remain separate.
python3 tests/security.py --base-url "https://localhost:8443" --ca-file "secrets/tls/lab7-ingress-ca.crt"
PASS: trusted HTTPS works; untrusted CA rejected. This does NOT verify internal mTLS, OpenBao or hardening.
```
Итог: все команды завершились успешно.
+21
View File
@@ -0,0 +1,21 @@
# Общий лимит входа по адресу клиента
Предпосылки: подготовленный, разблокированный учебный стенд; Docker Compose, Python и локальный CA.
`sleep 60`
Код завершения: 0.
```text
```
`python3 scripts/check_login_rate.py`
Код завершения: 0.
```text
PASS: request 13 returned 429 with Retry-After across 2 API replicas
```
Итог: все команды завершились успешно.
+14
View File
@@ -0,0 +1,14 @@
# Допуск только роли LB на каждой реплике
Предпосылки: подготовленный, разблокированный учебный стенд; Docker Compose, Python и локальный CA.
`python3 scripts/check_ingress_mtls.py`
Код завершения: 0.
```text
PASS b99988e01dc1: LB permitted; absent, foreign CA and disallowed SAN rejected
PASS 10ef2f17b12c: LB permitted; absent, foreign CA and disallowed SAN rejected
```
Итог: все команды завершились успешно.
+16
View File
@@ -0,0 +1,16 @@
# Сервисная аутентификация и scoped policy OpenBao
Предпосылки: подготовленный, разблокированный учебный стенд; Docker Compose, Python и локальный CA.
`python3 scripts/check_bao_roles.py`
Код завершения: 0.
```text
PASS api: scoped read, cross-role login/read and write denied, TTL <= 900 s
PASS worker: scoped read, cross-role login/read and write denied, TTL <= 900 s
PASS dispatcher: scoped read, cross-role login/read and write denied, TTL <= 900 s
PASS cleanup: scoped read, cross-role login/read and write denied, TTL <= 900 s
```
Итог: все команды завершились успешно.
+47
View File
@@ -0,0 +1,47 @@
# Работа HTTPS API при остановленной второй реплике
Предпосылки: подготовленный, разблокированный учебный стенд; Docker Compose, Python и локальный CA.
`sleep 60`
Код завершения: 0.
```text
```
`./scripts/check_other_replica.sh`
Код завершения: 0.
```text
Stopped API instance 3b3f131fd602
python3 tests/smoke.py --base-url "https://localhost:8443" --ca-file "secrets/tls/lab7-ca.crt"
PASS: lab7 public API scenario; infrastructure and grading evidence remain separate.
Container highload-lab7-redis-1 Waiting
Container highload-lab7-db-1 Waiting
Container highload-lab7-s3-1 Waiting
Container highload-lab7-db-1 Healthy
Container highload-lab7-s3-1 Healthy
Container highload-lab7-redis-1 Healthy
Container highload-lab7-migrate-1 Starting
Container highload-lab7-s3-bootstrap-1 Starting
Container highload-lab7-migrate-1 Started
Container highload-lab7-migrate-1 Waiting
Container highload-lab7-s3-bootstrap-1 Started
Container highload-lab7-s3-bootstrap-1 Waiting
Container highload-lab7-migrate-1 Exited
Container highload-lab7-s3-bootstrap-1 Exited
Container highload-lab7-api-2 Starting
Container highload-lab7-api-2 Started
```
`./scripts/wait_ingress.sh`
Код завершения: 0.
```text
```
Итог: все команды завершились успешно.
+139
View File
@@ -0,0 +1,139 @@
# Сохранность после перезапуска хранилищ и отказ API
Предпосылки: подготовленный, разблокированный учебный стенд; Docker Compose, Python и локальный CA.
`sleep 60`
Код завершения: 0.
```text
```
`make test-persistence ACTION_SCRIPT=scripts/restart.sh CA_FILE=secrets/tls/lab7-ca.crt`
Код завершения: 0.
```text
python3 tests/resilience.py --mode persistence --action-script "scripts/restart.sh" --base-url "https://localhost:8443" --ca-file "secrets/tls/lab7-ca.crt"
Executing explicit persistence hook; hook output suppressed to avoid leaking secrets.
PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence.
```
`make test-failover ACTION_SCRIPT=scripts/stop-one.sh CA_FILE=secrets/tls/lab7-ca.crt`
Код завершения: 0.
```text
python3 tests/resilience.py --mode failover --action-script "scripts/stop-one.sh" --base-url "https://localhost:8443" --ca-file "secrets/tls/lab7-ca.crt"
Executing explicit failover hook; hook output suppressed to avoid leaking secrets.
{"observed_instances_before": 2, "successful_writes": 124, "failed_attempts": 1, "first_recovery_seconds": 1.454}
PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.
```
`docker compose start api`
Код завершения: 0.
```text
Container highload-lab7-db-1 Waiting
Container highload-lab7-s3-1 Waiting
Container highload-lab7-redis-1 Waiting
Container highload-lab7-s3-1 Healthy
Container highload-lab7-redis-1 Healthy
Container highload-lab7-db-1 Healthy
Container highload-lab7-s3-bootstrap-1 Starting
Container highload-lab7-migrate-1 Starting
Container highload-lab7-s3-bootstrap-1 Started
Container highload-lab7-s3-bootstrap-1 Waiting
Container highload-lab7-migrate-1 Started
Container highload-lab7-migrate-1 Waiting
Container highload-lab7-migrate-1 Exited
Container highload-lab7-s3-bootstrap-1 Exited
Container highload-lab7-api-1 Starting
Container highload-lab7-api-1 Started
```
`./scripts/wait_ingress.sh`
Код завершения: 0.
```text
```
`sleep 60`
Код завершения: 0.
```text
```
`make test-failover ACTION_SCRIPT=scripts/stop-one.sh CA_FILE=secrets/tls/lab7-ca.crt`
Код завершения: 0.
```text
python3 tests/resilience.py --mode failover --action-script "scripts/stop-one.sh" --base-url "https://localhost:8443" --ca-file "secrets/tls/lab7-ca.crt"
Executing explicit failover hook; hook output suppressed to avoid leaking secrets.
{"observed_instances_before": 2, "successful_writes": 122, "failed_attempts": 2, "first_recovery_seconds": 1.58}
PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.
```
`docker compose start api`
Код завершения: 0.
```text
Container highload-lab7-redis-1 Waiting
Container highload-lab7-s3-1 Waiting
Container highload-lab7-db-1 Waiting
Container highload-lab7-s3-1 Healthy
Container highload-lab7-s3-bootstrap-1 Starting
Container highload-lab7-redis-1 Healthy
Container highload-lab7-db-1 Healthy
Container highload-lab7-migrate-1 Starting
Container highload-lab7-s3-bootstrap-1 Started
Container highload-lab7-s3-bootstrap-1 Waiting
Container highload-lab7-migrate-1 Started
Container highload-lab7-migrate-1 Waiting
Container highload-lab7-migrate-1 Exited
Container highload-lab7-s3-bootstrap-1 Exited
Container highload-lab7-api-1 Starting
Container highload-lab7-api-1 Started
```
`./scripts/wait_ingress.sh`
Код завершения: 0.
```text
```
Состояния контейнеров (restart):
```text
/highload-lab7-api-1 2026-11-14T17:24:55.651185572Z
/highload-lab7-api-2 2026-11-14T17:24:55.819757058Z
/highload-lab7-broker-1 2026-11-14T17:23:53.398757712Z
/highload-lab7-db-1 2026-11-14T17:23:50.651938319Z
/highload-lab7-redis-1 2026-11-14T17:23:50.615434358Z
/highload-lab7-s3-1 2026-11-14T17:23:50.777141988Z
/highload-lab7-api-1 2026-11-14T17:26:42.238931046Z
/highload-lab7-api-2 2026-11-14T17:26:42.256578225Z
/highload-lab7-broker-1 2026-11-14T17:26:34.665663438Z
/highload-lab7-db-1 2026-11-14T17:26:33.677826831Z
/highload-lab7-redis-1 2026-11-14T17:26:33.695777808Z
/highload-lab7-s3-1 2026-11-14T17:26:33.780920926Z
```
Состояния контейнеров (failover):
```text
/highload-lab7-api-1 exited
/highload-lab7-api-1 exited
```
Итог: все команды завершились успешно.
+13
View File
@@ -0,0 +1,13 @@
# Изоляция, порты, лимиты и локальное размещение секретов
Предпосылки: подготовленный, разблокированный учебный стенд; Docker Compose, Python и локальный CA.
`python3 scripts/check_grade3_runtime.py`
Код завершения: 0.
```text
PASS: local-only ports, runtime secrets on tmpfs, non-root app roles with limits and dropped capabilities
```
Итог: все команды завершились успешно.
+13
View File
@@ -0,0 +1,13 @@
# Доставка логов, метрик и трейсов по защищённым связям
Предпосылки: подготовленный, разблокированный учебный стенд; Docker Compose, Python и локальный CA.
`python3 scripts/check_grade3_signals.py`
Код завершения: 0.
```text
PASS: Prometheus scrape up; fresh messenger API log and trace found through mTLS
```
Итог: все команды завершились успешно.
+83
View File
@@ -0,0 +1,83 @@
# Сетевой аудит, ACL и отсутствие контрольного секрета в телеметрии
Предпосылки: подготовленный, разблокированный учебный стенд; Docker Compose, Python и локальный CA.
`python3 scripts/check_grade3_tls.py`
Код завершения: 0.
```text
PASS s3:9000: verified service SAN; foreign CA rejected
PASS broker:5671: verified service SAN; foreign CA rejected; foreign client identity rejected
PASS redis:6379: verified service SAN; foreign CA rejected
PASS db:5432: verified service SAN; foreign CA rejected
PASS Collector: missing client certificate rejected
PASS otel-collector:4318: verified service SAN; foreign CA rejected; foreign client identity rejected
PASS otel-collector:8889: verified service SAN; foreign CA rejected; foreign client identity rejected
PASS prometheus:9090: verified service SAN; foreign CA rejected; foreign client identity rejected
PASS loki:3100: verified service SAN; foreign CA rejected; foreign client identity rejected
PASS loki:3100: verified service SAN; foreign CA rejected; foreign client identity rejected
PASS tempo:4318: verified service SAN; foreign CA rejected; foreign client identity rejected
PASS tempo:3200: verified service SAN; foreign CA rejected; foreign client identity rejected
PASS grafana:3000: verified service SAN; foreign CA rejected
PASS openbao:8200: verified service SAN; foreign CA rejected
```
`python3 scripts/check_grade4_broker.py`
Код завершения: 0.
```text
PASS: no certificate
PASS: wrong-role
PASS: messenger_unregistered
PASS: messenger_worker
PASS: messenger_dispatcher
```
`./scripts/check_dependency_roles.sh`
Код завершения: 0.
```text
PASS PostgreSQL: worker can update attachments; user table and schema creation denied
PASS RabbitMQ: producer can publish and consumer can read attachments.v1
PASS Redis: application role cannot read server configuration
```
`python3 scripts/check_bao_roles.py`
Код завершения: 0.
```text
PASS api: scoped read, cross-role login/read and write denied, TTL <= 900 s
PASS worker: scoped read, cross-role login/read and write denied, TTL <= 900 s
PASS dispatcher: scoped read, cross-role login/read and write denied, TTL <= 900 s
PASS cleanup: scoped read, cross-role login/read and write denied, TTL <= 900 s
```
`sleep 60`
Код завершения: 0.
```text
```
`python3 scripts/check_grade3_http.py --ca-file secrets/tls/lab7-ca.crt`
Код завершения: 0.
```text
PASS: upload size/MIME, forged identity, cross-chat attachment, wrong TLS hostname
```
`python3 scripts/check_grade4_telemetry.py`
Код завершения: 0.
```text
PASS: synthetic Basic secret absent from correlated log, correlated trace, metric labels and container logs
```
Итог: все команды завершились успешно.
+13
View File
@@ -0,0 +1,13 @@
# Права контейнеров, лимиты и эксплуатационные границы
Предпосылки: подготовленный, разблокированный учебный стенд; Docker Compose, Python и локальный CA.
`python3 scripts/check_grade3_runtime.py`
Код завершения: 0.
```text
PASS: local ports, read-only app/ingress/telemetry, no-new-privileges, dropped capabilities, worker CPU/RAM/time budget
```
Итог: все команды завершились успешно.
+28
View File
@@ -0,0 +1,28 @@
# Ротация секрета и leaf, отзыв, expiry и seal
Предпосылки: подготовленный, разблокированный учебный стенд; Docker Compose, Python и локальный CA.
`python3 scripts/check_grade4_lifecycle.py`
Код завершения: 0.
```text
Container highload-lab7-api-2 Recreate
Container highload-lab7-api-1 Recreate
Container highload-lab7-api-1 Recreated
Container highload-lab7-api-2 Recreated
Container highload-lab7-api-1 Starting
Container highload-lab7-api-1 Started
Container highload-lab7-api-2 Starting
Container highload-lab7-api-2 Started
OpenBao unsealed
PASS api: scoped read, cross-role login/read and write denied, TTL <= 900 s
PASS worker: scoped read, cross-role login/read and write denied, TTL <= 900 s
PASS dispatcher: scoped read, cross-role login/read and write denied, TTL <= 900 s
PASS cleanup: scoped read, cross-role login/read and write denied, TTL <= 900 s
PASS: replaced leaf, revoked old token, expired short-lived token
PASS: database secret rotated, old password refused, API ready without rebuilding
PASS: sealed OpenBao denies new login; running API remains ready using loaded credentials
```
Итог: все команды завершились успешно.
+86
View File
@@ -0,0 +1,86 @@
# Rehash в PostgreSQL и бюджет защиты входа
Предпосылки: подготовленный, разблокированный учебный стенд; Docker Compose, Python и локальный CA.
`docker compose --profile test build backend-tests`
Код завершения: 0.
```text
Image highload-lab7-backend-tests Building
#1 [internal] load local bake definitions
#1 reading from stdin 1.66kB done
#1 DONE 0.0s
#2 [internal] load build definition from Dockerfile
#2 transferring dockerfile: 816B done
#2 DONE 0.0s
#3 [internal] load metadata for docker.io/library/golang:1.22.2-bookworm
#3 DONE 1.3s
#4 [internal] load .dockerignore
#4 transferring context: 327B done
#4 DONE 0.0s
#5 [builder 1/7] FROM docker.io/library/golang:1.22.2-bookworm@sha256:d0902bacefdde1cf45528c098d14e55d78c107def8a22d148eabd71582d7a99f
#5 resolve docker.io/library/golang:1.22.2-bookworm@sha256:d0902bacefdde1cf45528c098d14e55d78c107def8a22d148eabd71582d7a99f 0.0s done
#5 DONE 0.0s
#6 [internal] load build context
#6 transferring context: 13.81kB done
#6 DONE 0.0s
#7 [builder 2/7] RUN apt-get update && apt-get install -y --no-install-recommends webp && rm -rf /var/lib/apt/lists/*
#7 CACHED
#8 [builder 3/7] WORKDIR /src
#8 CACHED
#9 [builder 4/7] COPY go.mod go.sum ./
#9 CACHED
#10 [builder 5/7] RUN go mod download
#10 CACHED
#11 [builder 6/7] COPY backend ./backend
#11 CACHED
#12 [builder 7/7] RUN CGO_ENABLED=0 GOOS=linux go build -trimpath -ldflags="-s -w" -o /out/messenger ./backend
#12 CACHED
#13 exporting to image
#13 exporting layers done
#13 exporting manifest sha256:283fe5f0706359d473b23cc42eb6f2a6eb75a0d12b0f5b23a591907e982c8181 done
#13 exporting config sha256:f1628c981eb4ff32d1b96c99c17b0dc4556a0a9fc01033b1ad0802e2428aa0d7 done
#13 exporting attestation manifest sha256:1d394f0779fff35a4bdb3df53f1c276fbaa3ad3826c1620e226215638ac81a3a 0.0s done
#13 exporting manifest list sha256:16c3bf9e4a23386b4f3bfb72d5b57fdf8f98ca796a286c299699a2727eeac5ad done
#13 naming to docker.io/library/highload-lab7-backend-tests:latest done
#13 unpacking to docker.io/library/highload-lab7-backend-tests:latest done
#13 DONE 0.1s
#14 resolving provenance for metadata file
#14 DONE 0.0s
Image highload-lab7-backend-tests Built
```
`sleep 60`
Код завершения: 0.
```text
```
`python3 scripts/check_grade4_login.py`
Код завершения: 0.
```text
Container highload-lab7-backend-tests-run-fd0b3ea519f7 Creating
Container highload-lab7-backend-tests-run-fd0b3ea519f7 Created
PASS: successful login upgraded the stored legacy Argon2id hash
{"requests": 24, "status": {"201": 8, "429": 16}, "p50_ms": 548, "max_ms": 1156}
```
Итог: все команды завершились успешно.
+46
View File
@@ -0,0 +1,46 @@
# Проверка ролей RabbitMQ и Collector
Предпосылки: подготовленный, разблокированный учебный стенд; Docker Compose, Python и локальный CA.
`python3 scripts/check_grade4_broker.py`
Код завершения: 0.
```text
PASS: no certificate
PASS: wrong-role
PASS: messenger_unregistered
PASS: messenger_worker
PASS: messenger_dispatcher
```
`python3 scripts/check_grade3_tls.py`
Код завершения: 0.
```text
PASS s3:9000: verified service SAN; foreign CA rejected
PASS broker:5671: verified service SAN; foreign CA rejected; foreign client identity rejected
PASS redis:6379: verified service SAN; foreign CA rejected
PASS db:5432: verified service SAN; foreign CA rejected
PASS Collector: missing client certificate rejected
PASS otel-collector:4318: verified service SAN; foreign CA rejected; foreign client identity rejected
PASS otel-collector:8889: verified service SAN; foreign CA rejected; foreign client identity rejected
PASS prometheus:9090: verified service SAN; foreign CA rejected; foreign client identity rejected
PASS loki:3100: verified service SAN; foreign CA rejected; foreign client identity rejected
PASS loki:3100: verified service SAN; foreign CA rejected; foreign client identity rejected
PASS tempo:4318: verified service SAN; foreign CA rejected; foreign client identity rejected
PASS tempo:3200: verified service SAN; foreign CA rejected; foreign client identity rejected
PASS grafana:3000: verified service SAN; foreign CA rejected
PASS openbao:8200: verified service SAN; foreign CA rejected
```
`python3 scripts/check_grade3_signals.py`
Код завершения: 0.
```text
PASS: Prometheus scrape up; fresh messenger API log and trace found through mTLS
```
Итог: все команды завершились успешно.
+25
View File
@@ -0,0 +1,25 @@
# Применимость результатов сканирования
Trivy 0.69.3 проверил `src/go.mod` и `src/frontend/package-lock.json`.
Для Go найдены 94 совпадения. Для npm на момент проверки совпадений нет.
Оценки ниже учитывают пути выполнения и границы доверия.
| Результат | Применимость и решение |
| --- | --- |
| CVE-2026-77405, amqp091-go 1.10.0 | Проблема связана с неявным минимумом протокола из AMQPS URI. Рабочий клиент явно задаёт TLS 1.2, проверяет сервер и предъявляет сертификат. Ветка с URI для прежних локальных тестов продолжает зависеть от настроек Go. Обновление клиента до 1.13.0 также необходимо для закрытия остальных найденных дефектов. |
| CVE-2026-33815 и CVE-2026-33816, pgx 5.5.5 | Драйвер исполняется в каждом приложении. PostgreSQL доступен через внутреннюю сеть с проверкой TLS и раздельными ролями. Компрометация доверенной БД остаётся релевантной. Снятие риска требует обновления pgx как минимум до 5.9.0 и повтора миграций и проверок отказа. |
| CVE-2026-33186, gRPC-Go 1.69.4 | Уязвимость требует gRPC-сервера с соответствующей авторизацией. Мессенджер принимает HTTP, его OTLP-экспорт идёт по HTTP. Это заключение относится к бинарнику мессенджера. Collector и Tempo оцениваются отдельно по образам и включённым listeners. |
| CVE-2025-22869, x/crypto 0.32.0 | Описанный путь проходит через SSH/SFTP-сервер. Стенд не предоставляет SSH/SFTP. Использование Argon2id из того же модуля не включает этот путь. |
| CVE-2026-46603, x/image | Затронут VP8L-декодер. Код приложения принимает PNG/JPEG, а WebP создаёт через cwebp. Go-декодер WebP используется в тестах для проверки результата. При добавлении входного WebP нужно пересмотреть применимость. |
| Находка в образе | Применимость и решение |
| --- | --- |
| OpenBao, CVE-2026-33757 и CVE-2026-33758 | Обе проблемы затрагивают OIDC. На стенде настроен cert auth, операторский endpoint доступен локально. Перед включением OIDC нужно обновить OpenBao и проверить настройки callback. |
| MinIO, CVE-2026-33322 и CVE-2026-33419 | Затронуты OIDC и LDAP. На стенде они отключены, операции S3 используют подпись SigV4. Включение внешней федерации требует повторной оценки. |
| OpenSSL, CVE-2026-31789 | База указывает на 32-битную обработку крупных X.509. Стенд использует amd64. При переносе на 32-битную архитектуру вывод изменится. |
| Tempo, CVE-2026-33186 | Библиотека есть в бинарнике Tempo. Внешняя поставка трейсов идёт по OTLP/HTTP, внутренний gRPC не публикуется на хосте. Влияние через скомпрометированный telemetry-компонент требует отдельной проверки. |
| Grafana, CVE-2025-68121 | Сканер нашёл проблему проверки TLS при возобновлении сессии в собранном Go. Grafana использует TLS, поэтому требуется обновление образа с исправленным runtime и проверка datasource-подключений. Локальная публикация и закрытая сеть уменьшают достижимость. |
Состав образов и версии исправлений приведены в [результате Trivy](grade4-scan.md).
Это анализ применимости для учебной конфигурации с текущим профилем.
Невключённый путь выполнения не устраняет наличие уязвимого пакета.
+50
View File
@@ -0,0 +1,50 @@
# Сканирование зависимостей и образов
Предпосылки: подготовленный, разблокированный учебный стенд; Docker Compose, Python и локальный CA.
`./scripts/scan_grade4.sh`
Код завершения: 0.
```text
Scanning highload-lab7-migrate (sha256:12ea72de68ee2d99865a1ea90ccb8162d8f1bbbde6b44d126781d39d5cc42b8c)
Scanning highload-lab7-backend-tests (sha256:16c3bf9e4a23386b4f3bfb72d5b57fdf8f98ca796a286c299699a2727eeac5ad)
Scanning postgres:16.10-bookworm (sha256:38471f330eb885e04de130b768d6db4e10469e2311879c7e5c699f6d2d8a1c74)
Scanning grafana/tempo:2.7.1 (sha256:4443be217c396b065ee34845534199c36fdba4dc619cb96550e228d73fba6e69)
Scanning redis:7.4.11-bookworm (sha256:44960b3100c48159a66d28c18b0588c193492e25f7551d60e51dd464fc3eca5b)
Container image sha256:4f567dd06e97febea3c531bfdc7e8d58055e292c31132a4b64f3858f2a99d0ed is untagged/unavailable; scanning configured image highload-lab7-s3
Scanning highload-lab7-s3 (sha256:cfd603a785197d29fb1509d599c7fdcac782b7e8f288ef22084f6e10ec891c3a)
Scanning highload-lab7-upload-cleanup (sha256:4fe5584ba26276e2d003012bb32ad34f43d965d2d8c8a39bb17f7603b6aa35cc)
Scanning grafana/loki:3.4.2 (sha256:58a6c186ce78ba04d58bfe2a927eff296ba733a430df09645d56cdc158f3ba08)
Scanning openbao/openbao:2.4.4 (sha256:595c83b42614a4d2b044608e4593c05b019c5db25bc9c185d8fff3ac96c03ddd)
Scanning haproxy:3.2.25-alpine (sha256:5d97434a423c2533cfeb42874d45cf6d69a840b60334582dfbfd5008e94c80be)
Scanning rabbitmq:4.2.0-management (sha256:6431bbd0f22df99ace800138399367d7d6181442dfff09feb2795e15d4bd4b7a)
Scanning prom/prometheus:v3.2.1 (sha256:6927e0919a144aa7616fd0137d4816816d42f6b816de3af269ab065250859a62)
Scanning highload-lab7-s3-bootstrap (sha256:79b6e5b43792b2063e8154b3ef52bfb111ad2096eda8166ba8bcd392eef85ae2)
Scanning otel/opentelemetry-collector-contrib:0.120.0 (sha256:85ac41c2db88d0df9bd6145e608a3cb023f5d8443868adbfbbf66efb51087917)
Scanning grafana/grafana:11.5.2 (sha256:8b37a2f028f164ce7b9889e1765b9d6ee23fec80f871d156fbf436d6198d32b7)
Scanning highload-lab7-worker (sha256:929b6d0c28efc0f004d2a7c3b0bcb06e92ddca5f6d2ec631f8bf4724862d7573)
Scanning highload-lab7-dispatcher (sha256:9a11add2b62e99e9abb61f4292b54175778b5fd722069aeea1dfd3eaafca0f3e)
Scanning highload-lab7-api (sha256:d33fb3ff589306b6bf4d47ec657196a198da2b25c36745fc63294163fe999436)
dependencies.json {'CRITICAL': 6, 'HIGH': 38, 'LOW': 6, 'MEDIUM': 41, 'UNKNOWN': 3}
image-sha256:12ea72de68ee2d99865a1ea90ccb8162d8f1bbbde6b44d126781d39d5cc42b8c.json {'CRITICAL': 10, 'HIGH': 121, 'LOW': 162, 'MEDIUM': 200, 'UNKNOWN': 10}
image-sha256:16c3bf9e4a23386b4f3bfb72d5b57fdf8f98ca796a286c299699a2727eeac5ad.json {'CRITICAL': 44, 'HIGH': 1802, 'LOW': 2016, 'MEDIUM': 6804, 'UNKNOWN': 27}
image-sha256:38471f330eb885e04de130b768d6db4e10469e2311879c7e5c699f6d2d8a1c74.json {'CRITICAL': 20, 'HIGH': 196, 'LOW': 225, 'MEDIUM': 313, 'UNKNOWN': 18}
image-sha256:4443be217c396b065ee34845534199c36fdba4dc619cb96550e228d73fba6e69.json {'CRITICAL': 4, 'HIGH': 89, 'LOW': 44, 'MEDIUM': 112, 'UNKNOWN': 3}
image-sha256:44960b3100c48159a66d28c18b0588c193492e25f7551d60e51dd464fc3eca5b.json {'CRITICAL': 1, 'HIGH': 41, 'LOW': 87, 'MEDIUM': 104, 'UNKNOWN': 7}
image-sha256:4fe5584ba26276e2d003012bb32ad34f43d965d2d8c8a39bb17f7603b6aa35cc.json {'CRITICAL': 10, 'HIGH': 121, 'LOW': 162, 'MEDIUM': 200, 'UNKNOWN': 10}
image-sha256:58a6c186ce78ba04d58bfe2a927eff296ba733a430df09645d56cdc158f3ba08.json {'CRITICAL': 2, 'HIGH': 58, 'LOW': 5, 'MEDIUM': 63, 'UNKNOWN': 6}
image-sha256:595c83b42614a4d2b044608e4593c05b019c5db25bc9c185d8fff3ac96c03ddd.json {'CRITICAL': 9, 'HIGH': 91, 'LOW': 64, 'MEDIUM': 82, 'UNKNOWN': 3}
image-sha256:5d97434a423c2533cfeb42874d45cf6d69a840b60334582dfbfd5008e94c80be.json {'HIGH': 4, 'MEDIUM': 1}
image-sha256:6431bbd0f22df99ace800138399367d7d6181442dfff09feb2795e15d4bd4b7a.json {'HIGH': 7, 'LOW': 73, 'MEDIUM': 320}
image-sha256:6927e0919a144aa7616fd0137d4816816d42f6b816de3af269ab065250859a62.json {'CRITICAL': 2, 'HIGH': 56, 'LOW': 8, 'MEDIUM': 65, 'UNKNOWN': 3}
image-sha256:79b6e5b43792b2063e8154b3ef52bfb111ad2096eda8166ba8bcd392eef85ae2.json {'CRITICAL': 10, 'HIGH': 121, 'LOW': 162, 'MEDIUM': 200, 'UNKNOWN': 10}
image-sha256:85ac41c2db88d0df9bd6145e608a3cb023f5d8443868adbfbbf66efb51087917.json {'CRITICAL': 7, 'HIGH': 81, 'LOW': 13, 'MEDIUM': 84, 'UNKNOWN': 3}
image-sha256:8b37a2f028f164ce7b9889e1765b9d6ee23fec80f871d156fbf436d6198d32b7.json {'CRITICAL': 7, 'HIGH': 91, 'LOW': 35, 'MEDIUM': 109, 'UNKNOWN': 3}
image-sha256:929b6d0c28efc0f004d2a7c3b0bcb06e92ddca5f6d2ec631f8bf4724862d7573.json {'CRITICAL': 10, 'HIGH': 121, 'LOW': 162, 'MEDIUM': 200, 'UNKNOWN': 10}
image-sha256:9a11add2b62e99e9abb61f4292b54175778b5fd722069aeea1dfd3eaafca0f3e.json {'CRITICAL': 10, 'HIGH': 121, 'LOW': 162, 'MEDIUM': 200, 'UNKNOWN': 10}
image-sha256:cfd603a785197d29fb1509d599c7fdcac782b7e8f288ef22084f6e10ec891c3a.json {'CRITICAL': 8, 'HIGH': 176, 'LOW': 157, 'MEDIUM': 212, 'UNKNOWN': 8}
image-sha256:d33fb3ff589306b6bf4d47ec657196a198da2b25c36745fc63294163fe999436.json {'CRITICAL': 10, 'HIGH': 121, 'LOW': 162, 'MEDIUM': 200, 'UNKNOWN': 10}
```
Итог: все команды завершились успешно.
+42
View File
@@ -0,0 +1,42 @@
global
# Operator access only inside this container; no host port is published.
stats socket ipv4@127.0.0.1:9999 level admin
log stdout format raw local0
maxconn 1024
defaults
log global
mode http
option httplog
timeout connect 1s
timeout client 35s
timeout server 35s
timeout check 1s
# Let the client decide which failed POST is safe to repeat.
retries 0
# Docker returns the current addresses of all containers of the api service.
resolvers docker
nameserver docker_dns 127.0.0.11:53
accepted_payload_size 8192
resolve_retries 3
timeout resolve 1s
timeout retry 1s
hold valid 1s
hold obsolete 1s
hold nx 1s
hold timeout 1s
hold refused 1s
hold other 1s
frontend api_entry
bind :8443 ssl crt /run/tls/ingress.pem ssl-min-ver TLSv1.2
http-request set-header X-Forwarded-For %[src]
default_backend api_replicas
backend api_replicas
balance roundrobin
option httpchk GET /health/ready
http-check expect status 200
# Separate slots receive distinct Docker DNS addresses. No sticky sessions.
server-template api "${API_REPLICAS}" "api:${APP_PORT}" ssl verify required ca-file /run/tls/ca.crt crt /run/tls/lb.pem verifyhost api ssl-min-ver TLSv1.2 check inter 1s fall 2 rise 2 resolvers docker resolve-prefer ipv4 init-addr libc,none
+19
View File
@@ -0,0 +1,19 @@
FROM golang:1.24.2-bookworm AS builder
WORKDIR /source
RUN git init . \
&& git remote add origin https://github.com/minio/minio.git \
&& git fetch --depth 1 origin 0d7408fc9969caf07de6a8c3a84f9fbb10a6739e \
&& git checkout --detach FETCH_HEAD
RUN CGO_ENABLED=0 go build -trimpath \
-ldflags="-s -w -X github.com/minio/minio/cmd.Version=2025-04-22T22:12:26Z -X github.com/minio/minio/cmd.ReleaseTag=RELEASE.2025-04-22T22-12-26Z -X github.com/minio/minio/cmd.CommitID=0d7408fc9969caf07de6a8c3a84f9fbb10a6739e" \
-o /out/minio .
FROM debian:12.15-slim
RUN apt-get update && apt-get install -y --no-install-recommends ca-certificates curl \
&& rm -rf /var/lib/apt/lists/* \
&& groupadd --system storage && useradd --system --gid storage storage \
&& mkdir /data && chown storage:storage /data
COPY --from=builder /out/minio /usr/local/bin/minio
USER storage:storage
ENTRYPOINT ["minio"]
CMD ["server", "/data", "--console-address", ":9001"]
+9
View File
@@ -0,0 +1,9 @@
api_addr = "https://openbao:8200"
storage "file" {
path = "/openbao/file"
}
listener "tcp" {
address = "0.0.0.0:8200"
tls_cert_file = "/run/tls/bao.crt"
tls_key_file = "/run/tls/bao.key"
}
+20
View File
@@ -0,0 +1,20 @@
groups:
- name: messenger
interval: 5s
rules:
- alert: MessengerHighErrorRate
expr: (sum(rate(messenger_http_requests_total{http_status_class="5xx"}[1m])) / sum(rate(messenger_http_requests_total[1m]))) > 0.05
for: 15s
labels:
severity: warning
annotations:
summary: More than 5% of business requests return 5xx over one minute
runbook: Check API error spans and PostgreSQL operations in Tempo.
- alert: MessengerQueueLag
expr: max(messenger_queue_oldest_seconds) > 20
for: 15s
labels:
severity: warning
annotations:
summary: Oldest outstanding attachment job is over 20 seconds old
runbook: Check outstanding jobs, worker saturation and consumer traces.
+61
View File
@@ -0,0 +1,61 @@
receivers:
otlp:
protocols:
http:
endpoint: 0.0.0.0:4318
tls:
cert_file: /run/otel/server.crt
key_file: /run/otel/server.key
client_ca_file: /run/otel/ingest-ca.crt
processors:
memory_limiter:
check_interval: 1s
limit_mib: 128
spike_limit_mib: 32
batch:
timeout: 1s
send_batch_size: 256
send_batch_max_size: 512
exporters:
prometheus:
endpoint: 0.0.0.0:8889
tls:
cert_file: /run/otel/server.crt
key_file: /run/otel/server.key
client_ca_file: /run/otel/export-ca.crt
resource_to_telemetry_conversion:
enabled: true
otlphttp/loki:
endpoint: https://loki:3100/otlp
tls:
ca_file: /run/tls/ca.crt
cert_file: /run/otel/client.crt
key_file: /run/otel/client.key
sending_queue:
queue_size: 64
retry_on_failure:
max_elapsed_time: 30s
otlphttp/tempo:
endpoint: https://tempo:4318
tls:
ca_file: /run/tls/ca.crt
cert_file: /run/otel/client.crt
key_file: /run/otel/client.key
sending_queue:
queue_size: 64
retry_on_failure:
max_elapsed_time: 30s
service:
pipelines:
metrics:
receivers: [otlp]
processors: [memory_limiter, batch]
exporters: [prometheus]
logs:
receivers: [otlp]
processors: [memory_limiter, batch]
exporters: [otlphttp/loki]
traces:
receivers: [otlp]
processors: [memory_limiter, batch]
exporters: [otlphttp/tempo]
@@ -0,0 +1,513 @@
{
"uid": "messenger-lab6",
"title": "Мессенджер · API и worker",
"schemaVersion": 39,
"refresh": "5s",
"time": {
"from": "now-15m",
"to": "now"
},
"panels": [
{
"id": 1,
"title": "API · запросы в секунду",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "lab6-prometheus"
},
"targets": [
{
"refId": "A",
"datasource": {
"type": "prometheus",
"uid": "lab6-prometheus"
},
"expr": "sum(rate(messenger_http_requests_total[1m]))",
"legendFormat": ""
}
],
"gridPos": {
"x": 0,
"y": 0,
"w": 12,
"h": 8
},
"fieldConfig": {
"defaults": {
"unit": "reqps"
},
"overrides": []
}
},
{
"id": 2,
"title": "API · 5xx в секунду",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "lab6-prometheus"
},
"targets": [
{
"refId": "A",
"datasource": {
"type": "prometheus",
"uid": "lab6-prometheus"
},
"expr": "sum(rate(messenger_http_requests_total{http_status_class=\"5xx\"}[1m])) or 0 * sum(rate(messenger_http_requests_total[1m]))",
"legendFormat": ""
}
],
"gridPos": {
"x": 12,
"y": 0,
"w": 12,
"h": 8
},
"fieldConfig": {
"defaults": {
"unit": "reqps"
},
"overrides": []
}
},
{
"id": 3,
"title": "API · p95 длительности",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "lab6-prometheus"
},
"targets": [
{
"refId": "A",
"datasource": {
"type": "prometheus",
"uid": "lab6-prometheus"
},
"expr": "histogram_quantile(0.95, sum by (le) (rate(messenger_http_duration_seconds_bucket[1m])))",
"legendFormat": ""
}
],
"gridPos": {
"x": 0,
"y": 8,
"w": 12,
"h": 8
},
"fieldConfig": {
"defaults": {
"unit": "s"
},
"overrides": []
}
},
{
"id": 4,
"title": "Worker · задания в минуту",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "lab6-prometheus"
},
"targets": [
{
"refId": "A",
"datasource": {
"type": "prometheus",
"uid": "lab6-prometheus"
},
"expr": "60 * sum by (job_outcome) (rate(messenger_worker_jobs_total[1m]))",
"legendFormat": "{{job_outcome}}"
}
],
"gridPos": {
"x": 12,
"y": 8,
"w": 12,
"h": 8
},
"fieldConfig": {
"defaults": {
"unit": "short"
},
"overrides": []
}
},
{
"id": 5,
"title": "Worker · p95 длительности",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "lab6-prometheus"
},
"targets": [
{
"refId": "A",
"datasource": {
"type": "prometheus",
"uid": "lab6-prometheus"
},
"expr": "histogram_quantile(0.95, sum by (le,job_outcome) (rate(messenger_worker_duration_seconds_bucket[1m])))",
"legendFormat": "{{job_outcome}}"
}
],
"gridPos": {
"x": 0,
"y": 16,
"w": 12,
"h": 8
},
"fieldConfig": {
"defaults": {
"unit": "s"
},
"overrides": []
}
},
{
"id": 6,
"title": "API и worker · события",
"type": "logs",
"datasource": {
"type": "loki",
"uid": "lab6-loki"
},
"targets": [
{
"refId": "A",
"datasource": {
"type": "loki",
"uid": "lab6-loki"
},
"expr": "{service_name=~\"messenger-api|messenger-worker\"}",
"legendFormat": "",
"queryType": "range"
}
],
"gridPos": {
"x": 12,
"y": 16,
"w": 12,
"h": 8
},
"fieldConfig": {
"defaults": {
"unit": ""
},
"overrides": []
}
},
{
"id": 7,
"title": "API · p50 длительности",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "lab6-prometheus"
},
"gridPos": {
"x": 0,
"y": 24,
"w": 12,
"h": 8
},
"targets": [
{
"refId": "A",
"expr": "histogram_quantile(0.5, sum by (le) (rate(messenger_http_duration_seconds_bucket[1m])))",
"legendFormat": ""
}
],
"fieldConfig": {
"defaults": {
"unit": "s"
},
"overrides": []
}
},
{
"id": 8,
"title": "API · p99 длительности",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "lab6-prometheus"
},
"gridPos": {
"x": 12,
"y": 24,
"w": 12,
"h": 8
},
"targets": [
{
"refId": "A",
"expr": "histogram_quantile(0.99, sum by (le) (rate(messenger_http_duration_seconds_bucket[1m])))",
"legendFormat": ""
}
],
"fieldConfig": {
"defaults": {
"unit": "s"
},
"overrides": []
}
},
{
"id": 9,
"title": "API · доля 5xx",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "lab6-prometheus"
},
"gridPos": {
"x": 0,
"y": 32,
"w": 12,
"h": 8
},
"targets": [
{
"refId": "A",
"expr": "(sum(rate(messenger_http_requests_total{http_status_class=\"5xx\"}[1m])) or 0 * sum(rate(messenger_http_requests_total[1m]))) / sum(rate(messenger_http_requests_total[1m]))",
"legendFormat": ""
}
],
"fieldConfig": {
"defaults": {
"unit": "percentunit"
},
"overrides": []
}
},
{
"id": 10,
"title": "Очередь attachments.v1 · незавершённые задания",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "lab6-prometheus"
},
"gridPos": {
"x": 12,
"y": 32,
"w": 12,
"h": 8
},
"targets": [
{
"refId": "A",
"expr": "max(messenger_queue_depth)",
"legendFormat": ""
}
],
"fieldConfig": {
"defaults": {
"unit": "short"
},
"overrides": []
}
},
{
"id": 11,
"title": "Очередь attachments.v1 · возраст старейшего задания",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "lab6-prometheus"
},
"gridPos": {
"x": 0,
"y": 40,
"w": 12,
"h": 8
},
"targets": [
{
"refId": "A",
"expr": "max(messenger_queue_oldest_seconds)",
"legendFormat": ""
}
],
"fieldConfig": {
"defaults": {
"unit": "s"
},
"overrides": []
}
},
{
"id": 12,
"title": "PostgreSQL · занятые соединения",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "lab6-prometheus"
},
"gridPos": {
"x": 12,
"y": 40,
"w": 12,
"h": 8
},
"targets": [
{
"refId": "A",
"expr": "sum by (service_name) (messenger_db_pool_used)",
"legendFormat": "{{service_name}}"
}
],
"fieldConfig": {
"defaults": {
"unit": "short"
},
"overrides": []
}
},
{
"id": 13,
"title": "PostgreSQL · запросы к пулу, ожидавшие соединение, в секунду",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "lab6-prometheus"
},
"gridPos": {
"x": 0,
"y": 48,
"w": 12,
"h": 8
},
"targets": [
{
"refId": "A",
"expr": "sum by (service_name) (rate(messenger_db_pool_waited_total[1m]))",
"legendFormat": "{{service_name}}"
}
],
"fieldConfig": {
"defaults": {
"unit": "ops"
},
"overrides": []
}
},
{
"id": 14,
"title": "Worker · загрузка",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "lab6-prometheus"
},
"gridPos": {
"x": 12,
"y": 48,
"w": 12,
"h": 8
},
"targets": [
{
"refId": "A",
"expr": "sum(messenger_worker_busy) / sum(messenger_worker_capacity)",
"legendFormat": ""
}
],
"fieldConfig": {
"defaults": {
"unit": "percentunit"
},
"overrides": []
}
},
{
"id": 15,
"title": "Вложения · p95 до ready",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "lab6-prometheus"
},
"gridPos": {
"x": 0,
"y": 56,
"w": 12,
"h": 8
},
"targets": [
{
"refId": "A",
"expr": "histogram_quantile(0.95, sum by (le) (rate(messenger_attachment_ready_seconds_bucket[1m])))",
"legendFormat": ""
}
],
"fieldConfig": {
"defaults": {
"unit": "s"
},
"overrides": []
}
},
{
"id": 16,
"title": "Процессы · загрузка CPU",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "lab6-prometheus"
},
"gridPos": {
"x": 12,
"y": 56,
"w": 12,
"h": 8
},
"targets": [
{
"refId": "A",
"expr": "sum by (service_name) (rate(messenger_process_cpu_seconds_total[1m]))",
"legendFormat": "{{service_name}}"
}
],
"fieldConfig": {
"defaults": {
"unit": "cores"
},
"overrides": []
}
},
{
"id": 17,
"title": "Процессы · резидентная память",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "lab6-prometheus"
},
"gridPos": {
"x": 0,
"y": 64,
"w": 12,
"h": 8
},
"targets": [
{
"refId": "A",
"expr": "sum by (service_name) (messenger_process_rss_bytes)",
"legendFormat": "{{service_name}}"
}
],
"fieldConfig": {
"defaults": {
"unit": "bytes"
},
"overrides": []
}
}
]
}
@@ -0,0 +1,6 @@
apiVersion: 1
providers:
- name: Messenger
type: file
options:
path: /var/lib/grafana/dashboards
@@ -0,0 +1,45 @@
apiVersion: 1
datasources:
- name: Prometheus
uid: lab6-prometheus
type: prometheus
access: proxy
url: https://prometheus:9090
isDefault: true
secureJsonData:
tlsClientCert: ${GRAFANA_READER_CERT}
tlsClientKey: ${GRAFANA_READER_KEY}
jsonData:
tlsAuth: true
- name: Loki
uid: lab6-loki
type: loki
access: proxy
url: https://loki:3100
secureJsonData:
tlsClientCert: ${GRAFANA_READER_CERT}
tlsClientKey: ${GRAFANA_READER_KEY}
jsonData:
tlsAuth: true
derivedFields:
- name: Trace
matcherType: label
matcherRegex: trace_id
datasourceUid: lab6-tempo
url: '$${__value.raw}'
- name: Tempo
uid: lab6-tempo
type: tempo
access: proxy
url: https://tempo:3200
secureJsonData:
tlsClientCert: ${GRAFANA_READER_CERT}
tlsClientKey: ${GRAFANA_READER_KEY}
jsonData:
tlsAuth: true
tracesToLogsV2:
datasourceUid: lab6-loki
spanStartTimeShift: '-5s'
spanEndTimeShift: '5s'
customQuery: true
query: '{service_name=~"messenger-api|messenger-worker"} | trace_id="$${__span.traceId}"'
+36
View File
@@ -0,0 +1,36 @@
auth_enabled: false
server:
http_listen_port: 3100
http_tls_config:
cert_file: /run/otel/server.crt
key_file: /run/otel/server.key
client_auth_type: RequireAndVerifyClientCert
client_ca_file: /run/otel/loki-client-ca.crt
common:
path_prefix: /loki
replication_factor: 1
ring:
instance_addr: 127.0.0.1
kvstore:
store: inmemory
storage:
filesystem:
chunks_directory: /loki/chunks
rules_directory: /loki/rules
schema_config:
configs:
- from: 2024-01-01
store: tsdb
object_store: filesystem
schema: v13
index:
prefix: index_
period: 24h
limits_config:
allow_structured_metadata: true
otlp_config:
resource_attributes:
ignore_defaults: true
attributes_config:
- action: index_label
attributes: [service.name, deployment.environment.name]
+5
View File
@@ -0,0 +1,5 @@
tls_server_config:
cert_file: /run/otel/server.crt
key_file: /run/otel/server.key
client_auth_type: RequireAndVerifyClientCert
client_ca_file: /run/otel/read-ca.crt
+13
View File
@@ -0,0 +1,13 @@
global:
scrape_interval: 5s
rule_files:
- /etc/prometheus/alerts.yaml
scrape_configs:
- job_name: otel-collector
scheme: https
tls_config:
ca_file: /run/tls/ca.crt
cert_file: /run/otel/client.crt
key_file: /run/otel/client.key
static_configs:
- targets: [otel-collector:8889]
+27
View File
@@ -0,0 +1,27 @@
# Диагностика API и обработки вложений
SLI доступности API — доля бизнес-запросов без 5xx. Цель — ≥99% за 30 минут при наличии трафика. Проверка:
```promql
1 - (sum(increase(messenger_http_requests_total{http_status_class="5xx"}[30m])) or 0 * sum(increase(messenger_http_requests_total[30m]))) / sum(increase(messenger_http_requests_total[30m]))
```
Health исключён из обоих счётчиков. `MessengerHighErrorRate` срабатывает при доле 5xx выше 5% за минуту в течение 15 секунд. В Explore → Tempo найти ошибочный серверный спан и дочерний `postgresql query`. Сравнить длительность дочернего спана с длительностью HTTP, затем сверить занятые соединения и rate запросов, ожидавших пул.
`MessengerQueueLag` срабатывает, когда старейшее незавершённое задание старше 20 секунд в течение 15 секунд. В dashboard проверить число незавершённых заданий и наличие worker. Если возраст растёт при отсутствии новых consumer spans, проверить процесс worker. После восстановления дождаться перехода заданий в ready и исчезновения активного alert.
Глубина очереди измеряется по устойчивым заданиям в PostgreSQL. В неё входят работа в RabbitMQ, leases и ожидание retry. Возраст измеряется от создания задания. Время до ready включает ожидание и повторные попытки. Гистограммы времени используют секунды. Загрузка worker — доля занятых слотов, по одному слоту на процесс. Ожидание пула отображается как частота успешных получений соединения после ожидания. Метрики CPU и RSS относятся к основным процессам Go.
Правила доступны через API Prometheus `/api/v1/alerts` или запрос `ALERTS{alertstate="firing"}` в Grafana. При недоступности Loki или Tempo приложение продолжает работу. Collector ограничивает память и очередь экспорта, повторяет отправку до 30 секунд. Сигналы сверх лимитов или после истечения retry могут быть потеряны.
Команды локальной проверки после `make app-up`:
```sh
python3 scripts/check_observability.py pipeline
python3 scripts/check_observability_incidents.py worker
python3 scripts/check_observability_incidents.py database
python3 scripts/check_observability_incidents.py retry
python3 scripts/check_observability_incidents.py telemetry
```
Сценарии выполняются последовательно на учебном стенде. DB-инцидент держит временную блокировку таблицы чатов. Проверка retry кратко останавливает S3 и проверяет номера попыток в одной трассе исходного upload. Проверка telemetry останавливает Tempo и одновременно отправляет сообщения и обрабатывает вложение.
+27
View File
@@ -0,0 +1,27 @@
server:
http_listen_port: 3200
http_tls_config:
cert_file: /run/otel/server.crt
key_file: /run/otel/server.key
client_auth_type: RequireAndVerifyClientCert
client_ca_file: /run/otel/read-ca.crt
distributor:
receivers:
otlp:
protocols:
http:
endpoint: 0.0.0.0:4318
tls:
cert_file: /run/otel/server.crt
key_file: /run/otel/server.key
client_ca_file: /run/otel/export-ca.crt
storage:
trace:
backend: local
wal:
path: /var/tempo/wal
local:
path: /var/tempo/blocks
compactor:
compaction:
block_retention: 48h
+2
View File
@@ -0,0 +1,2 @@
local all all trust
hostssl all all all scram-sha-256
+34
View File
@@ -0,0 +1,34 @@
\getenv api_password api_password
\getenv worker_password worker_password
\getenv dispatcher_password dispatcher_password
\getenv cleanup_password cleanup_password
REVOKE CREATE ON SCHEMA public FROM PUBLIC;
SELECT format('CREATE ROLE %I LOGIN', 'messenger_' || role)
FROM unnest(ARRAY['api', 'worker', 'dispatcher', 'cleanup']) AS role
WHERE NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'messenger_' || role)
\gexec
ALTER ROLE messenger_api PASSWORD :'api_password';
ALTER ROLE messenger_worker PASSWORD :'worker_password';
ALTER ROLE messenger_dispatcher PASSWORD :'dispatcher_password';
ALTER ROLE messenger_cleanup PASSWORD :'cleanup_password';
GRANT SELECT, INSERT ON users, chats, chat_members, messages, message_attachments TO messenger_api;
GRANT UPDATE (password_hash) ON users TO messenger_api;
GRANT SELECT, INSERT, UPDATE ON attachments, direct_uploads TO messenger_api;
GRANT SELECT, INSERT, DELETE ON upload_attempts, message_idempotency TO messenger_api;
GRANT UPDATE ON message_idempotency TO messenger_api;
GRANT UPDATE ON upload_attempts TO messenger_api;
GRANT DELETE ON direct_uploads TO messenger_cleanup;
GRANT INSERT ON attachment_jobs TO messenger_api;
GRANT SELECT, UPDATE ON attachments, attachment_jobs TO messenger_worker;
GRANT SELECT, INSERT, UPDATE ON attachment_job_attempts, attachment_job_failures TO messenger_worker;
GRANT INSERT ON attachment_publications TO messenger_worker;
GRANT SELECT ON attachments, attachment_jobs, attachment_job_failures TO messenger_dispatcher;
GRANT UPDATE (published_at) ON attachment_jobs TO messenger_dispatcher;
GRANT SELECT, UPDATE ON upload_attempts, direct_uploads TO messenger_cleanup;
GRANT DELETE ON upload_attempts TO messenger_cleanup;
GRANT SELECT ON attachments TO messenger_cleanup;
GRANT USAGE ON SEQUENCE attachment_publications_id_seq TO messenger_worker;
GRANT SELECT ON attachment_jobs, attachment_job_failures TO messenger_cleanup;
GRANT UPDATE (staging_deleted_at) ON attachment_jobs TO messenger_cleanup;
GRANT UPDATE (updated_at) ON attachments TO messenger_cleanup;
+13
View File
@@ -0,0 +1,13 @@
#!/bin/sh
set -eu
until rabbitmq-diagnostics -q check_running >/dev/null 2>&1; do sleep 1; done
for role in dispatcher worker; do
username="messenger_$role"
if ! rabbitmqctl -q list_users | cut -f1 | grep -qx "$username"; then
rabbitmqctl -q add_user "$username" ""
fi
rabbitmqctl -q clear_password "$username"
if [ "$role" = dispatcher ]; then write='^amq\.default$'; read='^$'
else write='^$'; read='^attachments\.v1$'; fi
rabbitmqctl -q set_permissions "$username" '^attachments\.v1$' "$write" "$read"
done
+1
View File
@@ -0,0 +1 @@
[rabbitmq_management,rabbitmq_prometheus,rabbitmq_auth_mechanism_ssl].
+12
View File
@@ -0,0 +1,12 @@
listeners.tcp = none
listeners.ssl.default = 5671
ssl_options.cacertfile = /run/tls/broker-client-ca.crt
ssl_options.certfile = /run/tls/broker.crt
ssl_options.keyfile = /run/tls/broker.key
ssl_options.verify = verify_peer
ssl_options.fail_if_no_peer_cert = true
management.tcp.port = 15672
management.tcp.ip = 127.0.0.1
prometheus.tcp.ip = 127.0.0.1
auth_mechanisms.1 = EXTERNAL
ssl_cert_login_from = common_name
+55
View File
@@ -0,0 +1,55 @@
#!/usr/bin/env python3
"""Initialize or unseal the local OpenBao; keep operator material on the host."""
import argparse
import json
import os
import ssl
from pathlib import Path
from urllib.error import HTTPError
from urllib.request import Request, urlopen
SECRETS = Path(__file__).resolve().parents[1] / "secrets"
MATERIAL = SECRETS / "bao-operator.json"
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("action", choices=("init", "unseal"))
action = parser.parse_args().action
context = ssl.create_default_context(cafile=str(SECRETS / "tls/lab7-ca.crt"))
def request(path, data):
req = Request(
"https://localhost:8200/v1/sys/" + path,
method="PUT",
data=json.dumps(data).encode(),
headers={"Content-Type": "application/json"},
)
with urlopen(req, context=context, timeout=15) as response:
return json.load(response)
if action == "init":
# Refuse to overwrite the operator's only copy of the material.
fd = os.open(MATERIAL, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600)
with os.fdopen(fd, "w") as output:
try:
data = request("init", {"secret_shares": 3, "secret_threshold": 2})
except HTTPError:
MATERIAL.unlink()
raise
json.dump(data, output)
output.flush()
os.fsync(output.fileno())
print("Operator material saved in secrets/bao-operator.json")
return
data = json.loads(MATERIAL.read_text())
for share in data["keys_base64"]:
if not request("unseal", {"key": share})["sealed"]:
print("OpenBao unsealed")
return
raise SystemExit("OpenBao remains sealed")
if __name__ == "__main__":
main()
+92
View File
@@ -0,0 +1,92 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
umask 077
dir=secrets/tls
mkdir -p "$dir"
chmod 700 "$dir"
make_key() {
if [ "$algorithm" = ec ]; then
openssl genpkey -algorithm EC -pkeyopt ec_paramgen_curve:P-256 -out "$dir/$name.key"
else
openssl genpkey -algorithm ED25519 -out "$dir/$name.key"
fi
}
make_ca() {
name=$1 algorithm=${2:-ed25519}
if [ ! -e "$dir/$name.crt" ]; then
make_key
openssl req -x509 -new -key "$dir/$name.key" -days 365 \
-subj "/CN=$name" -addext "basicConstraints=critical,CA:TRUE,pathlen:0" \
-addext "keyUsage=critical,keyCertSign,cRLSign" \
-out "$dir/$name.crt"
fi
}
make_leaf() {
name=$1 ca=$2 usage=$3 san=$4 algorithm=${5:-ed25519}
if [ ! -e "$dir/$name.crt" ]; then
make_key
openssl req -new -key "$dir/$name.key" -subj "/CN=$name" -out "$dir/$name.csr"
printf 'basicConstraints=critical,CA:FALSE\nkeyUsage=critical,digitalSignature\nextendedKeyUsage=%s\nsubjectAltName=%s\nsubjectKeyIdentifier=hash\nauthorityKeyIdentifier=keyid:always\n' \
"$usage" "$san" > "$dir/$name.ext"
openssl x509 -req -in "$dir/$name.csr" -CA "$dir/$ca.crt" -CAkey "$dir/$ca.key" \
-CAcreateserial -days 30 -extfile "$dir/$name.ext" -out "$dir/$name.crt"
rm "$dir/$name.ext" "$dir/$name.csr"
fi
# Containers can only read files individually mounted into them.
chmod 644 "$dir/$name.crt" "$dir/$name.key"
}
make_ca lab7-ca
make_ca untrusted-ca
chmod 644 "$dir/lab7-ca.crt"
make_ca lab7-ingress-ca ec
chmod 644 "$dir/lab7-ingress-ca.crt"
make_leaf ingress-browser lab7-ingress-ca serverAuth DNS:localhost,IP:127.0.0.1 ec
make_leaf api lab7-ca serverAuth DNS:api
for service in db redis s3 broker; do
san="DNS:$service"
if [ "$service" = s3 ]; then san="$san,DNS:localhost,IP:127.0.0.1"; fi
make_leaf "$service" lab7-ca serverAuth "$san"
done
for service in otel-collector loki tempo prometheus grafana; do
san="DNS:$service"
if [ "$service" = grafana ]; then san="$san,DNS:localhost,IP:127.0.0.1"; fi
make_leaf "$service" lab7-ca serverAuth "$san"
done
make_ca telemetry-read-ca
make_ca telemetry-export-ca
chmod 644 "$dir/telemetry-read-ca.crt" "$dir/telemetry-export-ca.crt"
make_leaf grafana-reader telemetry-read-ca clientAuth URI:urn:lab7:grafana
make_leaf collector-exporter telemetry-export-ca clientAuth URI:urn:lab7:collector
make_leaf prometheus-scraper telemetry-export-ca clientAuth URI:urn:lab7:prometheus
cat "$dir/telemetry-read-ca.crt" "$dir/telemetry-export-ca.crt" > "$dir/loki-client-ca.crt"
chmod 644 "$dir/loki-client-ca.crt"
make_ca telemetry-ingest-ca
chmod 644 "$dir/telemetry-ingest-ca.crt"
for role in api worker dispatcher; do
make_leaf "otel-$role" telemetry-ingest-ca clientAuth "URI:urn:lab7:otel:$role"
done
make_ca broker-client-ca
chmod 644 "$dir/broker-client-ca.crt"
for role in dispatcher worker unregistered; do
make_leaf "messenger_$role" broker-client-ca clientAuth "URI:urn:lab7:broker:$role"
done
make_leaf openbao lab7-ca serverAuth DNS:openbao,DNS:localhost,IP:127.0.0.1
for role in api worker dispatcher cleanup; do
make_leaf "bao-$role" lab7-ca clientAuth "URI:urn:lab7:bao:$role"
done
make_leaf lb lab7-ca clientAuth URI:urn:lab7:load-balancer
make_leaf wrong-role lab7-ca clientAuth URI:urn:lab7:probe
make_leaf wrong-ca untrusted-ca clientAuth URI:urn:lab7:load-balancer
# The Collector trusts only the leaf certificates registered for ingest.
cat "$dir/otel-api.crt" "$dir/otel-worker.crt" "$dir/otel-dispatcher.crt" > "$dir/telemetry-ingest-clients.crt"
chmod 644 "$dir/telemetry-ingest-clients.crt"
make_leaf otel-wrong-role telemetry-ingest-ca clientAuth URI:urn:lab7:otel:unregistered
for name in ingress-browser lb; do
cat "$dir/$name.crt" "$dir/$name.key" > "$dir/$name.pem"
chmod 644 "$dir/$name.pem"
done
+45
View File
@@ -0,0 +1,45 @@
#!/usr/bin/env python3
"""Prove each runtime identity can read only its own OpenBao secret."""
import json
import ssl
from pathlib import Path
from urllib.error import HTTPError
from urllib.request import Request, urlopen
TLS = Path(__file__).resolve().parents[1] / "secrets/tls"
ROLES = ("api", "worker", "dispatcher", "cleanup")
def request(context, path, *, body=None, token="", status=200):
req = Request(
f"https://localhost:8200/v1/{path}",
data=None if body is None else json.dumps(body).encode(),
headers={"X-Vault-Token": token, "Content-Type": "application/json"},
)
try:
with urlopen(req, context=context, timeout=5) as response:
assert response.status == status, (path, response.status, status)
return json.load(response) if response.status != 204 else None
except HTTPError as error:
assert error.code == status, (path, error.code, status)
def main():
for role in ROLES:
context = ssl.create_default_context(cafile=str(TLS / "lab7-ca.crt"))
context.load_cert_chain(TLS / f"bao-{role}.crt", TLS / f"bao-{role}.key")
auth = request(context, "auth/cert/login", body={"name": role})["auth"]
assert set(auth["policies"]) == {role}
assert 0 < auth["lease_duration"] <= 900
token = auth["client_token"]
request(context, f"messenger/data/{role}", token=token)
request(context, f"messenger/data/{role}", token=token, body={"data": {}}, status=403)
request(context, f"sys/policies/acl/{role}", token=token, body={"policy": ""}, status=403)
for other in set(ROLES) - {role}:
request(context, f"messenger/data/{other}", token=token, status=403)
request(context, "auth/cert/login", body={"name": other}, status=400)
print(f"PASS {role}: scoped read, cross-role login/read and write denied, TTL <= 900 s")
if __name__ == "__main__":
main()
+26
View File
@@ -0,0 +1,26 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
docker compose exec -T db sh -ec '
export PGPASSWORD=$(cat /run/secrets/database_worker_password)
export PGCHANNELBINDING=disable PGSSLMODE=verify-full PGSSLROOTCERT=/run/tls/ca.crt
psql -X -h db -U messenger_worker "$POSTGRES_DB" -Atc "
SELECT has_table_privilege(current_user, '\''attachments'\'', '\''UPDATE'\'')
AND NOT has_table_privilege(current_user, '\''users'\'', '\''SELECT'\'')
AND NOT has_schema_privilege(current_user, '\''public'\'', '\''CREATE'\'');
" | grep -qx t
'
printf '%s\n' 'PASS PostgreSQL: worker can update attachments; user table and schema creation denied'
docker compose exec -T broker rabbitmqctl -q list_permissions -p / |
awk -F '\t' '
$1=="messenger_dispatcher" && $2=="^attachments\\.v1$" && $3=="^amq\\.default$" && $4=="^$" { p=1 }
$1=="messenger_worker" && $2=="^attachments\\.v1$" && $3=="^$" && $4=="^attachments\\.v1$" { c=1 }
END {exit !(p && c)}
'
printf '%s\n' 'PASS RabbitMQ: producer can publish and consumer can read attachments.v1'
docker compose exec -T redis sh -ec '
export REDISCLI_AUTH=$(cat /run/secrets/redis_password)
redis-cli --tls --cacert /run/tls/ca.crt -h redis --user messenger CONFIG GET dir | grep -q NOPERM
'
printf '%s\n' 'PASS Redis: application role cannot read server configuration'
+46
View File
@@ -0,0 +1,46 @@
#!/usr/bin/env python3
"""Negative HTTP cases beyond the supplied smoke."""
import ssl
import socket
import sys
import uuid
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "tests"))
from client import Client, arguments, run # noqa: E402
def main():
args = arguments(__doc__)
client = Client(args.base_url, args.ca_file)
alice, _, eve, chat = client.bootstrap()
upload = f"/api/v1/chats/{chat['id']}/attachments?filename=probe.png"
headers = {"Content-Type": "image/png"}
client.call("POST", upload, 413, actor=alice, raw=b"x" * (10 * 1024 * 1024 + 1), headers=headers)
client.call("POST", upload, 415, actor=alice, raw=b"not an image", headers=headers)
attachment, _ = client.attachment(alice, chat)
path = "/api/v1/attachments/" + attachment["id"]
spoofed = {"X-User-Id": alice["id"]}
for suffix in ("", "/content"):
client.call("GET", path + suffix, 404, actor=eve, headers=spoofed)
other = client.call("POST", "/api/v1/chats", 201, actor=eve,
body={"title": "ACL probe", "member_ids": [eve["id"], alice["id"]]}, schema="Chat")["json"]
client.call("POST", f"/api/v1/chats/{other['id']}/messages", 404, actor=eve,
body={"text": "", "attachment_ids": [attachment["id"]]},
headers={"Idempotency-Key": uuid.uuid4().hex, **spoofed})
context = ssl.create_default_context(cafile=args.ca_file)
from urllib.parse import urlsplit
endpoint = urlsplit(client.base)
with socket.create_connection((endpoint.hostname, endpoint.port or 443), timeout=5) as connection:
try:
with context.wrap_socket(connection, server_hostname="wrong.lab.invalid"):
raise AssertionError("TLS accepted a wrong hostname")
except ssl.SSLCertVerificationError as error:
if error.verify_code != 62:
raise
print("PASS: upload size/MIME, forged identity, cross-chat attachment, wrong TLS hostname")
if __name__ == "__main__":
run(main)
+46
View File
@@ -0,0 +1,46 @@
#!/usr/bin/env python3
"""Inspect runtime constraints without printing inspect output or secrets."""
import json
import subprocess
APP_ROLES = {"api", "worker", "dispatcher", "upload-cleanup"}
READ_ONLY = APP_ROLES | {"haproxy", "otel-collector", "prometheus", "loki", "tempo", "grafana"}
ALLOWED_PORTS = {("haproxy", "8443/tcp"), ("grafana", "3000/tcp"),
("openbao", "8200/tcp"), ("s3", "9000/tcp")}
def main():
ids = subprocess.check_output(["docker", "compose", "ps", "-q"], text=True).split()
assert ids, "No running Compose services"
services = json.loads(subprocess.check_output(["docker", "inspect", *ids], text=True))
for service in services:
name = service["Config"]["Labels"]["com.docker.compose.service"]
config = service["HostConfig"]
assert not config["Privileged"] and config["NetworkMode"] != "host", name
mounts = [mount["Destination"] for mount in service["Mounts"]]
assert not any("docker.sock" in path for path in mounts), name
for port, bindings in (config.get("PortBindings") or {}).items():
assert (name, port) in ALLOWED_PORTS, (name, port)
assert all(binding["HostIp"] == "127.0.0.1" for binding in bindings), name
if name in READ_ONLY:
assert config["ReadonlyRootfs"] and config["CapDrop"] == ["ALL"], name
assert "no-new-privileges:true" in config["SecurityOpt"], name
assert config["Memory"] > 0 and config["NanoCpus"] > 0, name
assert config["Tmpfs"], name
if name == "worker":
env = dict(item.split("=", 1) for item in service["Config"]["Env"] if "=" in item)
assert 0 < int(env["JOB_TIMEOUT_SECONDS"]) <= 60, name
assert config["Memory"] <= 512 * 1024 * 1024 and config["NanoCpus"] <= 10**9, name
if name in APP_ROLES:
assert service["Config"]["User"] == "app:app", name
assert config["CapDrop"] == ["ALL"] and not config["CapAdd"], name
assert config["Memory"] > 0 and config["NanoCpus"] > 0, name
assert config["ReadonlyRootfs"] and "/run/runtime-secrets" in config["Tmpfs"], name
assert not any(path.startswith("/run/secrets") for path in mounts), name
assert not any("BAO_TOKEN=" in field for field in service["Config"]["Env"]), name
assert APP_ROLES <= {svc["Config"]["Labels"]["com.docker.compose.service"] for svc in services}
print("PASS: local ports, read-only app/ingress/telemetry, no-new-privileges, dropped capabilities, worker CPU/RAM/time budget")
if __name__ == "__main__":
main()
+63
View File
@@ -0,0 +1,63 @@
#!/usr/bin/env python3
"""Confirm metrics, logs and traces reach their HTTPS stores."""
import json
import ssl
import subprocess
import time
from pathlib import Path
from urllib.parse import urlencode
from urllib.request import urlopen
from urllib.error import HTTPError
import http.client
TLS = Path(__file__).resolve().parents[1] / "secrets/tls"
def query(service, port, path, params, *, allow_missing=False):
context = ssl.create_default_context(cafile=str(TLS / "lab7-ca.crt"))
context.load_cert_chain(TLS / "grafana-reader.crt", TLS / "grafana-reader.key")
container = subprocess.check_output(["docker", "compose", "ps", "-q", service], text=True).strip()
address = subprocess.check_output(["docker", "inspect", "-f",
'{{(index .NetworkSettings.Networks "highload-lab7_telemetry").IPAddress}}', container],
text=True).strip()
connection = http.client.HTTPSConnection(service, port, context=context, timeout=5)
create_connection = connection._create_connection
connection._create_connection = lambda _, *args: create_connection((address, port), *args)
try:
connection.request("GET", f"{path}?{urlencode(params)}")
response = connection.getresponse()
if allow_missing and response.status == 404:
return None
assert response.status == 200, (service, response.status)
return json.load(response)
finally:
connection.close()
def main():
try:
with urlopen("https://localhost:8443/api/v1/users/me", context=ssl.create_default_context(
cafile=str(TLS / "lab7-ingress-ca.crt")), timeout=5):
raise AssertionError("Anonymous request should be rejected")
except HTTPError as error:
assert error.code == 401
since = int(time.time()) - 10
for _ in range(30):
now = int(time.time())
metrics = query("prometheus", 9090, "/api/v1/query", {
"query": 'up{job="otel-collector"} == 1'})["data"]["result"]
logs = query("loki", 3100, "/loki/api/v1/query_range", {
"query": '{service_name="messenger-api"}', "start": since * 10**9,
"end": now * 10**9, "limit": 1})["data"]["result"]
traces = query("tempo", 3200, "/api/search", {
"q": '{ resource.service.name = "messenger-api" }',
"start": since, "end": now, "limit": 1}).get("traces")
if metrics and logs and traces:
print("PASS: Prometheus scrape up; fresh messenger API log and trace found through mTLS")
return
time.sleep(2)
raise SystemExit("Telemetry signal missing within 60 seconds")
if __name__ == "__main__":
main()
+75
View File
@@ -0,0 +1,75 @@
#!/usr/bin/env python3
"""Verify trusted TLS on data-plane endpoints, from the Linux Docker host."""
import json
import socket
import ssl
import subprocess
from pathlib import Path
TLS = Path(__file__).resolve().parents[1] / "secrets/tls"
ENDPOINTS = (
("s3", 9000, None, False),
("broker", 5671, "messenger_worker", False),
("redis", 6379, None, False),
("db", 5432, None, True),
("otel-collector", 4318, "otel-api", False),
("otel-collector", 8889, "prometheus-scraper", False),
("prometheus", 9090, "grafana-reader", False),
("loki", 3100, "collector-exporter", False),
("loki", 3100, "grafana-reader", False),
("tempo", 4318, "collector-exporter", False),
("tempo", 3200, "grafana-reader", False),
("grafana", 3000, None, False),
("openbao", 8200, None, False),
)
def probe(address, service, port, identity, pg, ca):
context = ssl.create_default_context(cafile=str(TLS / f"{ca}.crt"))
if identity:
context.load_cert_chain(TLS / f"{identity}.crt", TLS / f"{identity}.key")
with socket.create_connection((address, port), timeout=5) as conn:
if pg:
conn.sendall(bytes.fromhex("0000000804d2162f"))
assert conn.recv(1) == b"S", "PostgreSQL did not accept TLS negotiation"
with context.wrap_socket(conn, server_hostname=service) as encrypted:
assert encrypted.version() in {"TLSv1.2", "TLSv1.3"}
if identity or (service, port) == ("otel-collector", 4318):
encrypted.sendall(b"GET / HTTP/1.0\r\n\r\n")
encrypted.recv(1)
def main():
for service, port, identity, pg in ENDPOINTS:
container = subprocess.check_output(["docker", "compose", "ps", "-q", service], text=True).strip()
networks = json.loads(subprocess.check_output(
["docker", "inspect", "-f", "{{json .NetworkSettings.Networks}}", container], text=True))
address = next(network["IPAddress"] for network in networks.values() if network["IPAddress"])
probe(address, service, port, identity, pg, "lab7-ca")
try:
probe(address, service, port, identity, pg, "untrusted-ca")
except ssl.SSLCertVerificationError:
pass
else:
raise AssertionError(f"{service}:{port} accepted a foreign CA")
if (service, port) == ("otel-collector", 4318):
try:
probe(address, service, port, None, pg, "lab7-ca")
except ssl.SSLError:
print("PASS Collector: missing client certificate rejected")
else:
raise AssertionError("Collector accepted an anonymous client")
if identity:
foreign = "otel-wrong-role" if (service, port) == ("otel-collector", 4318) else "wrong-role"
try:
probe(address, service, port, foreign, pg, "lab7-ca")
except ssl.SSLError:
pass
else:
raise AssertionError(f"{service}:{port} accepted a foreign client identity")
print(f"PASS {service}:{port}: verified service SAN; foreign CA rejected"
+ ("; foreign client identity rejected" if identity else ""))
if __name__ == "__main__":
main()
+91
View File
@@ -0,0 +1,91 @@
#!/usr/bin/env python3
"""Exercise AMQP EXTERNAL authentication and role ACLs on new connections."""
import socket
import ssl
import struct
import subprocess
from pathlib import Path
TLS = Path(__file__).resolve().parents[1] / "secrets/tls"
def read_exact(conn, size):
result = bytearray()
while len(result) < size:
chunk = conn.recv(size - len(result))
if not chunk:
raise EOFError("RabbitMQ closed the connection")
result.extend(chunk)
return bytes(result)
def read_method(conn):
kind, _, size = struct.unpack(">BHI", read_exact(conn, 7))
assert kind == 1 and size >= 4
body = read_exact(conn, size)
assert read_exact(conn, 1) == b"\xce"
return struct.unpack(">HH", body[:4]), body[4:]
def send_method(conn, channel, klass, method, args):
body = struct.pack(">HH", klass, method) + args
conn.sendall(struct.pack(">BHI", 1, channel, len(body)) + body + b"\xce")
def short(value):
return bytes([len(value)]) + value
def authenticate(conn):
conn.sendall(b"AMQP\x00\x00\x09\x01")
method, data = read_method(conn)
assert method == (10, 10) and b"EXTERNAL" in data
send_method(conn, 0, 10, 11, b"\x00" * 4 + short(b"EXTERNAL") + b"\x00" * 4 + short(b"en_US"))
try:
method, data = read_method(conn)
except EOFError:
return False
if method == (10, 50):
assert struct.unpack_from(">H", data)[0] == 403
return False
assert method == (10, 30)
send_method(conn, 0, 10, 31, data)
send_method(conn, 0, 10, 40, short(b"/") + short(b"") + b"\x00")
assert read_method(conn)[0] == (10, 41)
return True
def main():
container = subprocess.check_output(["docker", "compose", "ps", "-q", "broker"], text=True).strip()
address = subprocess.check_output(["docker", "inspect", "-f",
'{{(index .NetworkSettings.Networks "highload-lab7_data").IPAddress}}', container], text=True).strip()
for identity in (None, "wrong-role", "messenger_unregistered", "messenger_worker", "messenger_dispatcher"):
context = ssl.create_default_context(cafile=str(TLS / "lab7-ca.crt"))
if identity:
context.load_cert_chain(TLS / f"{identity}.crt", TLS / f"{identity}.key")
with socket.create_connection((address, 5671), timeout=5) as tcp:
try:
with context.wrap_socket(tcp, server_hostname="broker") as conn:
admitted = authenticate(conn)
assert admitted == (identity in ("messenger_worker", "messenger_dispatcher"))
if admitted:
send_method(conn, 1, 20, 10, short(b""))
assert read_method(conn)[0] == (20, 11)
# Worker cannot publish. Dispatcher cannot read using basic.get.
if identity == "messenger_worker":
send_method(conn, 1, 60, 40, b"\x00\x00" + short(b"") + short(b"attachments.v1") + b"\x00")
conn.sendall(struct.pack(">BHIHHQH", 2, 1, 14, 60, 0, 0, 0) + b"\xce")
else:
send_method(conn, 1, 60, 70, b"\x00\x00" + short(b"attachments.v1") + b"\x01")
method, data = read_method(conn)
assert method == (20, 40) and struct.unpack_from(">H", data)[0] == 403
except ssl.SSLError:
if identity not in (None, "wrong-role"):
raise
else:
assert identity not in (None, "wrong-role"), "TLS accepted an untrusted peer"
print(f"PASS: {identity or 'no certificate'}")
if __name__ == "__main__":
main()
+118
View File
@@ -0,0 +1,118 @@
#!/usr/bin/env python3
"""Rotate a real API database password and OpenBao client leaf on the lab stand."""
import json
import os
import secrets
import ssl
import subprocess
import time
from pathlib import Path
from urllib.error import HTTPError
from urllib.parse import urlsplit, urlunsplit
from check_bao_roles import request
ROOT = Path(__file__).resolve().parents[1]
FILES = ROOT / "secrets"
TLS = FILES / "tls"
CA = TLS / "lab7-ca.crt"
def run(*args, **kwargs):
return subprocess.run(args, cwd=ROOT, check=True, **kwargs)
def identity(cert, key):
context = ssl.create_default_context(cafile=str(CA))
context.load_cert_chain(cert, key)
return context
def main():
admin = json.loads((FILES / "bao-operator.json").read_text())["root_token"]
base = ssl.create_default_context(cafile=str(CA))
old = identity(TLS / "bao-api.crt", TLS / "bao-api.key")
login = lambda ctx: request(ctx, "auth/cert/login", body={"name": "api"})["auth"]
policy_path = "auth/cert/certs/api"
secret_path = "messenger/data/api"
old_auth = login(old)
values = request(old, secret_path, token=old_auth["client_token"])["data"]["data"]
old_password = urlsplit(values["database_url"]).password
password = secrets.token_hex(24)
for name in ("bao-api-next.crt", "bao-api-next.key"):
if (TLS / name).exists():
(TLS / name).unlink()
run("openssl", "req", "-new", "-newkey", "ed25519", "-nodes", "-subj", "/CN=bao-api",
"-keyout", str(TLS / "bao-api-next.key"), "-out", str(TLS / "bao-api-next.csr"),
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
run("openssl", "x509", "-req", "-in", str(TLS / "bao-api-next.csr"), "-CA", str(CA),
"-CAkey", str(TLS / "lab7-ca.key"), "-CAcreateserial", "-days", "30",
"-extfile", "/dev/stdin", "-out", str(TLS / "bao-api-next.crt"), input=
"basicConstraints=critical,CA:FALSE\nkeyUsage=critical,digitalSignature\nextendedKeyUsage=clientAuth\n"
"subjectAltName=URI:urn:lab7:bao:api\nsubjectKeyIdentifier=hash\nauthorityKeyIdentifier=keyid:always\n",
text=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
new_cert, new_key = TLS / "bao-api-next.crt", TLS / "bao-api-next.key"
new = identity(new_cert, new_key)
leaf = new_cert.read_text()
def role(ttl):
request(base, policy_path, token=admin, status=204, body={
"certificate": leaf, "token_policies": ["api"], "token_no_default_policy": True,
"token_ttl": ttl, "token_max_ttl": ttl,
})
def db_login(candidate, allowed):
result = subprocess.run(["docker", "compose", "exec", "-T", "db", "sh", "-ec",
'IFS= read -r PGPASSWORD; export PGPASSWORD PGCHANNELBINDING=disable '
'PGSSLMODE=verify-full PGSSLROOTCERT=/run/tls/ca.crt; '
'psql -X -h db -U messenger_api "$POSTGRES_DB" -Atc "SELECT 1"'],
cwd=ROOT, input=candidate+"\n", text=True, capture_output=True)
assert (result.returncode == 0 and result.stdout.strip() == "1") == allowed
if not allowed:
assert "password authentication failed" in result.stderr
role("3s")
try:
request(old, "auth/cert/login", body={"name": "api"}, status=400)
request(base, "auth/token/revoke-accessor", token=admin, status=204,
body={"accessor": old_auth["accessor"]})
request(old, secret_path, token=old_auth["client_token"], status=403)
auth = login(new)
request(new, secret_path, token=auth["client_token"])
time.sleep(auth["lease_duration"] + 1)
request(new, secret_path, token=auth["client_token"], status=403)
print("PASS: replaced leaf, revoked old token, expired short-lived token")
finally:
role("15m")
for source, target in ((new_cert, TLS / "bao-api.crt"), (new_key, TLS / "bao-api.key")):
source.chmod(0o644)
os.replace(source, target)
(TLS / "bao-api-next.csr").unlink(missing_ok=True)
db_login(old_password, True)
run("docker", "compose", "exec", "-T", "db", "sh", "-ec",
'psql -X -v ON_ERROR_STOP=1 -U "$POSTGRES_USER" "$POSTGRES_DB"',
text=True, input=f"ALTER ROLE messenger_api PASSWORD '{password}';\n", stdout=subprocess.DEVNULL)
parts = urlsplit(values["database_url"])
values["database_url"] = urlunsplit(parts._replace(netloc=f"{parts.username}:{password}@{parts.hostname}:{parts.port}"))
request(base, secret_path, token=admin, status=200, body={"data": values})
for path, content in (("database_api_password", password), ("database_api_url", values["database_url"])):
(FILES / path).write_text(content + "\n")
db_login(old_password, False)
db_login(password, True)
run("docker", "compose", "up", "-d", "--no-deps", "--no-build", "--force-recreate", "api")
run("./scripts/wait_ingress.sh")
print("PASS: database secret rotated, old password refused, API ready without rebuilding")
try:
request(base, "sys/seal", token=admin, status=204, body={})
request(new, "auth/cert/login", body={"name": "api"}, status=503)
run("./scripts/wait_ingress.sh")
print("PASS: sealed OpenBao denies new login; running API remains ready using loaded credentials")
finally:
run("python3", "scripts/bao_operator.py", "unseal")
run("python3", "scripts/check_bao_roles.py")
if __name__ == "__main__":
main()
+66
View File
@@ -0,0 +1,66 @@
"""Verify rehash in PostgreSQL and measure rejected concurrent login work."""
import base64
import concurrent.futures
import json
import os
import ssl
import subprocess
import sys
import time
from pathlib import Path
from urllib.error import HTTPError
from urllib.request import Request, urlopen
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "tests"))
from client import Client
def main():
ca = "secrets/tls/lab7-ingress-ca.crt"
endpoint = "https://localhost:8443"
api = Client(endpoint, ca)
actor = api.user("rehash")
registration = actor["registration"]
user_id = actor["id"]
def sql(statement):
return subprocess.check_output(["docker", "compose", "exec", "-T", "db", "sh", "-ec",
'psql -X -v ON_ERROR_STOP=1 -U "$POSTGRES_USER" "$POSTGRES_DB" -At'],
text=True, input=statement, cwd=ROOT).strip()
# Derive a synthetic legacy hash with the same Go implementation in the test image.
legacy = subprocess.check_output(["docker", "compose", "--profile", "test", "run", "--rm", "-T",
"--no-deps", "-e", "LAB7_REHASH_FIXTURE", "backend-tests",
"go", "test", "./backend/services", "-run", "^TestPrintLegacyFixture$", "-v"],
cwd=ROOT, text=True, env=os.environ | {"LAB7_REHASH_FIXTURE": registration["password"]})
old = next(line for line in legacy.splitlines() if line.startswith("$argon2id$"))
sql(f"UPDATE users SET password_hash=convert_to('{old}','UTF8') WHERE id='{user_id}';")
auth = "Basic " + base64.b64encode(
f"{registration['username']}:{registration['password']}".encode()).decode()
api.call("POST", "/api/v1/auth/sessions", 201, headers={"Authorization": auth}, schema="Session")
assert sql(f"SELECT split_part(convert_from(password_hash,'UTF8'),'$',4) FROM users WHERE id='{user_id}';") == "m=65536,t=3,p=2"
print("PASS: successful login upgraded the stored legacy Argon2id hash")
context = ssl.create_default_context(cafile=ca)
def login(_):
started = time.monotonic()
req = Request(endpoint + "/api/v1/auth/sessions", data=b"", headers={"Authorization": auth})
try:
with urlopen(req, context=context, timeout=10) as response:
return response.status, time.monotonic()-started
except HTTPError as error:
assert error.code == 429 and error.headers["Retry-After"]
return error.code, time.monotonic()-started
with concurrent.futures.ThreadPoolExecutor(max_workers=24) as pool:
result = list(pool.map(login, range(24)))
status = {code: sum(c == code for c, _ in result) for code in (201, 429)}
assert all(status.values()), status
latencies = sorted(duration for _, duration in result)
print(json.dumps({"requests": len(result), "status": status,
"p50_ms": round(latencies[11]*1000), "max_ms": round(latencies[-1]*1000)}))
if __name__ == "__main__":
main()
+56
View File
@@ -0,0 +1,56 @@
"""Send a synthetic marker as a credential and check exported signals for it."""
import base64
import json
import secrets
import ssl
import subprocess
import time
from urllib.error import HTTPError
from urllib.request import Request, urlopen
from check_grade3_signals import TLS, query
def main():
marker = "lab7-secret-" + secrets.token_hex(16)
encoded = base64.b64encode(("lab7_probe:" + marker).encode()).decode()
context = ssl.create_default_context(cafile=str(TLS / "lab7-ingress-ca.crt"))
started = int(time.time()) - 1
req = Request("https://localhost:8443/api/v1/auth/sessions", data=b"",
headers={"Authorization": "Basic " + encoded})
try:
with urlopen(req, context=context, timeout=5):
raise AssertionError("Synthetic invalid login succeeded")
except HTTPError as error:
assert error.code == 401
request_id = error.headers["X-Request-Id"]
assert request_id
for _ in range(30):
now = int(time.time()) + 1
selector = '{service_name="messenger-api"} | request_id=`' + request_id + '`'
window = {"start": started * 10**9, "end": now * 10**9, "limit": 10}
logs = query("loki", 3100, "/loki/api/v1/query_range",
{**window, "query": selector})
correlated = query("loki", 3100, "/loki/api/v1/query_range",
{**window, "query": selector + ' | line_format "{{.trace_id}}"'})
trace_id = next((entry[1] for stream in correlated["data"]["result"]
for entry in stream["values"] if len(entry[1]) == 32), None)
if trace_id:
trace = query("tempo", 3200, "/api/traces/" + trace_id, {}, allow_missing=True)
metrics = query("prometheus", 9090, "/api/v1/query", {
"query": 'messenger_http_requests_total{http_route="/api/v1/auth/sessions",http_status_class="4xx"}',
})["data"]["result"]
if trace and metrics:
break
time.sleep(2)
else:
raise AssertionError("Missing correlated log, trace or ingress metric for synthetic request")
container_logs = subprocess.check_output(
["docker", "compose", "logs", "--since", str(started), "api", "otel-collector"], text=True)
for output in (json.dumps(logs), json.dumps(trace), json.dumps(metrics), container_logs):
assert marker not in output and encoded not in output
print("PASS: synthetic Basic secret absent from correlated log, correlated trace, metric labels and container logs")
if __name__ == "__main__":
main()
+56
View File
@@ -0,0 +1,56 @@
#!/usr/bin/env python3
"""Probe each API replica directly from the Docker host on Linux."""
import http.client
import json
import ssl
import socket
import subprocess
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
TLS = ROOT / "secrets/tls"
def check(address, port, identity, error):
context = ssl.create_default_context(cafile=str(TLS / "lab7-ca.crt"))
if identity:
context.load_cert_chain(TLS / f"{identity}.crt", TLS / f"{identity}.key")
with socket.create_connection((address, port), timeout=5) as tcp:
try:
with context.wrap_socket(tcp, server_hostname="api") as connection:
connection.sendall(b"GET /health/live HTTP/1.1\r\nHost: api\r\nConnection: close\r\n\r\n")
response = http.client.HTTPResponse(connection)
response.begin()
assert response.status == 200, response.status
except ssl.SSLError as exc:
if not error or error not in str(exc).lower():
raise
return
if error:
raise AssertionError(f"{identity or 'no certificate'} unexpectedly accepted")
def main():
config = subprocess.check_output(
["docker", "compose", "config", "--format", "json"], cwd=ROOT, text=True
)
port = int(json.loads(config)["services"]["api"]["environment"]["APP_PORT"])
replicas = subprocess.check_output(
["docker", "compose", "ps", "-q", "api"], cwd=ROOT, text=True
).split()
assert len(replicas) >= 2, "Expected at least two running API replicas"
for replica in replicas:
address = subprocess.check_output(
["docker", "inspect", "--format",
'{{(index .NetworkSettings.Networks "highload-lab7_proxy").IPAddress}}', replica],
text=True,
).strip()
check(address, port, "lb", None)
check(address, port, None, "certificate required")
check(address, port, "wrong-ca", "unknown ca")
check(address, port, "wrong-role", "bad certificate")
print(f"PASS {replica[:12]}: LB permitted; absent, foreign CA and disallowed SAN rejected")
if __name__ == "__main__":
main()
+40
View File
@@ -0,0 +1,40 @@
#!/usr/bin/env python3
"""Run against a quiet test ingress after at least 60 seconds without logins."""
import argparse
import ssl
from urllib.error import HTTPError
from urllib.request import Request, urlopen
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--base-url", default="https://localhost:8443")
parser.add_argument("--ca-file", default="secrets/tls/lab7-ingress-ca.crt")
args = parser.parse_args()
context = ssl.create_default_context(cafile=args.ca_file)
instances = set()
for attempt in range(1, 14):
request = Request(
args.base_url.rstrip("/") + "/api/v1/auth/sessions",
data=b"",
headers={"X-Forwarded-For": f"192.0.2.{attempt}"},
)
try:
with urlopen(request, context=context, timeout=5):
raise AssertionError("Missing credentials unexpectedly accepted")
except HTTPError as error:
with error:
instance = error.headers["X-Instance-Id"]
assert instance, "Missing API instance identity"
instances.add(instance)
if attempt <= 12:
assert error.code == 401, f"Attempt {attempt}: expected 401, got {error.code}"
else:
assert error.code == 429, f"Expected 429, got {error.code}"
assert 0 < int(error.headers["Retry-After"]) <= 60
assert len(instances) >= 2, "Expected requests to reach multiple API replicas"
print(f"PASS: request 13 returned 429 with Retry-After across {len(instances)} API replicas")
if __name__ == "__main__":
main()
+10
View File
@@ -0,0 +1,10 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
replica=$(docker compose ps -q api | xargs docker inspect -f '{{.Name}} {{.Config.Hostname}}' |
awk '$1=="/highload-lab7-api-2" {print $2}')
test -n "$replica"
trap 'docker compose start api >/dev/null' EXIT
TARGET_INSTANCE_ID="$replica" ./scripts/stop-one.sh
./scripts/wait_ingress.sh
make test CA_FILE=secrets/tls/lab7-ingress-ca.crt
+33
View File
@@ -0,0 +1,33 @@
#!/usr/bin/env python3
"""Use a stand started with SESSION_TTL_SECONDS between 3 and 15."""
import base64
import datetime as dt
import sys
import time
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "tests"))
from client import Client, arguments, check, run, timestamp # noqa: E402
def main():
args = arguments(__doc__)
client = Client(args.base_url, args.ca_file)
actor = client.user("expiry")
registration = actor["registration"]
credentials = base64.b64encode(
f"{registration['username']}:{registration['password']}".encode()
).decode()
session = client.call("POST", "/api/v1/auth/sessions", 201,
headers={"Authorization": "Basic " + credentials}, schema="Session")["json"]
actor["token"] = session["token"]
client.call("GET", "/api/v1/users/me", actor=actor, schema="User")
remaining = (timestamp(session["expires_at"]) - dt.datetime.now(dt.timezone.utc)).total_seconds()
check(0 < remaining <= 15, "Start rehearsal API with a short SESSION_TTL_SECONDS")
time.sleep(remaining + 0.3)
client.call("GET", "/api/v1/users/me", 401, actor=actor)
print("PASS: valid session accepted before expiry and rejected after expiry")
if __name__ == "__main__":
run(main)
+101
View File
@@ -0,0 +1,101 @@
#!/usr/bin/env python3
"""Run level-3 checks on the local rehearsal stand; write sanitized Markdown."""
import argparse
import os
import shlex
import subprocess
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
CA = "secrets/tls/lab7-ingress-ca.crt"
OUTPUT = ROOT / "evidence"
VALIDATION = {"CA_FILE": str(ROOT / CA)}
CHECKS = {
"grade3-https": ("Бизнес-API, проверка CA, logout и вложения по HTTPS", [
["make", "check"], ["make", "test", "test-security", f"CA_FILE={CA}"],
]),
"grade3-mtls": ("Допуск только роли LB на каждой реплике", [
["python3", "scripts/check_ingress_mtls.py"],
]),
"grade3-dependencies": ("TLS хранилищ, брокера и телеметрии", [
["python3", "scripts/check_grade3_tls.py"],
["./scripts/check_dependency_roles.sh"],
]),
"grade3-signals": ("Доставка логов, метрик и трейсов по защищённым связям", [
["python3", "scripts/check_grade3_signals.py"],
]),
"grade3-openbao": ("Сервисная аутентификация и scoped policy OpenBao", [
["python3", "scripts/check_bao_roles.py"],
]),
"grade3-runtime": ("Изоляция, порты, лимиты и локальное размещение секретов", [
["python3", "scripts/check_grade3_runtime.py"],
]),
"grade3-http-security": ("Лимиты, MIME, чужие вложения и неверный TLS hostname", [
["python3", "scripts/check_grade3_http.py", "--ca-file", CA],
]),
"grade3-resilience": ("Сохранность после перезапуска хранилищ и отказ API", [
["sleep", "60"],
["make", "test-persistence", "ACTION_SCRIPT=scripts/restart.sh", f"CA_FILE={CA}"],
["make", "test-failover", "ACTION_SCRIPT=scripts/stop-one.sh", f"CA_FILE={CA}"],
["docker", "compose", "start", "api"], ["./scripts/wait_ingress.sh"],
["sleep", "60"],
["make", "test-failover", "ACTION_SCRIPT=scripts/stop-one.sh", f"CA_FILE={CA}"],
["docker", "compose", "start", "api"], ["./scripts/wait_ingress.sh"],
]),
"grade3-other-replica": ("Работа HTTPS API при остановленной второй реплике", [
["sleep", "60"], ["./scripts/check_other_replica.sh"], ["./scripts/wait_ingress.sh"],
]),
"grade3-login": ("Общий лимит входа по адресу клиента", [
["sleep", "60"], ["python3", "scripts/check_login_rate.py"],
]),
"grade3-expiry": ("Истечение сессии на стенде с коротким TTL", [
["sleep", "60"],
["env", "SESSION_TTL_SECONDS=5", "docker", "compose", "up", "-d", "--no-deps", "api"],
["./scripts/wait_ingress.sh"],
["python3", "scripts/check_session_expiry.py", "--ca-file", CA],
["docker", "compose", "up", "-d", "--no-deps", "api"], ["./scripts/wait_ingress.sh"],
]),
"grade3-argon2": ("Стоимость проверки хеша Argon2id на лимитах API", [
["docker", "compose", "--profile", "test", "build", "backend-tests"],
["docker", "run", "--rm", "--network", "none", "--cpus", "0.5", "--memory", "512m",
"-e", "GOMAXPROCS=1", "highload-lab7-backend-tests", "go", "test", "./backend/services",
"-run", "^$", "-bench", "^BenchmarkPasswordVerification$", "-benchtime=3x", "-count=3"],
]),
}
def main(checks=CHECKS):
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--only", nargs="+", choices=checks)
selected = parser.parse_args().only or checks
OUTPUT.mkdir(exist_ok=True)
if "grade3-resilience" in selected:
for name in ("restart", "failover"):
path = ROOT / ".memory" / f"{name}-containers.txt"
path.parent.mkdir(exist_ok=True)
path.write_text("")
VALIDATION[f"{name.upper()}_EVIDENCE_FILE"] = str(path)
passed = True
for slug in selected:
title, commands = checks[slug]
report = [f"# {title}", "", "Предпосылки: подготовленный, разблокированный учебный стенд; Docker Compose, Python и локальный CA.", ""]
for command in commands:
result = subprocess.run(command, cwd=ROOT, env=os.environ | VALIDATION,
stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True)
passed &= result.returncode == 0
report.extend([f"`{shlex.join(command)}`", "", f"Код завершения: {result.returncode}.", "",
"```text", "\n".join(line.rstrip() for line in result.stdout.splitlines()).rstrip(), "```", ""])
if slug == "grade3-resilience":
for name in ("restart", "failover"):
rows = Path(VALIDATION[f"{name.upper()}_EVIDENCE_FILE"]).read_text().strip()
report.extend([f"Состояния контейнеров ({name}):", "", "```text", rows, "```", ""])
report.extend(["Итог: " + ("все команды завершились успешно." if all(
line == "Код завершения: 0." for line in report if line.startswith("Код завершения:")
) else "есть команды с ошибками."), ""])
(OUTPUT / f"{slug}.md").write_text("\n".join(report), encoding="utf-8")
print(f"Recorded {slug}", flush=True)
raise SystemExit(0 if passed else 1)
if __name__ == "__main__":
main()
+35
View File
@@ -0,0 +1,35 @@
"""Collect evidence for the level-4 criteria on the rehearsal stack."""
from collect_grade3 import main
CA = "secrets/tls/lab7-ingress-ca.crt"
CHECKS = {
"grade4-mtls": ("Проверка ролей RabbitMQ и Collector", [
["python3", "scripts/check_grade4_broker.py"],
["python3", "scripts/check_grade3_tls.py"],
["python3", "scripts/check_grade3_signals.py"],
]),
"grade4-login": ("Rehash в PostgreSQL и бюджет защиты входа", [
["docker", "compose", "--profile", "test", "build", "backend-tests"],
["sleep", "60"], ["python3", "scripts/check_grade4_login.py"],
]),
"grade4-lifecycle": ("Ротация секрета и leaf, отзыв, expiry и seal", [
["python3", "scripts/check_grade4_lifecycle.py"],
]),
"grade4-hardening": ("Права контейнеров, лимиты и эксплуатационные границы", [
["python3", "scripts/check_grade3_runtime.py"],
]),
"grade4-audit": ("Сетевой аудит, ACL и отсутствие контрольного секрета в телеметрии", [
["python3", "scripts/check_grade3_tls.py"],
["python3", "scripts/check_grade4_broker.py"],
["./scripts/check_dependency_roles.sh"],
["python3", "scripts/check_bao_roles.py"],
["sleep", "60"], ["python3", "scripts/check_grade3_http.py", "--ca-file", CA],
["python3", "scripts/check_grade4_telemetry.py"],
]),
"grade4-scan": ("Сканирование зависимостей и образов", [
["./scripts/scan_grade4.sh"],
]),
}
if __name__ == "__main__":
main(CHECKS)
+66
View File
@@ -0,0 +1,66 @@
#!/usr/bin/env python3
"""Configure an initialized, unsealed local OpenBao using an operator token."""
import argparse
import getpass
import json
import ssl
from pathlib import Path
from urllib.error import HTTPError
from urllib.request import Request, urlopen
ROOT = Path(__file__).resolve().parents[1]
SECRETS = ROOT / "secrets"
TLS = SECRETS / "tls"
def main():
context = ssl.create_default_context(cafile=str(TLS / "lab7-ca.crt"))
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--operator-file", type=Path)
args = parser.parse_args()
token = (json.loads(args.operator_file.read_text())["root_token"]
if args.operator_file else getpass.getpass("OpenBao operator token: "))
def request(path, body=None):
data = None if body is None else json.dumps(body).encode()
req = Request(
"https://localhost:8200/v1/" + path,
data=data,
headers={"Content-Type": "application/json", "X-Vault-Token": token},
)
try:
with urlopen(req, context=context, timeout=5) as response:
return json.load(response) if response.status != 204 else {}
except HTTPError as error:
raise SystemExit(f"OpenBao {path}: HTTP {error.code}")
if "cert/" not in request("sys/auth"):
request("sys/auth/cert", {"type": "cert"})
if "messenger/" not in request("sys/mounts"):
request("sys/mounts/messenger", {"type": "kv", "options": {"version": "2"}})
for role in ("api", "worker", "dispatcher", "cleanup"):
request(f"sys/policies/acl/{role}", {
"policy": f'path "messenger/data/{role}" {{ capabilities = ["read"] }}',
})
request(f"auth/cert/certs/{role}", {
"certificate": (TLS / f"bao-{role}.crt").read_text(),
"token_policies": [role],
"token_no_default_policy": True,
"token_ttl": "15m",
"token_max_ttl": "15m",
})
names = {"database_url": f"database_{role}_url"}
if role == "api":
names["cache_url"] = "cache_tls_url"
if role != "dispatcher":
for key in ("access_key_id", "secret_access_key"):
names[f"s3_{key}"] = f"s3_{role}_{key}"
values = {key: (SECRETS / name).read_text().strip() for key, name in names.items()}
request(f"messenger/data/{role}", {"data": values})
print(f"Configured OpenBao role {role}")
if __name__ == "__main__":
main()
+105
View File
@@ -0,0 +1,105 @@
import os
import secrets
from pathlib import Path
from urllib.parse import quote
ROOT = Path(__file__).resolve().parents[1]
def read_values(path):
values = {}
if path.exists():
for line in path.read_text(encoding="utf-8").splitlines():
line = line.strip()
if line and not line.startswith("#") and "=" in line:
key, value = line.split("=", 1)
values[key.strip()] = value.strip().strip("\"'")
return values
def main():
path = ROOT / ".env"
existing = read_values(path)
defaults = read_values(ROOT / ".env.example")
defaults.update(
POSTGRES_DB="messenger",
POSTGRES_USER="messenger",
POSTGRES_PASSWORD=secrets.token_hex(24),
)
defaults.update(API_HTTPS_PORT="8443", S3_ENDPOINT="https://s3:9000",
OTEL_EXPORTER_OTLP_ENDPOINT="https://otel-collector:4318", UPLOAD_CLEANUP_GRACE_SECONDS="600", UPLOAD_TMP_DIR="/tmp/uploads",
S3_PUBLIC_ENDPOINT="https://localhost:9000",
S3_CORS_ORIGINS="http://127.0.0.1:5173,http://localhost:5173")
values = defaults | existing
for name, lower, upper in (
("APP_PORT", 1, 65535),
("API_REPLICAS", 2, 32),
("SESSION_TTL_SECONDS", 1, 31536000),
("IDEMPOTENCY_TTL_SECONDS", 1, 31536000),
):
try:
valid = lower <= int(values[name]) <= upper
except (ValueError, KeyError):
valid = False
if not valid:
raise SystemExit(f"{name} must be between {lower} and {upper}")
for name in (
"POSTGRES_DB",
"POSTGRES_USER",
"POSTGRES_PASSWORD",
"APP_HOST",
"APP_ENV",
):
if not values[name]:
raise SystemExit(f"{name} must not be empty")
# Existing values, including credentials, are never replaced.
missing = {key: value for key, value in values.items() if key not in existing}
if missing:
prefix = "\n" if path.exists() and path.stat().st_size else ""
with path.open("a", encoding="utf-8") as file:
file.write(
prefix + "".join(f"{key}={value}\n" for key, value in missing.items())
)
os.chmod(path, 0o600)
secret_dir = ROOT / "secrets"
secret_dir.mkdir(mode=0o700, exist_ok=True)
def create_secret(name, value):
target = secret_dir / name
if not target.exists():
target.write_text(value + "\n", encoding="utf-8")
target.chmod(0o644)
return target.read_text(encoding="utf-8").strip()
create_secret("grafana_admin_password", secrets.token_hex(24))
database_url = "postgres://{}:{}@db:5432/{}?sslmode=verify-full&sslrootcert=/run/tls/ca.crt".format(
quote(values["POSTGRES_USER"], safe=""),
quote(values["POSTGRES_PASSWORD"], safe=""),
quote(values["POSTGRES_DB"], safe=""),
)
create_secret("broker_user", "messenger")
create_secret("broker_password", secrets.token_hex(32))
redis_password = create_secret("redis_password", secrets.token_hex(32))
cache_url = f"rediss://messenger:{redis_password}@redis:6379/0"
create_secret("redis_acl", "user default off\nuser messenger on >" + redis_password
+ " ~messenger:* +ping +get +set +del +incr +expire +ttl +eval +evalsha +hello +client|setinfo")
for name, content in (
("database_tls_url", database_url),
("cache_tls_url", cache_url),
):
create_secret(name, content)
create_secret("s3_access_key_id", secrets.token_hex(12))
create_secret("s3_secret_access_key", secrets.token_hex(32))
database = quote(values["POSTGRES_DB"], safe="")
for role in ("api", "worker", "dispatcher", "cleanup"):
password = create_secret(f"database_{role}_password", secrets.token_hex(24))
create_secret(f"database_{role}_url",
f"postgres://messenger_{role}:{password}@db:5432/{database}"
"?sslmode=verify-full&sslrootcert=/run/tls/ca.crt")
if role != "dispatcher":
create_secret(f"s3_{role}_access_key_id", secrets.token_hex(12))
create_secret(f"s3_{role}_secret_access_key", secrets.token_hex(32))
print("Local connection files ready; existing values and files preserved.")
if __name__ == "__main__":
main()
+13
View File
@@ -0,0 +1,13 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
docker compose up -d --wait db redis broker s3
docker compose run --rm migrate
docker compose exec -T db sh -ec '
for role in api worker dispatcher cleanup; do
export "${role}_password=$(cat "/run/secrets/database_${role}_password")"
done
psql -X -v ON_ERROR_STOP=1 -U "$POSTGRES_USER" "$POSTGRES_DB" -f -
' < infra/postgres-roles.sql
docker compose exec -T broker sh /bootstrap.sh
docker compose run --rm s3-bootstrap
+15
View File
@@ -0,0 +1,15 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
if [ -n "${RESTART_EVIDENCE_FILE:-}" ]; then
docker compose ps -q db redis s3 broker api | xargs docker inspect \
-f '{{.Name}} {{.State.StartedAt}}' > "$RESTART_EVIDENCE_FILE"
fi
docker compose --project-name highload-lab7 --env-file .env -f compose.yml restart db redis s3 broker >/dev/null
docker compose start --wait --wait-timeout 45 db redis s3 broker >/dev/null
docker compose --project-name highload-lab7 --env-file .env -f compose.yml restart api upload-cleanup worker dispatcher >/dev/null
./scripts/wait_ingress.sh
if [ -n "${RESTART_EVIDENCE_FILE:-}" ]; then
docker compose ps -q db redis s3 broker api | xargs docker inspect \
-f '{{.Name}} {{.State.StartedAt}}' >> "$RESTART_EVIDENCE_FILE"
fi
+26
View File
@@ -0,0 +1,26 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
cache=highload-lab7-trivy-cache
trivy=aquasec/trivy:0.69.3
mkdir -p .memory/scans
docker run --rm -v "$cache:/root/.cache/trivy" -v "$PWD/src:/src:ro" "$trivy" \
fs --timeout 20m --no-progress --quiet --scanners vuln --format json /src > .memory/scans/dependencies.raw
mv .memory/scans/dependencies.raw .memory/scans/dependencies.json
docker compose ps -aq | xargs docker inspect --format '{{.Image}} {{.Config.Image}}' |
sort -u | while read -r image configured; do
archive=$(mktemp)
trap 'rm -f "$archive"' EXIT HUP INT TERM
if ! docker image inspect "$image" >/dev/null 2>&1; then
echo "Container image $image is untagged/unavailable; scanning configured image $configured"
image=$(docker image inspect --format '{{.Id}}' "$configured")
fi
printf 'Scanning %s (%s)\n' "$configured" "$image"
docker image save "$image" -o "$archive"
docker run --rm -v "$cache:/root/.cache/trivy" -v "$archive:/scan.tar:ro" "$trivy" \
image --timeout 15m --no-progress --quiet --input /scan.tar --scanners vuln --format json > ".memory/scans/image-$image.raw"
mv ".memory/scans/image-$image.raw" ".memory/scans/image-$image.json"
rm -f "$archive"
trap - EXIT HUP INT TERM
done
python3 scripts/summarize_grade4_scan.py .memory/scans
+19
View File
@@ -0,0 +1,19 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
: "${TARGET_INSTANCE_ID:?Set TARGET_INSTANCE_ID to the X-Instance-Id of an API replica}"
for container in $(docker compose --project-name highload-lab7 --env-file .env -f compose.yml ps -q api); do
hostname=$(docker inspect --format '{{.Config.Hostname}}' "$container")
if [ "$hostname" = "$TARGET_INSTANCE_ID" ]; then
# Only containers from this Compose project's api service are eligible.
docker stop --timeout 8 "$container" >/dev/null
if [ -n "${FAILOVER_EVIDENCE_FILE:-}" ]; then
docker inspect -f '{{.Name}} {{.State.Status}}' "$container" >> "$FAILOVER_EVIDENCE_FILE"
fi
printf 'Stopped API instance %s\n' "$hostname"
exit 0
fi
done
printf 'No running API replica matches TARGET_INSTANCE_ID\n' >&2
exit 1
+15
View File
@@ -0,0 +1,15 @@
"""Print a compact inventory from raw Trivy JSON."""
import collections
import json
import sys
from pathlib import Path
for path in sorted(Path(sys.argv[1]).glob("*.json")):
report = json.loads(path.read_text())
matches = {}
for target in report.get("Results", []):
for entry in target.get("Vulnerabilities") or []:
key = entry["VulnerabilityID"], entry["PkgName"], entry["InstalledVersion"]
matches[key] = entry
counts = collections.Counter(entry["Severity"] for entry in matches.values())
print(path.name, dict(sorted(counts.items())))
+13
View File
@@ -0,0 +1,13 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
port=$(docker compose port haproxy 8443)
for attempt in $(seq 1 30); do
if curl --silent --fail --max-time 2 --cacert secrets/tls/lab7-ingress-ca.crt \
"https://localhost:${port##*:}/health/ready" >/dev/null; then
exit 0
fi
sleep 1
done
printf 'HTTPS ingress did not become ready\n' >&2
exit 1
+38 -9
View File
@@ -1,19 +1,48 @@
# Модель угроз — заполнить
# Модель угроз мессенджера
## Активы и акторы
TODO: пользовательские credentials/сессии, сообщения/вложения, service identities, CA/secrets, telemetry. Акторы: участник чата, посторонний пользователь, внешний клиент, скомпрометированный сервис, оператор. Опишите доступ атакующего и границы учебного стенда.
- Пароли и их Argon2id-хеши, Bearer-токены и записи сессий. Владение действующим токеном даёт права соответствующего пользователя.
- Сообщения, состав чатов, метаданные и байты вложений, включая staging и производные изображения. PostgreSQL хранит бизнес-состояние, S3 — файлы, RabbitMQ — доставку фоновых заданий, Redis — сессии и счётчики входа.
- Ключи CA и сервисов, учётные данные хранилищ и токены OpenBao. Отдельно защищаются телеметрия и её доступность, поскольку поддельные сигналы могут скрыть отказ, а утечка логов — раскрыть секреты.
Внешний клиент может формировать произвольные HTTP-запросы к ingress. Аутентифицированный посторонний знает или угадывает идентификаторы ресурсов в чужом чате. Участник чата может загружать специально подготовленные файлы. При компрометации контейнера атакующий получает доступ к его файлам, учётным данным и доступным ему сетям. Оператор управляет хостом, Docker, сертификатами и хранилищами. Компрометация хоста, Docker daemon или ключа CA выходит за границы защиты приложения. Стенд развёрнут на одном хосте и проверяет отказ отдельной API-реплики.
## Границы доверия
TODO: схема и ссылка на заполненную contracts/trust-matrix.md; публичные/внутренние интерфейсы, процессы, секреты и их выдача.
Клиент обращается к HAProxy по HTTPS. HAProxy подключается к API по mTLS, после чего приложение независимо проверяет пользовательскую сессию и членство в чате. API, worker, dispatcher и служебные задачи пересекают отдельную границу при доступе к данным. Collector принимает сигналы от приложений, а Grafana читает их из хранилищ. Административный доступ и выдача credentials образуют отдельную границу. Текущие и целевые настройки каждой связи перечислены в [матрице доверия](../contracts/trust-matrix.md).
| Угроза и конкретный путь | Нарушаемый инвариант | Контроль | Отрицательный тест | Остаточное ограничение |
| --- | --- | --- | --- | --- |
| TODO | TODO | TODO | TODO | TODO |
На 4 минимум 6 конкретных угроз. Не заменяйте сценарии словами «SQL injection», «безопасный Docker», «mTLS включён» без пути и проверки.
| Угроза и путь | Защищаемое свойство | Контроль и проверка | Остаточное ограничение |
| --- | --- | --- | --- |
| Подмена HTTPS-узла и перехват Basic или Bearer | Клиент передаёт credentials нужному серверу | HTTPS с проверкой CA и SAN. `make test-security CA_FILE=secrets/tls/lab7-ingress-ca.crt` подтвердил отказ без доверия к CA. Неверное имя проверяется в `scripts/check_grade3_http.py`. | Токен, украденный на клиенте, пригоден до logout или expiry. |
| Прямой запрос к API из соседнего контейнера или предъявление сертификата другого сервиса | API принимает только HAProxy | API не публикует порт. `python3 scripts/check_ingress_mtls.py` проверил обе реплики с сертификатом LB, без сертификата, с чужим CA и с чужим URI SAN от своего CA. | Кража ключа LB позволяет имитировать его до смены доверия. Пользовательские ACL продолжают действовать. |
| Чтение чужих сообщений или файлов, подмена `X-User-Id`, добавление чужого attachment ID | Ресурсы доступны участникам соответствующего чата | Bearer определяет пользователя, членство проверяется API. Публичный smoke прошёл по HTTPS. `scripts/check_grade3_http.py` проверяет metadata/content и сообщение с чужим вложением и поддельным заголовком. | Участник может скопировать легально полученные данные. |
| Повтор использования токена после logout или истечения срока | Отозванная сессия отклоняется всеми API | В Redis хранится ключ на основе SHA-256 токена с TTL, logout удаляет запись. Smoke проверил logout. Для expiry создать сессию на отдельном стенде с коротким TTL и ожидать 401 с тем же токеном после истечения. | Захваченная активная сессия имеет права пользователя. |
| Перебор пароля и нагрузка на Argon2id через чередование реплик и поддельный `X-Forwarded-For` | Ограничение входа действует независимо от реплики | HAProxy заменяет заголовок фактическим адресом, Redis атомарно ведёт общий счётчик. `python3 scripts/check_login_rate.py` подтвердил 429 и `Retry-After` на 13-й попытке. Пароли хешируются Argon2id с индивидуальной солью. | Распределённые источники имеют отдельные окна, пользователи за одним NAT делят лимит. |
| Загрузка слишком большого файла, изображения с чрезмерным числом пикселей или подменённым MIME | Ограничены память, CPU и объём входных данных | API ограничивает upload 10 МиБ и проверяет формат и размеры, обработка выполняется отдельно в worker. Проверить 413 для превышения размера/пикселей и 415 для ложного MIME, затем загрузить штатный файл и дождаться ready. | Параллельные загрузки конкурируют за общие ресурсы. |
| Компрометация worker и выход за пределы его роли | Worker имеет доступ только к ресурсам обработки | Действуют отдельные DB/S3/AMQP identities и scoped policy OpenBao. TLS, обработка в своей роли и запрет чтения чужого пути проверяются отдельно. | Компрометация затронет разрешённую область worker. |
| Поддельный OTLP ingest или чтение телеметрии соседом | Источник сигналов авторизован, хранилища доступны по ролям | Действует сетевая сегментация и mTLS с отдельными CA для ingest, export и read. Проверяется доставка в три хранилища и отказ чужого CA. | Скомпрометированное приложение может посылать ложные сигналы в пределах своей роли. |
| Утечка bootstrap credentials или остановка OpenBao | Сервис получает только свои секреты, отказ закрывает новый доступ | Cert auth выдаёт отдельный токен на 15 минут для чтения своего KV-пути. Проверяются запрет чтения соседнего пути, записи и отсутствие административных секретов в контейнерах. | Действующие подключения и уже загруженные credentials могут жить до закрытия соединения или отзыва в целевой системе. |
## Lifecycle и инциденты
TODO: bootstrap, expiry/renewal, revoke/rotation, компрометация identity, sealed OpenBao и восстановление. Указать владельца CA/unseal и способ защиты bootstrap credentials. Сопоставить scan findings с реальной применимостью и исправлением/остаточным риском.
Сейчас оператор выпускает локальный CA и leaf-сертификаты через `scripts/bootstrap_tls.sh`. CA действует 365 дней, leaf — 30 дней. Закрытый ключ CA остаётся у оператора, контейнеры получают отдельные файлы. Повторный запуск bootstrap сохраняет имеющиеся сертификаты.
OpenBao использует файловый backend на томе. Bootstrap сохраняет операторский материал на хосте с правами 0600. При эксплуатации оператор должен хранить доли ключа и административные credentials раздельно. После рестарта оператор выполняет unseal. API и worker заново получают scoped токены по своим клиентским сертификатам. При компрометации отзыв identity блокирует новую аутентификацию, отзыв токенов закрывает доступ к OpenBao, а смена выданных паролей и закрытие активных подключений ограничивают доступ к зависимостям. Проверки expiry, renewal и ротации выполняются на новых соединениях. результаты сканирования и применимость уязвимостей фиксируются в доказательствах.
## Проверки уровня «4»
- Подмена identity на брокере и Collector. Сертификат незарегистрированной роли
от доверенного CA отклоняется в новом сеансе. Негативные сценарии
воспроизводятся через `scripts/check_grade4_broker.py` и
`scripts/check_grade3_tls.py`. AMQP ACL дополнительно запрещают публикацию
воркером и чтение диспетчером.
- Отзыв OpenBao и компрометация пароля PostgreSQL. `scripts/check_grade4_lifecycle.py`
меняет leaf и пароль, проверяет отказ старым credentials на новом соединении,
истечение короткого токена и вход при sealed backend.
- Преднамеренная нагрузка на Argon2id. `scripts/check_grade4_login.py` подаёт
параллельные запросы с корректной парой имени и пароля и проверяет ограничение
до запуска лишней работы.
- Утечка credentials в наблюдаемость. `scripts/check_grade4_telemetry.py`
посылает синтетический секрет и проверяет соответствующую журнальную запись,
трейсы и метрики за выбранный интервал.
+26
View File
@@ -0,0 +1,26 @@
.git
.gitignore
.github
.env
.env.*
!.env.example
secrets
certs
volumes
data
backups
*.key
*.pem
*.p12
*.pfx
*.jks
.venv
__pycache__
node_modules
evidence
.DS_Store
README.md
REPORT.md
COURSE.md
CONTRIBUTING.md
MAINTAINERS.md
+39
View File
@@ -0,0 +1,39 @@
FROM golang:1.22.2-bookworm AS builder
RUN apt-get update && apt-get install -y --no-install-recommends webp && rm -rf /var/lib/apt/lists/*
WORKDIR /src
COPY go.mod go.sum ./
RUN go mod download
COPY backend ./backend
RUN CGO_ENABLED=0 \
GOOS=linux \
go build \
-trimpath \
-ldflags="-s -w" \
-o /out/messenger \
./backend
FROM debian:12.15-slim
RUN apt-get update && apt-get install -y --no-install-recommends webp && rm -rf /var/lib/apt/lists/* \
&& groupadd --system app \
&& useradd --system --gid app --no-create-home app
WORKDIR /app
COPY --from=builder /out/messenger ./messenger
ENV APP_HOST=0.0.0.0
ENV APP_PORT=8080
EXPOSE 8080
USER app:app
CMD ["./messenger"]
+155
View File
@@ -0,0 +1,155 @@
package app
import (
"context"
"encoding/json"
"errors"
"fmt"
"log"
"net"
"net/http"
"os"
"os/signal"
"runtime"
"syscall"
"time"
"messenger/src/backend/handlers"
"messenger/src/backend/repositories"
"messenger/src/backend/services"
"messenger/src/backend/telemetry"
)
func serverAddress() string {
host := os.Getenv("APP_HOST")
if host == "" {
host = "0.0.0.0"
}
port := os.Getenv("APP_PORT")
if port == "" {
port = "8080"
}
return net.JoinHostPort(host, port)
}
func RunHTTP() error {
tlsConfig, err := ingressTLS()
if err != nil {
return err
}
if _, err := repositories.UploadCleanupGrace(); err != nil {
return err
}
ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
db, err := repositories.OpenDatabase(ctx)
cancel()
if err != nil {
return fmt.Errorf("database initialization failed: %w", err)
}
defer db.Close()
// Opt-in experiment telemetry goes to container logs, never a public port.
if os.Getenv("BENCH_POOL_STATS") == "1" {
go func() {
for range time.Tick(time.Second) {
s := db.Stat()
data, _ := json.Marshal(map[string]any{
"time": time.Now().UTC(), "acquired": s.AcquiredConns(),
"total": s.TotalConns(), "max": s.MaxConns(),
"gomaxprocs": runtime.GOMAXPROCS(0), "goroutines": runtime.NumGoroutine(),
"acquire_count": s.AcquireCount(), "acquire_duration_ns": s.AcquireDuration().Nanoseconds(),
"empty_acquire_count": s.EmptyAcquireCount(), "canceled_acquire_count": s.CanceledAcquireCount(),
})
log.Printf("benchmark_pool %s", data)
}
}()
}
redisClient, err := repositories.RedisClientFromEnvironment()
if err != nil {
return err
}
defer redisClient.Close()
sessionStore := repositories.NewSessionStore(redisClient, "messenger:session:")
userStore := repositories.NewUserStore(db)
chatStore := repositories.NewChatStore(db)
idempotencyTTL, err := repositories.IdempotencyTTL()
if err != nil {
return err
}
messageStore := repositories.NewMessageStoreWithTTL(db, idempotencyTTL)
ttl, err := services.SessionTTL()
if err != nil {
return err
}
userService := services.NewUserService(userStore)
sessionService := services.NewSessionService(userService, sessionStore, ttl)
chatService := services.NewChatService(chatStore)
messageService := services.NewMessageService(messageStore)
objects, err := repositories.NewObjectStoreFromEnvironment()
if err != nil {
return err
}
attachmentService := services.NewAttachmentService(repositories.NewAttachmentStore(db), objects)
mux := handlers.NewRouter(services.NewHealthService(repositories.NewHealthStore(db), sessionStore, objects), userService, sessionService, chatService, messageService, attachmentService)
signals, stopTelemetry, err := telemetry.Start("messenger-api")
if err != nil {
return fmt.Errorf("telemetry initialization failed: %w", err)
}
defer stopTelemetry()
if err := signals.ObservePool(db); err != nil {
return err
}
server := &http.Server{
Addr: serverAddress(),
TLSConfig: tlsConfig,
Handler: signals.HTTP(mux),
ReadHeaderTimeout: 5 * time.Second,
ReadTimeout: 30 * time.Second,
WriteTimeout: 30 * time.Second,
IdleTimeout: 60 * time.Second,
}
shutdownContext, stop := signal.NotifyContext(
context.Background(),
os.Interrupt,
syscall.SIGTERM,
)
defer stop()
serverErrors := make(chan error, 1)
go func() {
log.Printf("server is listening on %s", server.Addr)
if err := server.ListenAndServeTLS(os.Getenv("TLS_CERT_FILE"), os.Getenv("TLS_KEY_FILE")); err != nil &&
!errors.Is(err, http.ErrServerClosed) {
serverErrors <- err
}
}()
select {
case err := <-serverErrors:
return err
case <-shutdownContext.Done():
log.Printf("shutdown signal received")
handlers.BeginShutdown()
shutdownDeadline, cancel := context.WithTimeout(
context.Background(),
10*time.Second,
)
defer cancel()
if err := server.Shutdown(shutdownDeadline); err != nil {
log.Printf("graceful shutdown failed: %v", err)
return nil
}
log.Printf("server stopped gracefully")
}
return nil
}
+32
View File
@@ -0,0 +1,32 @@
package app
import (
"crypto/tls"
"crypto/x509"
"errors"
"os"
)
func ingressTLS() (*tls.Config, error) {
ca, err := os.ReadFile(os.Getenv("TLS_CA_FILE"))
if err != nil {
return nil, err
}
roots := x509.NewCertPool()
if !roots.AppendCertsFromPEM(ca) {
return nil, errors.New("invalid ingress client CA")
}
return &tls.Config{
MinVersion: tls.VersionTLS12,
ClientAuth: tls.RequireAndVerifyClientCert,
ClientCAs: roots,
VerifyConnection: func(state tls.ConnectionState) error {
for _, identity := range state.PeerCertificates[0].URIs {
if identity.String() == "urn:lab7:load-balancer" {
return nil
}
}
return errors.New("client certificate does not identify the load balancer")
},
}, nil
}
+111
View File
@@ -0,0 +1,111 @@
package commands
import (
"bytes"
"crypto/tls"
"encoding/json"
"errors"
"fmt"
"net/http"
"net/url"
"os"
"path/filepath"
"time"
)
var secretFiles = map[string]string{
"database_url": "DATABASE_URL_FILE",
"cache_url": "CACHE_URL_FILE",
"broker_url": "BROKER_URL_FILE",
"s3_access_key_id": "S3_ACCESS_KEY_ID_FILE",
"s3_secret_access_key": "S3_SECRET_ACCESS_KEY_FILE",
}
func loadBaoSecrets() error {
role := os.Getenv("BAO_ROLE")
address, err := url.Parse(os.Getenv("BAO_ADDR"))
if err != nil || address.Scheme != "https" || address.Host == "" {
return errors.New("invalid OpenBao HTTPS address")
}
cert, err := tls.LoadX509KeyPair("/run/bao/client.crt", "/run/bao/client.key")
if err != nil {
return fmt.Errorf("load OpenBao service identity: %w", err)
}
transport := http.DefaultTransport.(*http.Transport).Clone()
transport.TLSClientConfig = &tls.Config{MinVersion: tls.VersionTLS12, Certificates: []tls.Certificate{cert}}
defer transport.CloseIdleConnections()
client := &http.Client{
Transport: transport,
Timeout: 5 * time.Second,
CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse },
}
request := func(method, path string, body []byte, token string, result any) error {
req, err := http.NewRequest(method, address.String()+path, bytes.NewReader(body))
if err != nil {
return err
}
req.Header.Set("Content-Type", "application/json")
if token != "" {
req.Header.Set("X-Vault-Token", token)
}
response, err := client.Do(req)
if err != nil {
return errors.New("OpenBao connection failed")
}
defer response.Body.Close()
if response.StatusCode != http.StatusOK {
return fmt.Errorf("OpenBao returned HTTP %d", response.StatusCode)
}
return json.NewDecoder(response.Body).Decode(result)
}
var login struct {
Auth struct {
Token string `json:"client_token"`
}
}
body, err := json.Marshal(map[string]string{"name": role})
if err != nil {
return err
}
if err = request("POST", "/v1/auth/cert/login", body, "", &login); err != nil {
return err
}
if login.Auth.Token == "" {
return errors.New("OpenBao returned an empty token")
}
var secret struct {
Data struct{ Data map[string]string }
}
if err = request("GET", "/v1/messenger/data/"+url.PathEscape(role), nil, login.Auth.Token, &secret); err != nil {
return err
}
for key, env := range secretFiles {
if os.Getenv(env) == "" {
continue
}
value := secret.Data.Data[key]
if value == "" {
return fmt.Errorf("missing %s in OpenBao", key)
}
dir := "/run/runtime-secrets"
path := filepath.Join(dir, key)
tmp, err := os.CreateTemp(dir, ".secret-")
if err != nil {
return err
}
_, writeErr := tmp.WriteString(value)
closeErr := tmp.Close()
if err := errors.Join(writeErr, closeErr); err != nil {
os.Remove(tmp.Name())
return err
}
if err := os.Rename(tmp.Name(), path); err != nil {
os.Remove(tmp.Name())
return err
}
if err := os.Setenv(env, path); err != nil {
return err
}
}
return nil
}
+64
View File
@@ -0,0 +1,64 @@
package commands
import (
"context"
"encoding/json"
"flag"
"log"
"os"
"os/signal"
"syscall"
"time"
"messenger/src/backend/repositories"
)
func runUploadCleanup(args []string) error {
flags := flag.NewFlagSet("attachments-cleanup", flag.ContinueOnError)
loop := flags.Bool("loop", false, "run every 60 seconds")
dryRun := flags.Bool("dry-run", false, "report without deleting anything")
if err := flags.Parse(args); err != nil {
return err
}
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
defer stop()
init, cancel := context.WithTimeout(ctx, 20*time.Second)
db, err := repositories.OpenDatabase(init)
cancel()
if err != nil {
return err
}
defer db.Close()
objects, err := repositories.NewObjectStoreFromEnvironment()
if err != nil {
return err
}
store := repositories.NewAttachmentStore(db)
for {
pass, cancel := context.WithTimeout(ctx, 30*time.Second)
result, err := store.CleanupUploads(pass, *dryRun, objects.Delete)
direct, directErr := store.CleanupDirectUploads(pass, *dryRun, objects.Delete)
jobs, jobsErr := store.CleanupJobs(pass, 24*time.Hour, *dryRun, objects.Delete)
cancel()
if encodeErr := json.NewEncoder(os.Stdout).Encode(map[string]any{"originals": result, "direct": direct, "job_staging": jobs}); encodeErr != nil {
return encodeErr
}
if !*loop {
if err == nil {
err = directErr
if err == nil {
err = jobsErr
}
}
return err
}
if err != nil || directErr != nil || jobsErr != nil {
log.Print("upload cleanup pass failed; journal retained for retry")
}
select {
case <-ctx.Done():
return nil
case <-time.After(60 * time.Second):
}
}
}
+76
View File
@@ -0,0 +1,76 @@
package commands
import (
"context"
"encoding/json"
"errors"
"os"
"strconv"
"time"
"github.com/google/uuid"
"messenger/src/backend/repositories"
)
func runJobAdmin(args []string) error {
if len(args) < 1 {
return errors.New("usage: jobs list-failed | replay JOB_ID REASON | cleanup-staging JOB_ID | upgrade ATTACHMENT_ID 2 REASON")
}
ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
defer cancel()
db, err := repositories.OpenDatabase(ctx)
if err != nil {
return errors.New("database unavailable")
}
defer db.Close()
store := repositories.NewAttachmentStore(db)
if args[0] == "list-failed" && len(args) == 1 {
jobs, e := store.FailedJobs(ctx)
if e != nil {
return errors.New("cannot read failed registry")
}
return json.NewEncoder(os.Stdout).Encode(jobs)
}
if len(args) < 2 {
return errors.New("job id required")
}
if _, err = uuid.Parse(args[1]); err != nil {
return errors.New("invalid job id")
}
objects, err := repositories.NewObjectStoreFromEnvironment()
if err != nil {
return err
}
var changed bool
switch {
case args[0] == "upgrade" && len(args) == 4:
version, e := strconv.Atoi(args[2])
if e != nil {
return errors.New("invalid pipeline version")
}
changed, err = store.UpgradePipeline(ctx, args[1], version, args[3], func(ctx context.Context, key string) error {
r, e := objects.Get(ctx, key)
if e == nil {
r.Close()
}
return e
})
case args[0] == "replay" && len(args) == 3:
changed, err = store.ReplayJob(ctx, args[1], args[2], func(ctx context.Context, key string) error {
r, e := objects.Get(ctx, key)
if e == nil {
r.Close()
}
return e
})
case args[0] == "cleanup-staging" && len(args) == 2:
changed, err = store.CleanupJobStaging(ctx, args[1], 24*time.Hour, false, objects.Delete)
default:
return errors.New("usage: jobs list-failed | replay JOB_ID REASON | cleanup-staging JOB_ID | upgrade ATTACHMENT_ID 2 REASON")
}
if err != nil {
return err
}
return json.NewEncoder(os.Stdout).Encode(map[string]bool{"changed": changed})
}
+84
View File
@@ -0,0 +1,84 @@
package commands
import (
"context"
"errors"
"log"
"net"
"net/http"
"os"
"time"
"messenger/src/backend/app"
"messenger/src/backend/repositories"
"messenger/src/backend/workers"
)
// Run dispatches the existing executable modes; the binary and CLI stay the same.
func Run(args []string) error {
if os.Getenv("BAO_ROLE") != "" && (len(args) == 0 || args[0] != "image-process") {
if err := loadBaoSecrets(); err != nil {
return err
}
}
if len(args) == 0 {
if environment := os.Getenv("APP_ENV"); environment != "" {
log.SetPrefix("[" + environment + "] ")
}
return app.RunHTTP()
}
switch args[0] {
case "image-process":
return workers.RunImageChild(args[1:])
case "jobs":
return runJobAdmin(args[1:])
case "worker", "dispatcher", "broker-publish":
return workers.RunPipeline(args[0], args[1:])
}
if environment := os.Getenv("APP_ENV"); environment != "" {
log.SetPrefix("[" + environment + "] ")
}
switch {
case args[0] == "s3-bootstrap" && len(args) == 1:
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
if err := repositories.BootstrapS3(ctx); err != nil {
return err
}
log.Print("private S3 bucket ready")
return nil
case args[0] == "attachments-cleanup":
return runUploadCleanup(args[1:])
case args[0] == "healthcheck" && len(args) == 1:
return healthcheck()
case args[0] == "migrate" && len(args) == 2 && (args[1] == "up" || args[1] == "down"):
url, err := repositories.DatabaseURL()
if err != nil {
return err
}
if err = repositories.RunMigrations(url, args[1]); err != nil {
return err
}
log.Printf("migration %s complete", args[1])
return nil
default:
return errors.New("usage: messenger migrate up|down")
}
}
func healthcheck() error {
client := http.Client{Timeout: 2 * time.Second}
port := os.Getenv("APP_PORT")
if port == "" {
port = "8080"
}
response, err := client.Get("http://" + net.JoinHostPort("127.0.0.1", port) + "/health/ready")
if err != nil {
return err
}
response.Body.Close()
if response.StatusCode != http.StatusOK {
return errors.New("API is not ready")
}
return nil
}
+165
View File
@@ -0,0 +1,165 @@
package handlers
import (
"context"
"errors"
"github.com/google/uuid"
"io"
"log"
"messenger/src/backend/repositories"
"messenger/src/backend/services"
"net/http"
"net/url"
"strconv"
"strings"
"time"
)
func uploadAttachmentHandler(chats *services.ChatService, attachments *services.AttachmentService) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
requestID := newRequestID()
w.Header().Set("X-Request-Id", requestID)
if r.Method != http.MethodPost {
writeError(w, 405, "method_not_allowed", "method not allowed", requestID)
return
}
userID, ok := requestUserID(w, r, requestID)
if !ok {
return
}
raw := strings.TrimSuffix(strings.TrimPrefix(r.URL.Path, "/api/v1/chats/"), "/attachments")
chatID, err := uuid.Parse(raw)
if err != nil {
writeError(w, 400, "invalid_request", "invalid chat id", requestID)
return
}
if _, err = chats.Get(r.Context(), userID, chatID.String()); err != nil {
writeServiceError(w, err, requestID)
return
}
query, err := urlQuery(r)
if err != nil {
writeServiceError(w, err, requestID)
return
}
filenames := query["filename"]
if len(filenames) != 1 {
writeError(w, 400, "invalid_request", "exactly one filename is required", requestID)
return
}
if err = services.ValidateFilename(filenames[0]); err != nil {
writeServiceError(w, err, requestID)
return
}
if r.ContentLength > services.MaxUploadBytes {
writeServiceError(w, services.ErrPayloadTooLarge, requestID)
return
}
r.Body = http.MaxBytesReader(w, r.Body, services.MaxUploadBytes)
ctx, cancel := context.WithTimeout(r.Context(), 25*time.Second)
defer cancel()
attachment, err := attachments.Upload(ctx, userID, chatID.String(), filenames[0], r.Header.Get("Content-Type"), r.Body)
var tooLarge *http.MaxBytesError
if errors.As(err, &tooLarge) {
err = services.ErrPayloadTooLarge
}
if err != nil {
writeServiceError(w, err, requestID)
return
}
writeJSON(w, 202, attachment)
}
}
func attachmentHandler(attachments *services.AttachmentService) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
requestID := newRequestID()
w.Header().Set("X-Request-Id", requestID)
if r.Method != http.MethodGet {
writeError(w, 405, "method_not_allowed", "method not allowed", requestID)
return
}
userID, ok := requestUserID(w, r, requestID)
if !ok {
return
}
path := strings.TrimPrefix(r.URL.Path, "/api/v1/attachments/")
parts := strings.Split(path, "/")
if len(parts) > 2 || (len(parts) == 2 && parts[1] != "content") {
writeServiceError(w, services.ErrAttachmentNotFound, requestID)
return
}
id, err := uuid.Parse(parts[0])
if err != nil {
writeError(w, 400, "invalid_request", "invalid attachment id", requestID)
return
}
a, err := attachments.Get(r.Context(), id.String(), userID)
if err != nil {
writeServiceError(w, err, requestID)
return
}
if len(parts) == 1 {
writeJSON(w, 200, a)
return
}
query, err := urlQuery(r)
if err != nil {
writeServiceError(w, err, requestID)
return
}
if a.Status != "ready" {
writeServiceError(w, services.ErrConflict, requestID)
return
}
if variants, exists := query["variant"]; exists {
if len(variants) != 1 {
writeError(w, 400, "invalid_request", "invalid variant", requestID)
return
}
if variants[0] == "thumbnail" || variants[0] == "crop" || variants[0] == "optimized" {
found := false
for _, raw := range a.Variants {
if v, ok := raw.(map[string]any); ok && v["name"] == variants[0] {
a.ObjectKey = repositories.VariantObjectKey(a.ID, a.PipelineVersion, variants[0])
a.ContentType = v["content_type"].(string)
a.SizeBytes = int64(v["size_bytes"].(float64))
found = true
break
}
}
if !found {
writeServiceError(w, services.ErrAttachmentNotFound, requestID)
return
}
} else if variants[0] != "original" {
writeServiceError(w, services.ErrAttachmentNotFound, requestID)
return
}
}
ctx, cancel := context.WithTimeout(r.Context(), 25*time.Second)
defer cancel()
content, err := attachments.Content(ctx, a)
if err != nil {
writeServiceError(w, err, requestID)
return
}
defer content.Close()
w.Header().Set("Content-Type", a.ContentType)
w.Header().Set("Content-Length", strconv.FormatInt(a.SizeBytes, 10))
w.Header().Set("X-Content-Type-Options", "nosniff")
w.Header().Set("Cache-Control", "private, no-store")
w.WriteHeader(200)
if _, err = io.Copy(w, content); err != nil {
log.Printf("attachment download interrupted, request %s", requestID)
}
}
}
func urlQuery(r *http.Request) (url.Values, error) {
q, err := url.ParseQuery(r.URL.RawQuery)
if err != nil {
return nil, services.InvalidInput{Message: "invalid query"}
}
return q, nil
}
+137
View File
@@ -0,0 +1,137 @@
package handlers
import (
"net/http"
"strings"
"messenger/src/backend/services"
"github.com/google/uuid"
)
type createChatRequest struct {
Title string `json:"title"`
MemberIDs []string `json:"member_ids"`
}
func createChatHandler(chats *services.ChatService) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
requestID := newRequestID()
w.Header().Set("X-Request-Id", requestID)
if r.Method != http.MethodPost {
writeError(w, 405, "method_not_allowed", "method not allowed", requestID)
return
}
userID, ok := requestUserID(w, r, requestID)
if !ok {
return
}
if !strings.HasPrefix(r.Header.Get("Content-Type"), "application/json") {
writeError(w, 415, "unsupported_media_type", "content type must be application/json", requestID)
return
}
var request createChatRequest
if err := decodeJSONBody(r, &request); err != nil {
writeError(w, 400, "invalid_request", "invalid JSON body", requestID)
return
}
chat, err := chats.Create(r.Context(), userID, request.Title, request.MemberIDs)
if err != nil {
writeServiceError(w, err, requestID)
return
}
writeJSON(w, 201, chat)
}
}
func getChatHandler(chats *services.ChatService) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
requestID := newRequestID()
w.Header().Set("X-Request-Id", requestID)
if r.Method != http.MethodGet {
writeError(w, 405, "method_not_allowed", "method not allowed", requestID)
return
}
userID, ok := requestUserID(w, r, requestID)
if !ok {
return
}
chatID := strings.TrimPrefix(r.URL.Path, "/api/v1/chats/")
if chatID == "" || strings.Contains(chatID, "/") {
writeError(w, 400, "invalid_request", "invalid chat id", requestID)
return
}
if _, err := uuid.Parse(chatID); err != nil {
writeError(w, 400, "invalid_request", "invalid chat id", requestID)
return
}
chat, err := chats.Get(r.Context(), userID, chatID)
if err != nil {
writeServiceError(w, err, requestID)
return
}
writeJSON(w, 200, chat)
}
}
func listChatsHandler(chats *services.ChatService) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
requestID := newRequestID()
w.Header().Set("X-Request-Id", requestID)
if r.Method != http.MethodGet {
writeError(w, 405, "method_not_allowed", "method not allowed", requestID)
return
}
userID, ok := requestUserID(w, r, requestID)
if !ok {
return
}
limit, cursor, ok := pageParams(w, r, requestID)
if !ok {
return
}
page, err := chats.List(r.Context(), userID, limit, cursor)
if err != nil {
writeServiceError(w, err, requestID)
return
}
writeJSON(w, 200, page)
}
}
func chatCollectionHandler(chats *services.ChatService) http.HandlerFunc {
create := createChatHandler(chats)
list := listChatsHandler(chats)
return func(w http.ResponseWriter, r *http.Request) {
if r.Method == http.MethodGet {
list(w, r)
} else {
create(w, r)
}
}
}
func chatSubresourceHandler(chats *services.ChatService, messages *services.MessageService, attachments *services.AttachmentService) http.HandlerFunc {
get := getChatHandler(chats)
createMessage := createMessageHandler(chats, messages)
listMessages := listMessagesHandler(chats, messages)
return func(w http.ResponseWriter, r *http.Request) {
if attachments != nil && strings.Contains(r.URL.Path, "/attachment-uploads") {
directUploadHandler(chats, attachments)(w, r)
return
}
if attachments != nil && strings.HasSuffix(r.URL.Path, "/attachments") {
uploadAttachmentHandler(chats, attachments)(w, r)
return
}
if strings.HasSuffix(r.URL.Path, "/messages") {
if r.Method == http.MethodGet {
listMessages(w, r)
} else {
createMessage(w, r)
}
return
}
get(w, r)
}
}
@@ -0,0 +1,81 @@
package handlers
import (
"context"
"encoding/json"
"errors"
"io"
"net/http"
"strings"
"time"
"messenger/src/backend/services"
"github.com/google/uuid"
)
func directUploadHandler(chats *services.ChatService, attachments *services.AttachmentService) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
requestID := newRequestID()
w.Header().Set("X-Request-Id", requestID)
if r.Method != http.MethodPost {
writeError(w, 405, "method_not_allowed", "method not allowed", requestID)
return
}
user, ok := requestUserID(w, r, requestID)
if !ok {
return
}
parts := strings.Split(strings.TrimPrefix(r.URL.Path, "/api/v1/chats/"), "/")
if (len(parts) != 2 && (len(parts) != 4 || parts[3] != "complete")) || parts[1] != "attachment-uploads" {
writeError(w, 404, "not_found", "upload endpoint not found", requestID)
return
}
chat, err := uuid.Parse(parts[0])
if err != nil {
writeError(w, 400, "invalid_request", "invalid chat id", requestID)
return
}
if _, err = chats.Get(r.Context(), user, chat.String()); err != nil {
writeServiceError(w, err, requestID)
return
}
ctx, cancel := context.WithTimeout(r.Context(), 25*time.Second)
defer cancel()
if len(parts) == 4 {
id, err := uuid.Parse(parts[2])
if err != nil {
writeError(w, 400, "invalid_request", "invalid upload id", requestID)
return
}
a, err := attachments.CompleteDirect(ctx, user, chat.String(), id.String())
if err != nil {
writeServiceError(w, err, requestID)
return
}
writeJSON(w, 202, a)
return
}
r.Body = http.MaxBytesReader(w, r.Body, 4096)
var input services.DirectUploadInput
decoder := json.NewDecoder(r.Body)
decoder.DisallowUnknownFields()
err = decoder.Decode(&input)
if err == nil {
var extra any
if next := decoder.Decode(&extra); !errors.Is(next, io.EOF) {
err = errors.New("expected one JSON object")
}
}
if err != nil {
writeError(w, 400, "invalid_request", "invalid upload JSON (maximum 4 KiB)", requestID)
return
}
result, err := attachments.InitDirect(ctx, user, chat.String(), input)
if err != nil {
writeServiceError(w, err, requestID)
return
}
writeJSON(w, 201, result)
}
}
+163
View File
@@ -0,0 +1,163 @@
package handlers
import (
"context"
"crypto/rand"
"encoding/hex"
"encoding/json"
"errors"
"log"
"net/http"
"strconv"
"time"
"messenger/src/backend/repositories"
"messenger/src/backend/services"
)
type healthResponse struct {
Status string `json:"status"`
}
type apiError struct {
Code string `json:"code"`
Message string `json:"message"`
RequestID string `json:"request_id"`
}
type errorResponse struct {
Error apiError `json:"error"`
}
func newRequestID() string {
bytes := make([]byte, 16)
if _, err := rand.Read(bytes); err != nil {
return "request-id-generation-failed"
}
return hex.EncodeToString(bytes)
}
func writeError(w http.ResponseWriter, status int, code, message, requestID string) {
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(status)
response := errorResponse{Error: apiError{Code: code, Message: message, RequestID: requestID}}
if err := json.NewEncoder(w).Encode(response); err != nil {
log.Printf("failed to encode error response: %v", err)
}
}
func writeServiceError(w http.ResponseWriter, err error, requestID string) {
var upload repositories.UploadError
if errors.As(err, &upload) {
writeError(w, upload.Status, upload.Code, upload.Message, requestID)
return
}
var invalid services.InvalidInput
switch {
case errors.As(err, &invalid):
writeError(w, 400, "invalid_request", invalid.Message, requestID)
case errors.Is(err, services.ErrPayloadTooLarge):
writeError(w, 413, "payload_too_large", "image exceeds upload limits", requestID)
case errors.Is(err, services.ErrUnsupportedMedia):
writeError(w, 415, "unsupported_media_type", "a valid PNG or JPEG matching Content-Type is required", requestID)
case errors.Is(err, services.ErrAttachmentNotFound):
writeError(w, 404, "not_found", "attachment not found", requestID)
case errors.Is(err, services.ErrConflict):
writeError(w, 409, "conflict", "request conflicts with an existing resource", requestID)
case errors.Is(err, services.ErrUnknownUser):
writeError(w, 401, "unauthorized", "unknown user", requestID)
case errors.Is(err, services.ErrChatNotFound):
writeError(w, 404, "not_found", "chat not found", requestID)
default:
writeError(w, 503, "unavailable", "dependency unavailable", requestID)
}
}
func writeJSON(w http.ResponseWriter, status int, value any) {
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(status)
if err := json.NewEncoder(w).Encode(value); err != nil {
log.Printf("failed to encode response: %v", err)
}
}
func requestUserID(w http.ResponseWriter, r *http.Request, requestID string) (string, bool) {
id := currentUserID(r)
if id == "" {
writeError(w, 401, "unauthorized", "valid Bearer session is required", requestID)
return "", false
}
return id, true
}
func pageParams(w http.ResponseWriter, r *http.Request, requestID string) (int, string, bool) {
limit := 50
if values, exists := r.URL.Query()["limit"]; exists {
if len(values) != 1 || values[0] == "" {
writeError(w, 400, "invalid_request", "invalid limit", requestID)
return 0, "", false
}
parsed, err := strconv.Atoi(values[0])
if err != nil {
writeError(w, 400, "invalid_request", "limit must be between 1 and 100", requestID)
return 0, "", false
}
limit = parsed
}
var cursor string
if values, exists := r.URL.Query()["cursor"]; exists {
if len(values) != 1 || values[0] == "" {
writeError(w, 400, "invalid_request", "invalid cursor", requestID)
return 0, "", false
}
cursor = values[0]
}
return limit, cursor, true
}
func healthHandler(w http.ResponseWriter, r *http.Request) {
requestID := newRequestID()
w.Header().Set("X-Request-Id", requestID)
if r.Method != http.MethodGet {
writeError(w, 405, "method_not_allowed", "method not allowed", requestID)
return
}
writeJSON(w, 200, healthResponse{"ok"})
}
func readyHandler(health *services.HealthService) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
requestID := newRequestID()
w.Header().Set("X-Request-Id", requestID)
if r.Method != http.MethodGet {
writeError(w, 405, "method_not_allowed", "method not allowed", requestID)
return
}
// Readiness must finish before HAProxy's one-second check read timeout.
ctx, cancel := context.WithTimeout(r.Context(), 500*time.Millisecond)
defer cancel()
if err := health.Ready(ctx); err != nil {
writeError(w, 503, "unavailable", "dependency unavailable", requestID)
return
}
writeJSON(w, 200, healthResponse{"ok"})
}
}
func NewRouter(health *services.HealthService, users *services.UserService, sessions *services.SessionService, chats *services.ChatService, messages *services.MessageService, attachmentServices ...*services.AttachmentService) http.Handler {
var attachments *services.AttachmentService
if len(attachmentServices) > 0 {
attachments = attachmentServices[0]
}
mux := http.NewServeMux()
mux.HandleFunc("/health/live", healthHandler)
mux.HandleFunc("/health/ready", readyHandler(health))
mux.Handle("/api/v1/users", createUserHandler(users))
mux.Handle("/api/v1/auth/sessions", sessionHandler(sessions))
mux.Handle("/api/v1/auth/sessions/current", authRequired(sessions, logoutHandler(sessions)))
mux.Handle("/api/v1/users/me", authRequired(sessions, currentUserHandler(users)))
mux.Handle("/api/v1/chats", authRequired(sessions, chatCollectionHandler(chats)))
mux.Handle("/api/v1/chats/", authRequired(sessions, chatSubresourceHandler(chats, messages, attachments)))
if attachments != nil {
mux.Handle("/api/v1/attachments/", authRequired(sessions, attachmentHandler(attachments)))
}
return withInstance(mux)
}
+34
View File
@@ -0,0 +1,34 @@
package handlers
import (
"net/http"
"os"
"sync/atomic"
"github.com/google/uuid"
)
// Docker's hostname lets the operator map an API response to its container.
var instanceID = func() string {
id, err := os.Hostname()
if err != nil || id == "" {
return uuid.NewString()
}
return id
}()
var draining atomic.Bool
func BeginShutdown() { draining.Store(true) }
func withInstance(next http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("X-Instance-Id", instanceID)
w.Header().Set("X-Request-Id", newRequestID())
if draining.Load() && r.URL.Path != "/health/live" {
writeError(w, 503, "unavailable", "instance is shutting down", w.Header().Get("X-Request-Id"))
return
}
next.ServeHTTP(w, r)
})
}
+127
View File
@@ -0,0 +1,127 @@
package handlers
import (
"encoding/json"
"net/http"
"strings"
"github.com/google/uuid"
"messenger/src/backend/services"
)
type createMessageRequest struct {
Text *string `json:"text"`
AttachmentIDs json.RawMessage `json:"attachment_ids"`
}
func messageChatID(path string) (string, bool) {
const prefix = "/api/v1/chats/"
const suffix = "/messages"
if !strings.HasPrefix(path, prefix) || !strings.HasSuffix(path, suffix) {
return "", false
}
chatID := strings.TrimSuffix(strings.TrimPrefix(path, prefix), suffix)
if chatID == "" || strings.Contains(chatID, "/") {
return "", false
}
return chatID, true
}
func createMessageHandler(chats *services.ChatService, messages *services.MessageService) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
requestID := newRequestID()
w.Header().Set("X-Request-Id", requestID)
if r.Method != http.MethodPost {
writeError(w, 405, "method_not_allowed", "method not allowed", requestID)
return
}
userID, ok := requestUserID(w, r, requestID)
if !ok {
return
}
chatID, ok := messageChatID(r.URL.Path)
if !ok {
writeError(w, 400, "invalid_request", "invalid chat id", requestID)
return
}
if _, err := uuid.Parse(chatID); err != nil {
writeError(w, 400, "invalid_request", "invalid chat id", requestID)
return
}
chat, err := chats.Get(r.Context(), userID, chatID)
if err != nil {
writeServiceError(w, err, requestID)
return
}
if !strings.HasPrefix(r.Header.Get("Content-Type"), "application/json") {
writeError(w, 415, "unsupported_media_type", "content type must be application/json", requestID)
return
}
var request createMessageRequest
if err := decodeJSONBody(r, &request); err != nil {
writeError(w, 400, "invalid_request", "invalid JSON body", requestID)
return
}
ids := []string{}
if request.Text == nil {
writeError(w, 400, "invalid_request", "text must be a string", requestID)
return
}
if len(request.AttachmentIDs) > 0 {
if err := json.Unmarshal(request.AttachmentIDs, &ids); err != nil || ids == nil {
writeError(w, 400, "invalid_request", "attachment_ids must be an array of UUIDs", requestID)
return
}
}
keys := r.Header.Values("Idempotency-Key")
if len(keys) != 1 {
writeError(w, 400, "invalid_request", "exactly one Idempotency-Key is required", requestID)
return
}
message, err := messages.CreateIdempotent(r.Context(), userID, chat, *request.Text, keys[0], ids)
if err != nil {
writeServiceError(w, err, requestID)
return
}
writeJSON(w, 201, message)
}
}
func listMessagesHandler(chats *services.ChatService, messages *services.MessageService) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
requestID := newRequestID()
w.Header().Set("X-Request-Id", requestID)
if r.Method != http.MethodGet {
writeError(w, 405, "method_not_allowed", "method not allowed", requestID)
return
}
userID, ok := requestUserID(w, r, requestID)
if !ok {
return
}
chatID, ok := messageChatID(r.URL.Path)
if !ok {
writeError(w, 400, "invalid_request", "invalid chat id", requestID)
return
}
if _, err := uuid.Parse(chatID); err != nil {
writeError(w, 400, "invalid_request", "invalid chat id", requestID)
return
}
chat, err := chats.Get(r.Context(), userID, chatID)
if err != nil {
writeServiceError(w, err, requestID)
return
}
limit, cursor, ok := pageParams(w, r, requestID)
if !ok {
return
}
page, err := messages.List(r.Context(), userID, chat, limit, cursor)
if err != nil {
writeServiceError(w, err, requestID)
return
}
writeJSON(w, 200, page)
}
}
+123
View File
@@ -0,0 +1,123 @@
package handlers
import (
"context"
"errors"
"net"
"net/http"
"strconv"
"strings"
"time"
"messenger/src/backend/services"
)
type authenticatedUserKey struct{}
func currentUserID(r *http.Request) string {
id, _ := r.Context().Value(authenticatedUserKey{}).(string)
return id
}
func authRequired(sessions *services.SessionService, next http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
authorization := r.Header.Get("Authorization")
if !strings.HasPrefix(authorization, "Bearer ") || strings.TrimSpace(strings.TrimPrefix(authorization, "Bearer ")) == "" {
unauthorized(w, "Bearer", newRequestID())
return
}
user, err := sessions.User(r.Context(), strings.TrimPrefix(authorization, "Bearer "))
if errors.Is(err, services.ErrInvalidSession) {
unauthorized(w, "Bearer", newRequestID())
return
}
if err != nil {
requestID := newRequestID()
w.Header().Set("X-Request-Id", requestID)
writeError(w, 503, "unavailable", "dependency unavailable", requestID)
return
}
next.ServeHTTP(w, r.WithContext(context.WithValue(r.Context(), authenticatedUserKey{}, user.ID)))
})
}
func unauthorized(w http.ResponseWriter, scheme, requestID string) {
w.Header().Set("X-Request-Id", requestID)
if scheme == "Basic" {
w.Header().Set("WWW-Authenticate", `Basic realm="messenger", charset="UTF-8"`)
} else {
w.Header().Set("WWW-Authenticate", `Bearer realm="messenger"`)
}
writeError(w, 401, "unauthorized", "invalid credentials", requestID)
}
func sessionHandler(sessions *services.SessionService) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
requestID := newRequestID()
w.Header().Set("X-Request-Id", requestID)
if r.Method != http.MethodPost {
writeError(w, 405, "method_not_allowed", "method not allowed", requestID)
return
}
// HAProxy overwrites this header and is the only mTLS identity allowed to reach the API.
if r.TLS != nil {
source := net.ParseIP(r.Header.Get("X-Forwarded-For"))
if source == nil {
writeError(w, 503, "unavailable", "missing ingress client address", requestID)
return
}
retryAfter, err := sessions.LoginRetryAfter(r.Context(), source.String())
if err != nil {
writeError(w, 503, "unavailable", "dependency unavailable", requestID)
return
}
if retryAfter > 0 {
w.Header().Set("Retry-After", strconv.Itoa(retryAfter))
writeError(w, 429, "rate_limited", "too many login attempts", requestID)
return
}
}
username, password, ok := r.BasicAuth()
if !ok || r.ContentLength > 0 {
unauthorized(w, "Basic", requestID)
return
}
user, token, expires, valid, err := sessions.Login(r.Context(), username, password)
if errors.Is(err, services.ErrPasswordBusy) {
w.Header().Set("Retry-After", "1")
writeError(w, 429, "rate_limited", "login capacity exhausted", requestID)
return
}
if err != nil {
writeError(w, 503, "unavailable", "dependency unavailable", requestID)
return
}
if !valid {
unauthorized(w, "Basic", requestID)
return
}
writeJSON(w, 201, struct {
Token string `json:"token"`
TokenType string `json:"token_type"`
ExpiresAt time.Time `json:"expires_at"`
User any `json:"user"`
}{token, "Bearer", expires, user})
}
}
func logoutHandler(sessions *services.SessionService) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
requestID := newRequestID()
w.Header().Set("X-Request-Id", requestID)
if r.Method != http.MethodDelete {
writeError(w, 405, "method_not_allowed", "method not allowed", requestID)
return
}
token := strings.TrimPrefix(r.Header.Get("Authorization"), "Bearer ")
if err := sessions.Revoke(r.Context(), token); err != nil {
writeError(w, 503, "unavailable", "dependency unavailable", requestID)
return
}
w.WriteHeader(204)
}
}
+103
View File
@@ -0,0 +1,103 @@
package handlers
import (
"encoding/json"
"fmt"
"io"
"net/http"
"reflect"
"strings"
"messenger/src/backend/services"
)
type createUserRequest struct {
Username string `json:"username"`
DisplayName string `json:"display_name"`
Password string `json:"password"`
}
func decodeJSONBody(r *http.Request, destination any) error {
decoder := json.NewDecoder(r.Body)
var body json.RawMessage
if err := decoder.Decode(&body); err != nil {
return err
}
var extra any
if err := decoder.Decode(&extra); err != io.EOF {
if err != nil {
return err
}
return fmt.Errorf("more than one JSON value")
}
var fields map[string]json.RawMessage
if err := json.Unmarshal(body, &fields); err != nil {
return err
}
if fields == nil {
return fmt.Errorf("JSON body must be an object")
}
// encoding/json matches struct fields case-insensitively, even with
// DisallowUnknownFields. Check exact contract names before decoding values.
typeOfRequest := reflect.TypeOf(destination).Elem()
allowed := make(map[string]bool, typeOfRequest.NumField())
for i := 0; i < typeOfRequest.NumField(); i++ {
name := strings.Split(typeOfRequest.Field(i).Tag.Get("json"), ",")[0]
if name != "" && name != "-" {
allowed[name] = true
}
}
for name := range fields {
if !allowed[name] {
return fmt.Errorf("unknown JSON field %q", name)
}
}
return json.Unmarshal(body, destination)
}
func createUserHandler(users *services.UserService) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
requestID := newRequestID()
w.Header().Set("X-Request-Id", requestID)
if r.Method != http.MethodPost {
writeError(w, 405, "method_not_allowed", "method not allowed", requestID)
return
}
if !strings.HasPrefix(r.Header.Get("Content-Type"), "application/json") {
writeError(w, 415, "unsupported_media_type", "content type must be application/json", requestID)
return
}
var request createUserRequest
if err := decodeJSONBody(r, &request); err != nil {
writeError(w, 400, "invalid_request", "invalid JSON body", requestID)
return
}
user, err := users.Create(r.Context(), request.Username, request.DisplayName, request.Password)
if err != nil {
writeServiceError(w, err, requestID)
return
}
writeJSON(w, 201, user)
}
}
func currentUserHandler(users *services.UserService) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
requestID := newRequestID()
w.Header().Set("X-Request-Id", requestID)
if r.Method != http.MethodGet {
writeError(w, 405, "method_not_allowed", "method not allowed", requestID)
return
}
userID, ok := requestUserID(w, r, requestID)
if !ok {
return
}
user, err := users.Get(r.Context(), userID)
if err != nil {
writeServiceError(w, err, requestID)
return
}
writeJSON(w, 200, user)
}
}
+64
View File
@@ -0,0 +1,64 @@
package faults
import (
"context"
"log"
"os"
"path/filepath"
"time"
"github.com/google/uuid"
)
// Only an operator-mounted directory enables faults; clients cannot set them.
// Rename makes each injected crash one-shot across automatic container restarts.
func Take(point, id string) bool {
directory := os.Getenv("PIPELINE_FAULT_DIR")
if directory == "" {
return false
}
if _, err := uuid.Parse(id); err != nil {
return false
}
name := filepath.Join(directory, point+"."+id)
return os.Rename(name, name+".hit") == nil
}
func Crash(point, id string) {
if Take(point, id) {
log.Printf("injected pipeline crash: %s, attachment %s", point, id)
os.Exit(86)
}
}
// Observe records a test-only branch without exposing an API or a lease token.
func Observe(point, id string) {
directory := os.Getenv("PIPELINE_FAULT_DIR")
if directory == "" {
return
}
if _, err := uuid.Parse(id); err != nil {
return
}
_ = os.WriteFile(filepath.Join(directory, point+"."+id+".observed"), []byte("observed\n"), 0644)
}
// Pause is operator-only and ends on release or context expiry.
func Pause(ctx context.Context, point, id string) {
if !Take(point, id) {
return
}
release := filepath.Join(os.Getenv("PIPELINE_FAULT_DIR"), point+"."+id+".release")
ticker := time.NewTicker(50 * time.Millisecond)
defer ticker.Stop()
for {
select {
case <-ctx.Done():
return
case <-ticker.C:
if _, err := os.Stat(release); err == nil {
_ = os.WriteFile(release+".observed", []byte("released\n"), 0644)
return
}
}
}
}
+17
View File
@@ -0,0 +1,17 @@
package main
import (
"log"
"os"
"messenger/src/backend/commands"
)
func main() {
if err := commands.Run(os.Args[1:]); err != nil {
if len(os.Args) > 1 && (os.Args[1] == "image-process" || os.Args[1] == "healthcheck") {
os.Exit(1)
}
log.Fatal(err)
}
}
+48
View File
@@ -0,0 +1,48 @@
package repositories
import (
"context"
"encoding/json"
"github.com/jackc/pgx/v5/pgxpool"
"time"
)
type Attachment struct {
ID string `json:"id"`
ChatID string `json:"chat_id"`
OwnerID string `json:"owner_id"`
Filename string `json:"filename"`
ContentType string `json:"content_type"`
SizeBytes int64 `json:"size_bytes"`
SHA256 string `json:"sha256"`
Status string `json:"status"`
Variants []any `json:"variants"`
ErrorCode *string `json:"error_code"`
CreatedAt time.Time `json:"created_at"`
UpdatedAt time.Time `json:"updated_at"`
ObjectKey string `json:"-"`
PipelineVersion int `json:"-"`
}
type AttachmentStore struct{ db *pgxpool.Pool }
func NewAttachmentStore(db *pgxpool.Pool) *AttachmentStore { return &AttachmentStore{db: db} }
func (s *AttachmentStore) Create(ctx context.Context, a Attachment) error {
ctx, cancel := boundedQuery(ctx)
defer cancel()
_, err := s.db.Exec(ctx, `INSERT INTO attachments(id,chat_id,owner_id,filename,object_key,content_type,size_bytes,sha256,created_at,updated_at) VALUES($1,$2,$3,$4,$5,$6,$7,$8,$9,$10)`, a.ID, a.ChatID, a.OwnerID, a.Filename, a.ObjectKey, a.ContentType, a.SizeBytes, a.SHA256, a.CreatedAt, a.UpdatedAt)
return err
}
func (s *AttachmentStore) GetForMember(ctx context.Context, id, userID string) (Attachment, error) {
ctx, cancel := boundedQuery(ctx)
defer cancel()
a := Attachment{Variants: []any{}}
var variants []byte
err := s.db.QueryRow(ctx, `SELECT a.id,a.chat_id,a.owner_id,a.filename,a.object_key,a.content_type,a.size_bytes,a.sha256,a.created_at,a.updated_at,a.status,a.variants,a.error_code,a.published_pipeline_version FROM attachments a JOIN chat_members cm ON cm.chat_id=a.chat_id WHERE a.id=$1 AND cm.user_id=$2`, id, userID).Scan(&a.ID, &a.ChatID, &a.OwnerID, &a.Filename, &a.ObjectKey, &a.ContentType, &a.SizeBytes, &a.SHA256, &a.CreatedAt, &a.UpdatedAt, &a.Status, &variants, &a.ErrorCode, &a.PipelineVersion)
if err == nil {
err = json.Unmarshal(variants, &a.Variants)
}
a.CreatedAt = a.CreatedAt.UTC()
a.UpdatedAt = a.UpdatedAt.UTC()
return a, err
}
+94
View File
@@ -0,0 +1,94 @@
package repositories
import (
"context"
"crypto/tls"
"os"
"errors"
"messenger/src/backend/telemetry"
"net"
"time"
amqp "github.com/rabbitmq/amqp091-go"
)
const jobQueue = "attachments.v1"
type JobBroker struct {
connection *amqp.Connection
channel *amqp.Channel
returned chan amqp.Return
}
func NewJobBroker(ctx context.Context, address string) (*JobBroker, error) {
config := amqp.Config{
Heartbeat: 10 * time.Second,
Dial: func(network, address string) (net.Conn, error) {
return (&net.Dialer{Timeout: 5 * time.Second}).DialContext(ctx, network, address)
},
}
if os.Getenv("BROKER_MTLS") == "1" {
cert, err := tls.LoadX509KeyPair("/run/broker/client.crt", "/run/broker/client.key")
if err != nil {
return nil, errors.New("RabbitMQ client certificate unavailable")
}
config.TLSClientConfig = &tls.Config{MinVersion: tls.VersionTLS12, Certificates: []tls.Certificate{cert}}
config.SASL = []amqp.Authentication{&amqp.ExternalAuth{}}
}
connection, err := amqp.DialConfig(address, config)
if err != nil {
return nil, errors.New("RabbitMQ connection unavailable")
}
channel, err := connection.Channel()
if err != nil {
connection.Close()
return nil, errors.New("RabbitMQ channel unavailable")
}
b := &JobBroker{connection: connection, channel: channel, returned: make(chan amqp.Return, 1)}
// Queue metadata and message bodies both survive a broker restart.
_, err = channel.QueueDeclare(jobQueue, true, false, false, false, nil)
if err == nil {
err = channel.Confirm(false)
}
if err == nil {
err = channel.Qos(1, 0, false)
}
if err != nil {
b.Close()
return nil, errors.New("RabbitMQ queue initialization failed")
}
channel.NotifyReturn(b.returned)
return b, nil
}
func (b *JobBroker) Close() { b.channel.Close(); b.connection.Close() }
func (b *JobBroker) Publish(ctx context.Context, body []byte) error {
// mandatory catches an unroutable message; confirms alone do not prove routing.
headers := amqp.Table{}
for key, value := range telemetry.TraceHeaders(ctx) {
headers[key] = value
}
confirmation, err := b.channel.PublishWithDeferredConfirmWithContext(ctx, "", jobQueue, true, false, amqp.Publishing{
DeliveryMode: amqp.Persistent, ContentType: "application/json", Body: body, Headers: headers,
})
if err != nil {
return errors.New("RabbitMQ publication failed")
}
confirmed, err := confirmation.WaitContext(ctx)
if err != nil || !confirmed {
return errors.New("RabbitMQ publication not confirmed")
}
// basic.return precedes basic.ack on the channel; NotifyReturn is buffered.
select {
case <-b.returned:
return errors.New("RabbitMQ message was not routed")
default:
return nil
}
}
func (b *JobBroker) Consume() (<-chan amqp.Delivery, error) {
return b.channel.Consume(jobQueue, "", false, false, false, false, nil)
}
+129
View File
@@ -0,0 +1,129 @@
package repositories
import (
"context"
"errors"
"time"
"github.com/google/uuid"
"github.com/jackc/pgx/v5/pgxpool"
)
type Chat struct {
ID string `json:"id"`
Title string `json:"title"`
MemberIDs []string `json:"member_ids"`
CreatedAt time.Time `json:"created_at"`
}
type ChatStore struct {
db *pgxpool.Pool
}
func NewChatStore(db *pgxpool.Pool) *ChatStore {
return &ChatStore{db: db}
}
var ErrUnknownMember = errors.New("unknown member")
func (s *ChatStore) Create(ctx context.Context, title string, memberIDs []string) (Chat, error) {
ctx, cancel := boundedQuery(ctx)
defer cancel()
membersCopy := append([]string(nil), memberIDs...)
chat := Chat{
ID: uuid.New().String(),
Title: title,
MemberIDs: membersCopy,
CreatedAt: time.Now().UTC().Truncate(time.Microsecond),
}
tx, err := s.db.Begin(ctx)
if err != nil {
return Chat{}, err
}
defer tx.Rollback(ctx)
for _, id := range membersCopy {
var exists bool
if err = tx.QueryRow(ctx, "SELECT EXISTS(SELECT 1 FROM users WHERE id=$1)", id).Scan(&exists); err != nil {
return Chat{}, err
}
if !exists {
return Chat{}, ErrUnknownMember
}
}
if _, err = tx.Exec(ctx, "INSERT INTO chats(id,title,created_at) VALUES($1,$2,$3)", chat.ID, title, chat.CreatedAt); err != nil {
return Chat{}, err
}
for _, id := range membersCopy {
if _, err = tx.Exec(ctx, "INSERT INTO chat_members(chat_id,user_id) VALUES($1,$2)", chat.ID, id); err != nil {
return Chat{}, err
}
}
if err = tx.Commit(ctx); err != nil {
return Chat{}, err
}
return chat, nil
}
func (s *ChatStore) GetByID(ctx context.Context, id string) (Chat, error) {
ctx, cancel := boundedQuery(ctx)
defer cancel()
var chat Chat
err := s.db.QueryRow(ctx, "SELECT id,title,created_at FROM chats WHERE id=$1", id).Scan(&chat.ID, &chat.Title, &chat.CreatedAt)
if err != nil {
return Chat{}, err
}
rows, err := s.db.Query(ctx, "SELECT user_id FROM chat_members WHERE chat_id=$1 ORDER BY user_id", id)
if err != nil {
return Chat{}, err
}
defer rows.Close()
chat.MemberIDs = []string{}
for rows.Next() {
var member string
if err = rows.Scan(&member); err != nil {
return Chat{}, err
}
chat.MemberIDs = append(chat.MemberIDs, member)
}
return chat, rows.Err()
}
func (s *ChatStore) ListPageForUser(ctx context.Context, userID string, cursor *ChatCursor, limit int) ([]Chat, bool, error) {
ctx, cancel := boundedQuery(ctx)
defer cancel()
afterTime := time.Time{}
afterID := "00000000-0000-0000-0000-000000000000"
if cursor != nil {
afterTime = cursor.CreatedAt
afterID = cursor.ChatID
}
rows, err := s.db.Query(ctx, `SELECT c.id,c.title,c.created_at FROM chats c JOIN chat_members m ON m.chat_id=c.id WHERE m.user_id=$1 AND (c.created_at,c.id)>($2,$3::uuid) ORDER BY c.created_at,c.id LIMIT $4`, userID, afterTime, afterID, limit+1)
if err != nil {
return nil, false, err
}
defer rows.Close()
chats := []Chat{}
for rows.Next() {
var c Chat
if err = rows.Scan(&c.ID, &c.Title, &c.CreatedAt); err != nil {
return nil, false, err
}
chats = append(chats, c)
}
if err = rows.Err(); err != nil {
return nil, false, err
}
more := len(chats) > limit
if more {
chats = chats[:limit]
}
for i := range chats {
chat, err := s.GetByID(ctx, chats[i].ID)
if err != nil {
return nil, false, err
}
chats[i] = chat
}
return chats, more, nil
}
@@ -0,0 +1,20 @@
package repositories
import (
"errors"
"os"
"strconv"
"time"
)
func UploadCleanupGrace() (time.Duration, error) {
value := os.Getenv("UPLOAD_CLEANUP_GRACE_SECONDS")
if value == "" {
value = "600"
}
seconds, err := strconv.Atoi(value)
if err != nil || seconds < 120 || seconds > 86400 {
return 0, errors.New("UPLOAD_CLEANUP_GRACE_SECONDS must be between 120 and 86400")
}
return time.Duration(seconds) * time.Second, nil
}
+47
View File
@@ -0,0 +1,47 @@
package repositories
import (
"errors"
"os"
"strconv"
"strings"
"time"
"github.com/redis/go-redis/v9"
)
func RedisClientFromEnvironment() (*redis.Client, error) {
if path := os.Getenv("CACHE_URL_FILE"); path != "" {
data, err := os.ReadFile(path)
if err != nil {
return nil, err
}
options, err := redis.ParseURL(strings.TrimSpace(string(data)))
if err != nil {
return nil, errors.New("invalid Redis URL in CACHE_URL_FILE")
}
options.PoolSize = 10
options.MaxRetries = -1
options.DialTimeout = time.Second
options.ReadTimeout = time.Second
options.WriteTimeout = time.Second
return redis.NewClient(options), nil
}
address := os.Getenv("REDIS_ADDR")
if address == "" {
address = "redis:6379"
}
return NewRedisClient(address), nil
}
func IdempotencyTTL() (time.Duration, error) {
value := os.Getenv("IDEMPOTENCY_TTL_SECONDS")
if value == "" {
value = "86400"
}
seconds, err := strconv.ParseInt(value, 10, 64)
if err != nil || seconds < 1 || seconds > 31536000 {
return 0, errors.New("IDEMPOTENCY_TTL_SECONDS must be between 1 and 31536000")
}
return time.Duration(seconds) * time.Second, nil
}
+15
View File
@@ -0,0 +1,15 @@
package repositories
import "time"
type ChatCursor struct {
UserID string `json:"user_id"`
CreatedAt time.Time `json:"created_at"`
ChatID string `json:"chat_id"`
}
type MessageCursor struct {
ChatID string `json:"chat_id"`
CreatedAt time.Time `json:"created_at"`
MessageID string `json:"message_id"`
}
+100
View File
@@ -0,0 +1,100 @@
package repositories
import (
"context"
"database/sql"
"embed"
"errors"
"os"
"strconv"
"strings"
"time"
"messenger/src/backend/telemetry"
"github.com/golang-migrate/migrate/v4"
pgxmigrate "github.com/golang-migrate/migrate/v4/database/pgx/v5"
"github.com/golang-migrate/migrate/v4/source/iofs"
"github.com/jackc/pgx/v5/pgxpool"
_ "github.com/jackc/pgx/v5/stdlib"
)
//go:embed migrations/*.sql
var migrationFiles embed.FS
func DatabaseURL() (string, error) {
if path := os.Getenv("DATABASE_URL_FILE"); path != "" {
data, err := os.ReadFile(path)
if err != nil {
return "", err
}
return strings.TrimSpace(string(data)), nil
}
if url := os.Getenv("DATABASE_URL"); url != "" {
return url, nil
}
return "", errors.New("DATABASE_URL_FILE or DATABASE_URL is required")
}
func OpenDatabase(ctx context.Context) (*pgxpool.Pool, error) {
url, err := DatabaseURL()
if err != nil {
return nil, err
}
cfg, err := pgxpool.ParseConfig(url)
if err != nil {
return nil, err
}
cfg.ConnConfig.Tracer = telemetry.PostgresTracer{}
cfg.MaxConns = 10
if value := os.Getenv("DATABASE_MAX_CONNS"); value != "" {
maxConns, parseErr := strconv.Atoi(value)
if parseErr != nil || maxConns < 1 || maxConns > 100 {
return nil, errors.New("DATABASE_MAX_CONNS must be between 1 and 100")
}
cfg.MaxConns = int32(maxConns)
}
cfg.ConnConfig.ConnectTimeout = 5 * time.Second
db, err := pgxpool.NewWithConfig(ctx, cfg)
if err != nil {
return nil, err
}
if err = db.Ping(ctx); err != nil {
db.Close()
return nil, err
}
return db, nil
}
// RunMigrations uses versioned up/down SQL embedded in the API binary.
// "down" rolls back exactly one version and is an explicit operator command.
func RunMigrations(url, direction string) error {
source, err := iofs.New(migrationFiles, "migrations")
if err != nil {
return err
}
sqlDB, err := sql.Open("pgx", url)
if err != nil {
return err
}
driver, err := pgxmigrate.WithInstance(sqlDB, &pgxmigrate.Config{})
if err != nil {
sqlDB.Close()
return err
}
m, err := migrate.NewWithInstance("iofs", source, "pgx5", driver)
if err != nil {
sqlDB.Close()
return err
}
defer m.Close()
if direction == "down" {
err = m.Steps(-1)
} else {
err = m.Up()
}
if errors.Is(err, migrate.ErrNoChange) {
return nil
}
return err
}
+106
View File
@@ -0,0 +1,106 @@
package repositories
import (
"context"
"errors"
"fmt"
"time"
"github.com/google/uuid"
"github.com/jackc/pgx/v5"
)
func (s *AttachmentStore) CleanupDirectUploads(ctx context.Context, dryRun bool, remove func(context.Context, string) error) (CleanupResult, error) {
result := CleanupResult{DryRun: dryRun}
rows, err := s.db.Query(ctx, `SELECT id::text FROM direct_uploads WHERE cleanup_after<=clock_timestamp() ORDER BY cleanup_after,id LIMIT 100`)
if err != nil {
return result, err
}
ids := []string{}
for rows.Next() {
var id string
if err = rows.Scan(&id); err != nil {
rows.Close()
return result, err
}
ids = append(ids, id)
}
err = rows.Err()
rows.Close()
if err != nil {
return result, err
}
result.Candidates = len(ids)
for _, id := range ids {
outcome, err := s.cleanupDirectUpload(ctx, id, dryRun, remove)
if err != nil {
result.Failed++
continue
}
switch outcome {
case "deleted":
result.Deleted++
case "eligible":
result.Eligible++
default:
result.Skipped++
}
}
if result.Failed > 0 {
return result, fmt.Errorf("%d direct upload cleanup operations failed", result.Failed)
}
return result, nil
}
func (s *AttachmentStore) cleanupDirectUpload(ctx context.Context, id string, dryRun bool, remove func(context.Context, string) error) (string, error) {
ctx, cancel := context.WithTimeout(ctx, 10*time.Second)
defer cancel()
tx, err := s.db.Begin(ctx)
if err != nil {
return "", err
}
defer tx.Rollback(context.Background())
var key, state string
var barrier, complete, purge, now time.Time
err = tx.QueryRow(ctx, `SELECT staging_key,state,complete_before+interval '600 seconds',complete_before,purge_after,clock_timestamp() FROM direct_uploads WHERE id=$1 AND cleanup_after<=clock_timestamp() FOR UPDATE SKIP LOCKED`, id).Scan(&key, &state, &barrier, &complete, &purge, &now)
if errors.Is(err, pgx.ErrNoRows) {
return "skipped", nil
}
if err != nil {
return "", err
}
parsed, err := uuid.Parse(id)
if err != nil || key != "staging/direct/"+parsed.String() {
return "", errors.New("invalid direct staging key")
}
if state == "pending" && now.Before(barrier) {
return "skipped", nil
}
if dryRun {
return "eligible", nil
}
if err = remove(ctx, key); err != nil {
return "", err
}
if now.After(purge) && !now.Before(barrier) {
_, err = tx.Exec(ctx, `DELETE FROM direct_uploads WHERE id=$1`, id)
} else {
if state == "pending" && !now.Before(complete) {
state = "expired"
}
// An early cleanup is not final: a still valid POST can recreate staging.
// Keep a second cleanup after the safety barrier, then retain the receipt 24h.
next := barrier
if !now.Before(barrier) {
next = purge
}
_, err = tx.Exec(ctx, `UPDATE direct_uploads SET state=$2,cleanup_after=$3 WHERE id=$1`, id, state, next)
}
if err != nil {
return "", err
}
if err = tx.Commit(ctx); err != nil {
return "", err
}
return "deleted", nil
}
+150
View File
@@ -0,0 +1,150 @@
package repositories
import (
"context"
"errors"
"time"
"github.com/jackc/pgx/v5"
)
// UploadError is also persisted for terminal validation failures, so retries agree.
type UploadError struct {
Status int
Code, Message string
}
func (e UploadError) Error() string { return e.Message }
func DirectError(code string) UploadError {
switch code {
case "upload_not_uploaded":
return UploadError{409, code, "file has not been uploaded"}
case "upload_expired":
return UploadError{409, code, "upload confirmation period has expired"}
case "upload_size_mismatch":
return UploadError{400, code, "uploaded size differs from declared size"}
case "upload_checksum_mismatch":
return UploadError{400, code, "uploaded SHA-256 differs from declared checksum"}
case "payload_too_large":
return UploadError{413, code, "image exceeds upload limits"}
default:
return UploadError{415, "unsupported_media_type", "a valid PNG or JPEG matching Content-Type is required"}
}
}
type DirectUpload struct {
ID string `json:"upload_id"`
AttachmentID string `json:"-"`
OwnerID string `json:"-"`
ChatID string `json:"-"`
Filename string `json:"-"`
ContentType string `json:"-"`
SizeBytes int64 `json:"-"`
SHA256 string `json:"-"`
StagingKey string `json:"-"`
FinalKey string `json:"-"`
State string `json:"-"`
ErrorCode *string `json:"-"`
ExpiresAt time.Time `json:"expires_at"`
CompleteBefore time.Time `json:"complete_before"`
}
const directColumns = `d.id,d.attachment_id,d.owner_id,d.chat_id,d.filename,d.content_type,d.size_bytes,d.sha256,d.staging_key,d.final_key,d.state,d.error_code,d.expires_at,d.complete_before`
func scanDirect(row pgx.Row) (DirectUpload, error) {
var u DirectUpload
err := row.Scan(&u.ID, &u.AttachmentID, &u.OwnerID, &u.ChatID, &u.Filename, &u.ContentType, &u.SizeBytes, &u.SHA256, &u.StagingKey, &u.FinalKey, &u.State, &u.ErrorCode, &u.ExpiresAt, &u.CompleteBefore)
return u, err
}
func (s *AttachmentStore) CreateDirect(ctx context.Context, u DirectUpload) (DirectUpload, error) {
ctx, cancel := boundedQuery(ctx)
defer cancel()
return scanDirect(s.db.QueryRow(ctx, `INSERT INTO direct_uploads AS d(id,attachment_id,owner_id,chat_id,filename,content_type,size_bytes,sha256,staging_key,final_key) VALUES($1,$2,$3,$4,$5,$6,$7,$8,$9,$10) RETURNING `+directColumns, u.ID, u.AttachmentID, u.OwnerID, u.ChatID, u.Filename, u.ContentType, u.SizeBytes, u.SHA256, u.StagingKey, u.FinalKey))
}
func (s *AttachmentStore) GetDirect(ctx context.Context, id, chat, user string) (DirectUpload, error) {
ctx, cancel := boundedQuery(ctx)
defer cancel()
return scanDirect(s.db.QueryRow(ctx, `SELECT `+directColumns+` FROM direct_uploads d JOIN chat_members cm ON cm.chat_id=d.chat_id AND cm.user_id=$3 WHERE d.id=$1 AND d.chat_id=$2 AND d.owner_id=$3`, id, chat, user))
}
// The session lock serializes complete across replicas. A separately committed
// reservation survives rollback/crash, while its row lock protects the final key.
func (s *AttachmentStore) CompleteDirect(ctx context.Context, id, chat, user string, validate func(context.Context, DirectUpload) error) (Attachment, error) {
var a Attachment
tx, err := s.db.Begin(ctx)
if err != nil {
return a, err
}
defer tx.Rollback(context.Background())
u, err := scanDirect(tx.QueryRow(ctx, `SELECT `+directColumns+` FROM direct_uploads d JOIN chat_members cm ON cm.chat_id=d.chat_id AND cm.user_id=$3 WHERE d.id=$1 AND d.chat_id=$2 AND d.owner_id=$3 FOR UPDATE OF d`, id, chat, user))
if err != nil {
return a, err
}
if u.StagingKey != "staging/direct/"+u.ID || u.FinalKey != "originals/"+u.AttachmentID {
return a, errors.New("invalid direct upload object keys")
}
if u.State == "ready" {
return s.GetForMember(ctx, u.AttachmentID, user)
}
if u.State == "rejected" && u.ErrorCode != nil {
return a, DirectError(*u.ErrorCode)
}
var expired bool
if err = tx.QueryRow(ctx, `SELECT clock_timestamp()>=$1`, u.CompleteBefore).Scan(&expired); err != nil {
return a, err
}
if expired || u.State == "expired" {
return a, DirectError("upload_expired")
}
grace, err := UploadCleanupGrace()
if err != nil {
return a, err
}
// ON CONFLICT handles retry after an interrupted COPY without losing tracking.
_, err = s.db.Exec(ctx, `INSERT INTO upload_attempts(id,object_key,cleanup_after) VALUES($1,$2,clock_timestamp()+$3*interval '1 second') ON CONFLICT(id) DO UPDATE SET cleanup_after=GREATEST(upload_attempts.cleanup_after,EXCLUDED.cleanup_after) WHERE upload_attempts.object_key=EXCLUDED.object_key`, u.AttachmentID, u.FinalKey, grace.Seconds())
if err != nil {
return a, err
}
var key string
if err = tx.QueryRow(ctx, `SELECT object_key FROM upload_attempts WHERE id=$1 AND cleanup_after>clock_timestamp() FOR UPDATE`, u.AttachmentID).Scan(&key); err != nil {
return a, err
}
if key != u.FinalKey {
return a, errors.New("direct upload reservation mismatch")
}
if err = validate(ctx, u); err != nil {
var invalid UploadError
if errors.As(err, &invalid) && (invalid.Status == 400 || invalid.Status == 413 || invalid.Status == 415) {
_, saveErr := tx.Exec(ctx, `UPDATE direct_uploads SET state='rejected',error_code=$2,cleanup_after=clock_timestamp(),purge_after=clock_timestamp()+interval '24 hours' WHERE id=$1`, id, invalid.Code)
if saveErr != nil {
return a, saveErr
}
if saveErr = tx.Commit(ctx); saveErr != nil {
return a, saveErr
}
}
return a, err
}
now := time.Now().UTC().Truncate(time.Microsecond)
a = Attachment{ID: u.AttachmentID, ChatID: u.ChatID, OwnerID: u.OwnerID, Filename: u.Filename, ContentType: u.ContentType, SizeBytes: u.SizeBytes, SHA256: u.SHA256, ObjectKey: u.FinalKey, Status: "queued", Variants: []any{}, CreatedAt: now, UpdatedAt: now}
_, err = tx.Exec(ctx, `INSERT INTO attachments(id,chat_id,owner_id,filename,object_key,content_type,size_bytes,sha256,created_at,updated_at) VALUES($1,$2,$3,$4,$5,$6,$7,$8,$9,$10)`, a.ID, a.ChatID, a.OwnerID, a.Filename, a.ObjectKey, a.ContentType, a.SizeBytes, a.SHA256, a.CreatedAt, a.UpdatedAt)
if err != nil {
return Attachment{}, err
}
if err = enqueueJob(ctx, tx, a, a.ObjectKey); err != nil {
return Attachment{}, err
}
if _, err = tx.Exec(ctx, `UPDATE direct_uploads SET state='ready',cleanup_after=clock_timestamp(),purge_after=clock_timestamp()+interval '24 hours' WHERE id=$1`, id); err != nil {
return Attachment{}, err
}
if _, err = tx.Exec(ctx, `DELETE FROM upload_attempts WHERE id=$1`, a.ID); err != nil {
return Attachment{}, err
}
if err = tx.Commit(ctx); err != nil {
return Attachment{}, err
}
return a, nil
}
+21
View File
@@ -0,0 +1,21 @@
package repositories
import (
"errors"
"github.com/jackc/pgx/v5"
"github.com/jackc/pgx/v5/pgconn"
)
func NotFound(err error) bool {
return errors.Is(err, pgx.ErrNoRows)
}
func CheckViolation(err error) bool {
var pgerr *pgconn.PgError
return errors.As(err, &pgerr) && pgerr.Code == "23514"
}
func UniqueViolation(err error) bool {
var pgerr *pgconn.PgError
return errors.As(err, &pgerr) && pgerr.Code == "23505"
}
+21
View File
@@ -0,0 +1,21 @@
package repositories
import (
"context"
"github.com/jackc/pgx/v5/pgxpool"
)
type HealthStore struct {
db *pgxpool.Pool
}
func NewHealthStore(db *pgxpool.Pool) *HealthStore {
return &HealthStore{db: db}
}
func (s *HealthStore) Ping(ctx context.Context) error {
ctx, cancel := boundedQuery(ctx)
defer cancel()
return s.db.Ping(ctx)
}
+95
View File
@@ -0,0 +1,95 @@
package repositories
import (
"context"
"errors"
"reflect"
"time"
"github.com/google/uuid"
"github.com/jackc/pgx/v5"
)
var ErrAttachmentNotFound = errors.New("attachment not found")
var ErrAttachmentFailed = errors.New("attachment failed")
var ErrIdempotencyConflict = errors.New("idempotency key belongs to a different message")
// CreateIdempotent commits the message and its replay record together.
func (s *MessageStore) CreateIdempotent(ctx context.Context, chatID, senderID, text, key string, attachments ...[]string) (Message, error) {
ids := []string{}
if len(attachments) > 0 && attachments[0] != nil {
ids = attachments[0]
}
ctx, cancel := boundedQuery(ctx)
defer cancel()
tx, err := s.db.Begin(ctx)
if err != nil {
return Message{}, err
}
defer tx.Rollback(context.Background())
// The transaction-scoped lock is shared by all API replicas. Hash collisions
// only serialize unrelated requests; the full scope is checked by the query.
scope := senderID + ":POST:/api/v1/chats/" + chatID + "/messages:" + key
if _, err = tx.Exec(ctx, "SELECT pg_advisory_xact_lock(hashtextextended($1, 0))", scope); err != nil {
return Message{}, err
}
// A shared row lock prevents failure publication racing with a new link.
hasFailed := false
for _, id := range ids {
var status string
err = tx.QueryRow(ctx, "SELECT status FROM attachments WHERE id=$1 AND chat_id=$2 FOR SHARE", id, chatID).Scan(&status)
if errors.Is(err, pgx.ErrNoRows) {
return Message{}, ErrAttachmentNotFound
}
if err != nil {
return Message{}, err
}
hasFailed = hasFailed || status == "failed"
}
var message Message
err = tx.QueryRow(ctx, `
SELECT m.id, m.chat_id, m.sender_id, m.text, m.created_at, ARRAY(SELECT attachment_id::text FROM message_attachments ma WHERE ma.message_id=m.id ORDER BY position)
FROM message_idempotency i JOIN messages m ON m.id = i.message_id
WHERE i.sender_id=$1 AND i.chat_id=$2 AND i.key=$3
AND i.expires_at > clock_timestamp()`, senderID, chatID, key).
Scan(&message.ID, &message.ChatID, &message.SenderID, &message.Text, &message.CreatedAt, &message.AttachmentIDs)
if err == nil {
message.CreatedAt = message.CreatedAt.UTC()
if message.Text != text || !reflect.DeepEqual(message.AttachmentIDs, ids) {
return Message{}, ErrIdempotencyConflict
}
return message, tx.Commit(ctx)
}
if !errors.Is(err, pgx.ErrNoRows) {
return Message{}, err
}
if hasFailed {
return Message{}, ErrAttachmentFailed
}
// Expiry permits key reuse; it never deletes the original message.
if _, err = tx.Exec(ctx, "DELETE FROM message_idempotency WHERE sender_id=$1 AND chat_id=$2 AND key=$3", senderID, chatID, key); err != nil {
return Message{}, err
}
message = Message{
AttachmentIDs: ids, ID: uuid.NewString(), ChatID: chatID, SenderID: senderID, Text: text,
CreatedAt: time.Now().UTC().Truncate(time.Microsecond),
}
if _, err = tx.Exec(ctx, "INSERT INTO messages(id,chat_id,sender_id,text,created_at) VALUES($1,$2,$3,$4,$5)", message.ID, chatID, senderID, text, message.CreatedAt); err != nil {
return Message{}, err
}
for position, id := range ids {
if _, err = tx.Exec(ctx, "INSERT INTO message_attachments(message_id,attachment_id,position) VALUES($1,$2,$3)", message.ID, id, position); err != nil {
return Message{}, err
}
}
if _, err = tx.Exec(ctx, `INSERT INTO message_idempotency(sender_id,chat_id,key,message_id,expires_at)
VALUES($1,$2,$3,$4,clock_timestamp() + $5 * interval '1 second')`, senderID, chatID, key, message.ID, s.idempotencyTTL.Seconds()); err != nil {
return Message{}, err
}
return message, tx.Commit(ctx)
}
+142
View File
@@ -0,0 +1,142 @@
package repositories
import (
"context"
"errors"
"github.com/jackc/pgx/v5"
"strings"
"time"
"unicode/utf8"
)
type FailedJob struct {
ID string `json:"job_id"`
AttachmentID string `json:"attachment_id"`
Code string `json:"error_code"`
Attempts int `json:"attempts"`
FailedAt time.Time `json:"failed_at"`
}
func (s *AttachmentStore) FailedJobs(ctx context.Context) ([]FailedJob, error) {
rows, err := s.db.Query(ctx, `SELECT j.id,j.attachment_id,f.error_code,f.attempts,f.failed_at FROM attachment_job_failures f JOIN attachment_jobs j ON j.id=f.job_id JOIN attachments a ON a.id=j.attachment_id WHERE a.status='failed' AND j.pipeline_version=a.pipeline_version AND j.superseded_at IS NULL ORDER BY f.failed_at DESC LIMIT 100`)
if err != nil {
return nil, err
}
defer rows.Close()
result := []FailedJob{}
for rows.Next() {
var j FailedJob
if err = rows.Scan(&j.ID, &j.AttachmentID, &j.Code, &j.Attempts, &j.FailedAt); err != nil {
return nil, err
}
result = append(result, j)
}
return result, rows.Err()
}
func (s *AttachmentStore) ReplayJob(ctx context.Context, id, reason string, exists func(context.Context, string) error) (bool, error) {
reason = strings.TrimSpace(reason)
if reason == "" || utf8.RuneCountInString(reason) > 200 {
return false, errors.New("replay reason must contain 1-200 characters")
}
tx, err := s.db.Begin(ctx)
if err != nil {
return false, err
}
defer tx.Rollback(context.Background())
var status, key, attachment string
var deleted, obsolete bool
err = tx.QueryRow(ctx, `SELECT a.status,j.staging_key,a.id,j.staging_deleted_at IS NOT NULL,j.pipeline_version<>a.pipeline_version OR j.superseded_at IS NOT NULL FROM attachment_jobs j JOIN attachments a ON a.id=j.attachment_id WHERE j.id=$1 FOR UPDATE OF j,a`, id).Scan(&status, &key, &attachment, &deleted, &obsolete)
if err != nil {
return false, err
}
if obsolete || status != "failed" {
return false, nil
}
if deleted {
return false, errors.New("staging has expired; replay unavailable")
}
if key != "staging/jobs/"+attachment && key != "originals/"+attachment {
return false, errors.New("invalid staging key")
}
if err = exists(ctx, key); err != nil {
return false, errors.New("staging unavailable; replay not scheduled")
}
if _, err = tx.Exec(ctx, `INSERT INTO attachment_job_replays(job_id,reason) VALUES($1,$2)`, id, reason); err != nil {
return false, err
}
if _, err = tx.Exec(ctx, `UPDATE attachment_jobs SET attempts=0,generation=generation+1,next_attempt_at=now(),lease_until=NULL,lease_token=NULL,published_at=NULL WHERE id=$1`, id); err != nil {
return false, err
}
if _, err = tx.Exec(ctx, `UPDATE attachments SET status='queued',error_code=NULL,updated_at=now() WHERE id=$1`, attachment); err != nil {
return false, err
}
return true, tx.Commit(ctx)
}
// Replay and cleanup lock the same rows. A ready source is removed only after
// final publication, failed sources expire after retention; queued work is safe.
func (s *AttachmentStore) CleanupJobStaging(ctx context.Context, id string, retention time.Duration, dryRun bool, remove func(context.Context, string) error) (bool, error) {
tx, err := s.db.Begin(ctx)
if err != nil {
return false, err
}
defer tx.Rollback(context.Background())
var key, attachment, status string
var eligible bool
err = tx.QueryRow(ctx, `SELECT j.staging_key,a.id,a.status,j.staging_deleted_at IS NULL AND (a.status='ready' AND jsonb_array_length(a.variants)>=3 OR a.status='failed' AND j.pipeline_version=a.pipeline_version AND COALESCE(f.failed_at,a.updated_at)<now()-$2*interval '1 second') FROM attachment_jobs j JOIN attachments a ON a.id=j.attachment_id LEFT JOIN attachment_job_failures f ON f.job_id=j.id WHERE j.id=$1 FOR UPDATE OF j,a SKIP LOCKED`, id, retention.Seconds()).Scan(&key, &attachment, &status, &eligible)
if errors.Is(err, pgx.ErrNoRows) {
return false, nil
}
if err != nil {
return false, err
}
if !eligible || key == "originals/"+attachment {
return false, nil
}
if key != "staging/jobs/"+attachment {
return false, errors.New("invalid staging key")
}
if dryRun {
return true, nil
}
if err = remove(ctx, key); err != nil {
return false, err
}
if _, err = tx.Exec(ctx, `UPDATE attachment_jobs SET staging_deleted_at=now() WHERE id=$1`, id); err != nil {
return false, err
}
return true, tx.Commit(ctx)
}
func (s *AttachmentStore) CleanupJobs(ctx context.Context, retention time.Duration, dryRun bool, remove func(context.Context, string) error) (int, error) {
rows, err := s.db.Query(ctx, `SELECT j.id FROM attachment_jobs j JOIN attachments a ON a.id=j.attachment_id LEFT JOIN attachment_job_failures f ON f.job_id=j.id WHERE j.staging_deleted_at IS NULL AND j.staging_key='staging/jobs/'||j.attachment_id::text AND (a.status='ready' AND jsonb_array_length(a.variants)>=3 OR a.status='failed' AND j.pipeline_version=a.pipeline_version AND COALESCE(f.failed_at,a.updated_at)<now()-$1*interval '1 second') ORDER BY j.created_at LIMIT 100`, retention.Seconds())
if err != nil {
return 0, err
}
ids := []string{}
for rows.Next() {
var id string
if err = rows.Scan(&id); err != nil {
rows.Close()
return 0, err
}
ids = append(ids, id)
}
err = rows.Err()
rows.Close()
if err != nil {
return 0, err
}
count := 0
for _, id := range ids {
done, e := s.CleanupJobStaging(ctx, id, retention, dryRun, remove)
if e != nil {
return count, e
}
if done {
count++
}
}
return count, nil
}

Some files were not shown because too many files have changed in this diff Show More