diff --git a/.dockerignore b/.dockerignore index bc16e55..4de53d2 100644 --- a/.dockerignore +++ b/.dockerignore @@ -18,3 +18,8 @@ __pycache__ node_modules evidence .DS_Store + +.lab7-faults +bin + +.cache diff --git a/.env.example b/.env.example index 2e9f054..4bbf8aa 100644 --- a/.env.example +++ b/.env.example @@ -1,41 +1,13 @@ -# Copy to .env for your LOCAL synthetic test environment; never commit .env. -# These are logical interface suggestions; map actual framework names in REPORT.md. -APP_HOST=0.0.0.0 -APP_PORT=8080 -APP_ENV=lab -SESSION_TTL_SECONDS=3600 -# Choose credentials locally. File paths contain no secret values. -DATABASE_URL_FILE=/run/secrets/database_url -# Optional on 3, required use on 4 in lab2; Redis OR Valkey: -SESSION_STORE=postgres -CACHE_URL_FILE=/run/secrets/cache_url -API_REPLICAS=2 -IDEMPOTENCY_TTL_SECONDS=86400 -S3_ENDPOINT=https://s3:9000 -S3_REGION=us-east-1 -S3_BUCKET=messenger-attachments -S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id -S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key -MAX_UPLOAD_BYTES=10485760 -MAX_IMAGE_PIXELS=40000000 -BROKER_URL_FILE=/run/secrets/broker_url -WORKER_CONCURRENCY=2 -JOB_MAX_ATTEMPTS=5 -JOB_TIMEOUT_SECONDS=60 -STAGING_RETENTION_SECONDS=86400 -OTEL_SERVICE_NAME=messenger-api -# Worker must set a distinct service.name. -OTEL_EXPORTER_OTLP_ENDPOINT=https://otel-collector:4318 -OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf -OTEL_RESOURCE_ATTRIBUTES=deployment.environment.name=lab -# TLS and OpenBao: actual paths/ports depend on your implemented trust matrix. -PUBLIC_BASE_URL=https://localhost:8443 -TLS_CA_FILE=/run/secrets/ca.crt -TLS_CERT_FILE=/run/secrets/service.crt -TLS_KEY_FILE=/run/secrets/service.key -BAO_ADDR=https://openbao:8200 -BAO_CACERT=/run/secrets/bao-ca.crt -# No root token here. Bootstrap operator -> short-lived service auth -> scoped token. -ARGON2_MEMORY_KIB=19456 -ARGON2_ITERATIONS=2 -ARGON2_PARALLELISM=1 +# No secret values. compose.yaml contains the default local synthetic profile. +# Export these only when running an additional isolated source snapshot: +# LAB7_PROJECT=mtusi2026lab7 +# LAB7_BASE_URL=https://localhost:8443 +# LAB7_BAO_OPERATOR_ADDR=https://localhost:18200 +# Service environment settings are configured in compose.yaml, not in a secret .env: +# SESSION_TTL_SECONDS=3600 +# ARGON2_MEMORY_KIB=65536 +# ARGON2_ITERATIONS=3 +# ARGON2_PARALLELISM=1 +# API/worker/dispatcher receive DB, S3 and Redis credentials via OpenBao. +# Their TLS identity is mounted at /run/identity; generated runtime secrets +# live in /run/credentials (tmpfs). Operator material stays outside containers. diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 8aef0ff..7615ff1 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -1,18 +1,11 @@ -## Лабораторная 7 +## Лабораторная 7 — Поликарпов Даниил Александрович, БПИ2303 -Автор / группа: TODO -Целевая оценка (3/4/5): TODO -Трек 5, если заявлен: TODO +Целевая оценка: 5. Трек: «Компрометация worker». -Что реализовано и что перенесено: TODO -Команды запуска из чистого clone: TODO -Результаты проверок и ссылка на REPORT.md: TODO -Ограничения / известные ошибки: TODO -Источники и использование ИИ: TODO +Перенесена собственная реализация lab6. Добавлены TLS/mTLS с проверкой service identity, scoped OpenBao policies и DB leases, ограничение login/KDF, rehash, минимальные DB/S3/broker права и воспроизводимые сценарии отзыва/восстановления. -- [ ] Весь обязательный минимум текущей и предыдущих работ сохранён с объявленными переходами. -- [ ] Контракт и публичные тесты не ослаблены. -- [ ] make check и make test выполнены; приложен реальный вывод. -- [ ] Сценарии отказов/безопасности своего уровня воспроизведены. -- [ ] Все секреты и приватные ключи находятся вне Git; отчёт не содержит tokens. -- [ ] README задания сохранён, REPORT.md содержит инструкцию именно моего решения. +Запуск: `make up`; HTTPS smoke: `make test test-security`; последовательная приёмка: `./scripts/final-checks.sh`. Методика, фактические результаты и ограничения находятся в [REPORT.md](../REPORT.md), [evidence](../evidence/) и [модели угроз](../security/THREAT_MODEL.md). + +Обязательные контракты и публичные tests не ослаблены. Secrets/CA key/unseal material создаются локально вне Git и image. Single-host стенд не обеспечивает HA DB/LB/host; законные worker privileges всё ещё позволяют повредить его разрешённые данные. Source/image scan относится к явно указанному scope. + +Источники и использование ИИ Codex раскрыты в отчёте. Публикация PR самим агентом не выполнялась. diff --git a/.github/workflows/application.yml b/.github/workflows/application.yml new file mode 100644 index 0000000..cca4900 --- /dev/null +++ b/.github/workflows/application.yml @@ -0,0 +1,20 @@ +name: Application HTTPS smoke +on: [push, pull_request] +permissions: + contents: read +jobs: + application: + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - name: Build and bootstrap isolated synthetic stand + run: make up + - name: Verify HTTPS API and external trust + run: make test test-security + - name: Stop stand + if: always() + run: ./scripts/compose.sh down -v diff --git a/.gitignore b/.gitignore index b5a7189..7769d51 100644 --- a/.gitignore +++ b/.gitignore @@ -30,3 +30,7 @@ coverage/ .vscode/ # Keep small, sanitized evidence summaries in evidence/ under version control. # Inspect git diff --cached: ignore rules are not a secret scanner. + +.lab7-faults/ + +.cache/ diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..d7586c1 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,38 @@ +FROM --platform=$BUILDPLATFORM golang:1.27.2-alpine3.23@sha256:9e45f0eb4a63ed37ad6e604950407558b7c738e34e015ae77a5d4ad369cde079 AS build + +ENV GOTOOLCHAIN=go1.27.4 +RUN attempt=0; until go version; do attempt=$((attempt + 1)); [ "$attempt" -lt 3 ] || exit 1; sleep 2; done +WORKDIR /src + +COPY service/go.mod service/go.sum ./ +RUN attempt=0; until go mod download; do attempt=$((attempt + 1)); [ "$attempt" -lt 3 ] || exit 1; sleep 2; done + +COPY service/ ./ + +ARG TARGETOS +ARG TARGETARCH +RUN CGO_ENABLED=0 GOOS=$TARGETOS GOARCH=$TARGETARCH \ + go build -mod=readonly -trimpath -ldflags="-s -w" -o /out/messenger ./cmd/app +RUN CGO_ENABLED=0 GOOS=$TARGETOS GOARCH=$TARGETARCH go build -mod=readonly -trimpath -ldflags="-s -w" -o /out/securityprobe ./cmd/securityprobe + +FROM alpine:3.23.6@sha256:85fe1e81d6758c208f3e1eed4338a1997e19d4be002d4dd32d3100c9a8c010a0 + +RUN apk add --no-cache musl=1.2.5-r24 musl-utils=1.2.5-r24 zlib=1.4-r0 imagemagick=7.1.2.35-r0 imagemagick-webp=7.1.2.35-r0 imagemagick-jpeg=7.1.2.35-r0 +COPY infra/imagemagick-policy.xml /etc/ImageMagick-7/policy.xml + +COPY --from=build --chmod=0555 /out/messenger /usr/local/bin/messenger + +COPY --from=build --chmod=0555 /out/securityprobe /usr/local/bin/securityprobe + +ENV APP_HOST=0.0.0.0 \ + APP_PORT=8080 \ + APP_SHUTDOWN_TIMEOUT=8s + +USER 65532:65532 +EXPOSE 8080 +STOPSIGNAL SIGTERM + +HEALTHCHECK --interval=10s --timeout=3s --start-period=5s --retries=3 \ + CMD wget -q -O /dev/null "http://127.0.0.1:8081/health/ready" || exit 1 + +ENTRYPOINT ["/usr/local/bin/messenger"] diff --git a/Makefile b/Makefile index 5ad43ad..8b9ea96 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ PYTHON ?= python3 BASE_URL ?= https://localhost:8443 -CA_FILE ?= +CA_FILE ?= $(CURDIR)/secrets/lab7/lb/ca.crt ACTION_SCRIPT ?= TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",) @@ -31,3 +31,27 @@ test-failover: test-security: $(PYTHON) tests/security.py --base-url "$(BASE_URL)" $(TLS_ARGS) + +.PHONY: up down build vet test-go test-internal-security prove-worker +up: + ./scripts/up.sh +down: + ./scripts/compose.sh down +build: + cd service && go build ./... +vet: + cd service && go vet ./... +test-go: + cd service && go test ./internal/... +test-internal-security: + $(PYTHON) scripts/security.py +prove-worker: + $(PYTHON) scripts/experiment-worker.py + +.PHONY: scan-dependencies scan-image +scan-dependencies: + mkdir -p bin + GOBIN="$(CURDIR)/bin" go install golang.org/x/vuln/cmd/govulncheck@v1.1.4 + cd service && ../bin/govulncheck ./... > ../evidence/govulncheck.txt +scan-image: + $(PYTHON) scripts/scan-images.py diff --git a/README.md b/README.md index b329f91..e94cc95 100644 --- a/README.md +++ b/README.md @@ -110,3 +110,7 @@ Threat model, заполненная trust matrix, TLS/mTLS конфигурац - [OpenBao policies](https://openbao.org/docs/concepts/policies/) - [OpenBao production hardening](https://openbao.org/docs/internals/security/) - [Docker security](https://docs.docker.com/engine/security/) + +## Реализация Поликарпова Даниила Александровича, БПИ2303 + +Инструкция реализации — [REPORT.md](REPORT.md), целевой результат — «5», трек «Компрометация worker». Старт: `make up`; проверка HTTPS: `make test test-security`. Фактические результаты перечислены в [evidence/verification-summary.md](evidence/verification-summary.md). Исходное задание и публичный контракт сохранены. diff --git a/REPORT.md b/REPORT.md index 9801db4..b8878f0 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,37 +1,136 @@ -# Отчёт — лабораторная 7 +# Отчёт — лабораторная работа №7 -- Автор, группа: TODO -- Целевая оценка; выбранный трек 5, если нужен: TODO -- Commit/PR: TODO после публикации -- Использованные источники, библиотеки и помощь ИИ: TODO +Автор: **Поликарпов Даниил Александрович**. Группа: **БПИ2303**. +Целевая оценка: **5**, трек **«Компрометация worker»**. Commit/PR не публиковались; доказательства относятся к рабочему дереву и идентификаторам образов из `evidence/`. ## Запуск из чистого clone -TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных. +Для локальных Go build/vet и source scanner нужен Go с поддержкой загрузки toolchain 1.27.4; Docker-сборка не зависит от host Go. Нужны Docker Engine/Desktop с Compose v2+, Python 3.9+, OpenSSL с поддержкой `req -addext`, доступ к container/module registries и свободные localhost-порты 8443, 13443, 18200. Для полного стенда рекомендуется выделить Docker не менее 8 GiB RAM. Приложение собирается из `service/`; Go toolchain 1.27.4 скачивается и проверяется штатным Go checksum mechanism. Базовый build image содержит Go 1.27.2, но `GOTOOLCHAIN=go1.27.4` закрепляет фактически используемый компилятор. Сетевые загрузки имеют не более трёх попыток. Для публичных base images build использует временную Docker CLI config без keychain helper; пользовательские credentials/config не изменяются. -## Архитектура и решения +```sh +make up +make test test-security +python3 -m venv .venv +.venv/bin/python -m pip install -r requirements-dev.txt +make check validate PYTHON=.venv/bin/python +make build vet test-go +make scan-dependencies scan-image +make test-internal-security +``` -TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab6, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации. +`make up` создаёт локальную CA и synthetic secrets, запускает persistent non-dev OpenBao, делает init/unseal, устанавливает policies и database engine, запускает миграции отдельным процессом, выдаёт DB privileges, создаёт приватный bucket и запускает две API-реплики, dispatcher и worker. `.env` с секретами не нужен. Реальные настройки находятся в `compose.yaml`; `.env.example` объясняет их соответствие. Повторный bootstrap сохраняет ключи, credentials и volumes; он не является автоматической ротацией. -## Доказательства критериев +HTTPS API: `https://localhost:8443`; Grafana: `https://localhost:13443`; OpenBao operator endpoint: `https://localhost:18200`. Публичный CA находится в `secrets/lab7/lb/ca.crt`. Не отключать проверку сертификатов. Пользователь Grafana — `admin`; пароль оператор читает из локального `secrets/lab7/operator/grafana_password`, не прикладывая его к отчёту. -| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | -| --- | --- | --- | --- | -| Публичный smoke | make test | TODO | TODO | -| Остальные пункты уровня 3 | TODO | TODO | TODO | -| Все пункты уровня 4, если заявлен | TODO | TODO | TODO | -| Выбранный трек 5, если заявлен | TODO | TODO | TODO | +```sh +# Остановка с сохранением данных: +make down +# После обычного рестарта sealed OpenBao оператор выполняет: +python3 scripts/bao.py unseal +# Полная последовательная приёмка на synthetic stand: +./scripts/final-checks.sh +# Проверка отдельного source-only snapshot с новыми CA/secrets/volumes: +python3 scripts/prove-clean-clone.py +# Намеренное удаление ВСЕХ учебных данных именно этого проекта: +./scripts/compose.sh down -v +``` -TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены. +Последняя команда разрушает учебные volumes. После неё старые operator unseal shares не подходят новому storage OpenBao: для полностью нового стенда также нужен новый локальный secrets-каталог; сохранённые secrets и storage должны восстанавливаться согласованно. Lab6 не затрагивается. -## Эксперименты и отказы +## Архитектура и границы ответственности -TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения. +Перенесены собственный Go-сервис, миграции 001–011, PostgreSQL/Redis, durable S3 staging, RabbitMQ/outbox, обработка изображений, сигналы OTel и Grafana provisioning. Публичные контракты и проверки lab7 сохранены. Исходные доказательства lab6 не переносились как результаты lab7. -| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | -| --- | --- | --- | --- | -| TODO | TODO | TODO | TODO | +Путь запроса: клиент → TLS → HAProxy → обязательное mTLS → API. Handler разбирает/валидирует вход; service проверяет доступ, выполняет use case и определяет транзакционные переходы; repository делает параметризованные SQL-операции и сохраняет результат. Новые transport/OpenBao/rate-limit интеграции находятся в `internal/pkg`. В PostgreSQL не добавлены HTTP processing, бизнес-процедуры или функции авторизации: DB roles и GRANT — инфраструктурное ограничение привилегий. -## Ограничения и следующие шаги +PostgreSQL остаётся источником истины. До upload 202 исходник записан в durable staging и зафиксированы metadata/job/outbox. Dispatcher публикует с confirm; worker подтверждает доставку после durable результата. At-least-once delivery компенсируется существующей идемпотентностью эффекта. Сессии и message idempotency хранятся в PostgreSQL, Redis используется как cache и общий rate limiter. -TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза. +| Роль | Service / storage | Проверка / сеть / credentials | +| --- | --- | --- | +| API ×2 | api, без локального persistent state | Health через закрытый loopback; LB readiness по mTLS; DB dynamic lease, scoped KV | +| Worker / dispatcher | отдельные worker/dispatcher процессы | Разные DB roles и broker identities; worker CPU 1, RAM 1 GiB, concurrency 2, job timeout 60 s | +| DB | postgres, postgres_data | TLS verify-full :5432; migrator отдельно от app; pg_isready | +| Cache/limit | redis, redis_data | TLS :6380; ACL user api; AOF, noeviction; default user off | +| S3 | SeaweedFS s3, s3_data | mTLS Envoy :9443; backend loopback :9000; разные SigV4 prefixes | +| Broker | RabbitMQ broker, broker_data | Native mTLS :5671, EXTERNAL → CN user; producer/consumer ACL | +| Telemetry | Collector + Prometheus/Loki/Tempo и их volumes | mTLS boundaries, bounded SDK/Collector queues, native storage | +| UI | grafana, grafana_data | Только localhost HTTPS; anonymous off; TLS datasource provisioning создаётся локально | +| Secrets | openbao, bao_data | Non-dev, TLS, cert auth, scoped policies, audit в persistent volume | + +Точные версии/digests, CPU/RAM limits, user, capabilities, mounts и ports закреплены в Compose; фактический запуск сохранён в `evidence/environment.json`. Сеть зависимостей internal; только LB, Grafana и OpenBao имеют дополнительные ingress connections с localhost port mappings. Подробная матрица: [contracts/trust-matrix.md](contracts/trust-matrix.md). + +App/worker/dispatcher работают как UID 65532, с read-only rootfs, drop ALL capabilities, no-new-privileges и tmpfs. OpenBao работает как UID 100:1000 с read-only rootfs; отдельный одноразовый helper меняет владельца только его data volume, не имеет сети и получает CHOWN/DAC_OVERRIDE. Collector и telemetry backends также используют read-only rootfs и выделенные writable volumes/tmpfs. PostgreSQL, Redis, RabbitMQ и SeaweedFS сохраняют writable stateful filesystem/штатный entrypoint: это исключение для инициализации, data/log paths и восстановления; privileged, host network и Docker socket не используются. + +## Пароли, сессии и lifecycle + +Argon2id хранит version, memory/time/parallelism, случайную 16-byte salt и 32-byte результат в PHC-подобной строке. Текущий профиль — 64 MiB, t=3, p=1; сохранённые параметры ограничены до выделения памяти. Существующий профиль lab6 p=4 проверяется и заменяется при успешном входе через compare-and-swap UPDATE. Для отсутствующего пользователя выполняется dummy KDF. На процесс допускаются два вычисления KDF, включая регистрацию; очередь ограничивается request deadline. Общая RAM для двух одновременных KDF — около 128 MiB без учёта runtime, при API limit 512 MiB. + +Login limit общий для реплик: 10/min по hash username и 60/min по hash IP. LB заменяет X-Forwarded-For; API доверяет ему только после проверки LB identity. При превышении — 429/Retry-After; при недоступности Redis login fail-closed с 503. JSON ограничен 1 MiB, upload — 10 MiB, decoded image — 40 млн pixels. Ошибки credentials одинаковые. Session fingerprints — SHA-256 высокоэнтропийных токенов, а не password hashes; logout/expiry проверяются по серверной записи. + +Operator bootstrap создаёт CA и leaf certificates на 24 часа. Закрытый каталог оператора 0700 не монтируется ни одному приложению. Каждый сервис получает только свою identity. Root token после начального provisioning отзывается и удаляется из init.json. Unseal требует 2 из 3 shares; в учебном варианте shares хранятся вместе на защищённом operator host. Это доверие к операторскому компьютеру, не распределённое хранение custodians. + +API/worker/dispatcher получают Bao token с TTL 300 s и max TTL 1800 s, DB lease 600 s/max 1800 s. Renewal выполняется раз в 15 s; независимый watchdog отменяет общий lifecycle context за 3 s до ближайшего token/DB/cert expiry, даже если HTTP renewal завис. Отмена останавливает HTTP/dependency операции и приём задач; рестарт требует новой успешной аутентификации. Изменение рабочего KV вызывает безопасный restart процесса без пересборки image. DB lease rotation ограничивает lifetime отдельных LOGIN roles. + +```sh +# Новый leaf LB с отзывом старого serial, без пересборки: +python3 scripts/renew-leaf.py lb +# Полный incident response worker: +python3 scripts/rotate-worker.py +# Измерение password/expiry и sealed behavior: +python3 scripts/prove-auth.py +python3 scripts/prove-sealed.py +``` + +Ротация worker отзывает Bao auth/tokens/DB leases, закрывает DB sessions, удаляет старого broker CN user, заменяет S3 key и leaf, обновляет CRL и пересоздаёт proxies. Новая worker identity получает новый CN; восстановление нового пользователя не возвращает права старому сертификату. Static S3 key сам по себе не истекает: его фактический отзыв на S3 обязателен и проверяется отдельно. + +## Доказательства и методика + +Фактические результаты и даты находятся в `evidence/`. Наличие скрипта не считается выполненным сценарием. Итоговый перечень проверок приведён в `evidence/verification-summary.md`; файлы с ошибками диагностических запусков не выдаются за PASS. + +| Критерий | Команда | Доказательство | +| --- | --- | --- | +| Контракты, JSON Schema, build/vet, перенесённые Go checks | make check validate; make build vet test-go | contracts.txt, go-checks.txt, govulncheck.txt | +| HTTPS API и внешний недоверенный CA | make test test-security | public-checks.txt, final-smoke.txt | +| Каждая API replica, SAN/CA/no-cert, broker/Collector, backend matrix и Redis ACL | make test-internal-security | security.json, security-run.txt | +| Password rehash, измеренный KDF/login budget, logout/expiry, 429, body limit | prove-auth.py; security.py | auth.json, security.json | +| OpenBao policy allow/deny и bounded sealed behavior | security.py; prove-sealed.py | security.json, sealed.json | +| Leaf/secret rotation и старые identities | renew-leaf.py; experiment-worker.py | leaf-rotation.json, worker-experiment.json | +| Реальные три сигнала и отсутствие canary credentials/body | prove-telemetry.py | telemetry.json | +| Форматы/геометрия, EXIF/alpha, ACL и oversized image | student-tests/images.py | images.json | +| Staging/retention/restore/replay и broker volume | student-tests/storage.py; broker_persistence.py | storage.json, broker-persistence.json | +| Реальный restart stores и прежняя сессия | test-persistence с restart.sh | persistence-run.txt, restart-evidence.json | +| Последовательный отказ обеих API-реплик | test-failover с stop-one.sh, restore между запусками | failover-first.txt, failover-second.txt | +| Source/image scan и applicability | govulncheck; scan-images.py | image-scan.json; security/SCAN_TRIAGE.md | +| Независимый запуск без lab6/secrets/volumes | prove-clean-clone.py | clean-clone.json | + +### Фактический итог 17 ноября 2026 + +Все строки итоговой сводки имеют выполненные команды и PASS. Source scan и scan OS/Go binaries application image дали 0 findings. Удалённый GitHub workflow только настроен; его запуск не заявляется. + +Argon2id (12 samples, 64 MiB/t=3/p=1): p50 103,545 ms, p95/max 181,749 ms. Login probe: 24 заведомо неверных входа, concurrency 8, 1,490 s, p95 536,995 ms; все ответы — ожидаемые 401, без 5xx. Это примерно 16,1 negative authentication responses/s, не измерение максимальной пропускной способности. Отдельная brute-force проверка получила 429/Retry-After через общий limiter. + +Sealed OpenBao с сокращённым token TTL 30 s привёл к fail-closed за 28,546 s; после operator unseal API восстановился. Публичные failover запуски подтвердили 122/123 успешных записей, по одному переходному отказу и first recovery 1,804/1,338 s. Дополнительный сценарий остановил каждый из двух физических контейнеров по очереди и проверил старую сессию и retry без дубля. Restart evidence подтвердил смену StartedAt всех девяти stores с сохранёнными volumes. + +Worker incident response: три повтора по пять файлов; recovery 40,323 / 39,502 / 43,285 s. Во всех повторах: 5 accepted → 5 ready, потеря 0, duplicate jobs 0, ровно 20 manifest records, оригинальные SHA-256 совпали. Старые S3 key, broker CN и Bao leaf отклонены. Resource snapshots и scope широкого/restricted профиля находятся в JSON. + +Source-only snapshot поднялся в отдельном проекте с новыми secrets/CA/volumes и отдельным image tag, прошёл HTTPS API/external trust. Независимая сборка дала побайтно одинаковые messenger/securityprobe binaries; Docker image manifest/provenance может иметь другой ID. Это зафиксировано отдельно в `image-equivalence.json`. + +### Исследовательский трек «Компрометация worker» + +Гипотеза: отдельные worker credentials ограничивают доступ к DB/S3/broker, а последовательный отзыв и замена identity восстанавливают обработку устойчиво принятых доброкачественных задач без потерь и дублей. + +Методика: synthetic stand, 3 повтора; в каждом worker остановлен, приняты пять PNG 320×200 из публичного fixture, проверены durable jobs. Одноразовые изолированные operator probes воспроизводят широкие права исходной lab6 и сравнивают их с restricted worker: DB password/session tables и DDL; S3 bucket-wide против prefix scope; broker configure/publish против consume. Широкие credentials не выдаются основным app/worker процессам. После incident response проверяются старый Bao leaf, broker CN, S3 key и новый mTLS access; worker восстанавливается. Сверяются IDs, пять ready, SHA-256 оригиналов и ровно 20 variant-manifest rows (original + три производных на файл). + +Профиль — пять последовательных acceptance requests при остановленном consumer, worker concurrency 2, без прогрева и без заявления максимального RPS. Сохраняются acceptance latency, recovery seconds, observation errors и Docker CPU/RAM snapshot. Сценарий исследует recovery и permissions, а не производительность под production нагрузкой. Результаты всех повторов — `worker-experiment.json`; measured login concurrency/RPS/latency — `auth.json`. + +Во время разработки обнаружен stale network namespace у Envoy после restart parent. Исправление: после рестарта родителя sidecar пересоздаётся. Диагностические попытки до исправления могли довести synthetic jobs до failed; они не включаются в успешные исследовательские повторы. Исправленный restart/rotation проходит отдельные проверки. Дополнительно для loopback GRPC listeners Loki/Tempo явно задан loopback advertised ring address; иначе их внутренний scheduler/ingester пытался обращаться к закрытому bridge address. + +Проверка локальных generated secrets, private keys и unseal shares против файлов сдачи: `python3 scripts/audit-secrets.py`, результат — `evidence/secret-audit.json`. Она дополняет canary-проверку telemetry. + +## Ограничения и источники + +Один Docker host, одна DB, один LB и один broker не защищают от потери host/storage. Worker с его законными правами способен испортить разрешённые job metadata и S3 prefixes; эксперимент не доказывает сохранность после произвольной злонамеренной порчи этих ресурсов. Leaf renewal в этом треке выполняет оператор; автоматический CA rollover не заявляется. Canaries проверяют записанные logs и выборочные backend signals, а не все возможные будущие потоки. Scan scope и остаточные findings явно перечислены в triage. + +Собственная база перенесена из lab6. Использованы Go, pgx, AWS SDK for Go v2, amqp091-go, go-redis, OTel SDK, ImageMagick и закреплённые Docker images. Реализация, скрипты и документация подготовлены с помощью ИИ Codex; результаты должны быть объяснимы автором на защите. + +Первичные источники: [Argon2id и OWASP](https://cheatsheetseries.owasp.org/cheatsheets/Password_Storage_Cheat_Sheet.html), [OpenBao cert auth](https://openbao.org/docs/auth/cert/), [OpenBao audit configuration](https://openbao.org/docs/configuration/audit/), [RabbitMQ TLS](https://www.rabbitmq.com/docs/ssl), [RabbitMQ access control](https://www.rabbitmq.com/docs/access-control), [SeaweedFS S3](https://github.com/seaweedfs/seaweedfs/wiki/S3-Configuration), [OTel Collector configuration](https://opentelemetry.io/docs/collector/configuration/). diff --git a/compose.faults.yaml b/compose.faults.yaml new file mode 100644 index 0000000..09a8dcd --- /dev/null +++ b/compose.faults.yaml @@ -0,0 +1,22 @@ +# Explicit isolated experiment profile. Never enabled by make up. +services: + api: + environment: + LAB5_FAULTS: "1" + volumes: + - ./.lab7-faults:/faults + worker: + environment: + LAB5_FAULTS: "1" + JOB_TIMEOUT: 5s + JOB_LEASE: 8s + volumes: + - ./.lab7-faults:/faults + dispatcher: + environment: + LAB5_FAULTS: "1" + volumes: + - ./.lab7-faults:/faults + operator: + environment: + LAB5_FAULTS: "1" diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 0000000..2f384b7 --- /dev/null +++ b/compose.yaml @@ -0,0 +1,1030 @@ +{ + "name": "mtusi2026lab7", + "services": { + "api": { + "image": "mtusi-messenger:lab7", + "build": ".", + "restart": "unless-stopped", + "user": "65532:65532", + "cpus": 1, + "mem_limit": "512m", + "environment": { + "APPLICATION_ENVIRONMENT": "prod", + "APPLICATION_LOG_LEVEL": "info", + "APPLICATION_SHUTDOWN_TIMEOUT": "8s", + "APP_ENABLED": "true", + "APP_HOST": "0.0.0.0", + "APP_IDLE_TIMEOUT": "60s", + "APP_PORT": 8080, + "APP_READ_TIMEOUT": "125s", + "APP_SHUTDOWN_TIMEOUT": "8s", + "APP_WRITE_TIMEOUT": "125s", + "BROKER_URL_FILE": "/run/identity/broker_url", + "GOMEMLIMIT": "384MiB", + "JOB_LEASE": "90s", + "JOB_MAX_ATTEMPTS": "5", + "JOB_TIMEOUT": "60s", + "JWT_ACCESS_TOKEN_LIFETIME": "15m", + "MAX_IMAGE_PIXELS": "40000000", + "MAX_UPLOAD_BYTES": "10485760", + "OTEL_EXPORTER_OTLP_ENDPOINT": "https://otel-collector:9443", + "OTEL_EXPORTER_OTLP_PROTOCOL": "http/protobuf", + "OTEL_SDK_DISABLED": "false", + "POSTGRES_CONNECT_TIMEOUT": "500ms", + "POSTGRES_DATABASE": "messenger", + "POSTGRES_HOST": "postgres", + "POSTGRES_MAX_CONNECTIONS": "16", + "POSTGRES_PORT": 5432, + "POSTGRES_QUERY_TIMEOUT": "2s", + "POSTGRES_RUN_MIGRATIONS": "false", + "POSTGRES_USER": "api", + "POSTGRES_VERSION_TABLE": "versions", + "RECONCILE_GRACE_PERIOD": "1h", + "REDIS_DB": 0, + "REDIS_HOST": "redis", + "REDIS_PORT": "6380", + "REDIS_USERNAME": "api", + "S3_BUCKET": "messenger-attachments", + "S3_ENDPOINT": "https://s3:9443", + "S3_REGION": "us-east-1", + "SESSION_TTL_SECONDS": "3600", + "STAGING_RETENTION": "24h", + "UPLOAD_TIMEOUT": "120s", + "WORKER_CONCURRENCY": "2", + "WORKER_PREFETCH": "2", + "TLS_REQUIRED": "true", + "JWT_PRIVATE_KEY_PATH": "/run/identity/jwt.key", + "JWT_PUBLIC_KEY_PATH": "/run/identity/jwt.pub", + "BAO_ADDR": "https://openbao:8200", + "SERVICE_ROLE": "api", + "ARGON2_MEMORY_KIB": "65536", + "ARGON2_ITERATIONS": "3", + "ARGON2_PARALLELISM": "1" + }, + "volumes": [ + "./secrets/lab7/api:/run/identity:ro" + ], + "networks": [ + "default" + ], + "read_only": true, + "cap_drop": [ + "ALL" + ], + "security_opt": [ + "no-new-privileges:true" + ], + "tmpfs": [ + "/tmp:size=256m,mode=1777", + "/run/credentials:uid=65532,gid=65532,mode=0700,size=1m" + ], + "stop_grace_period": "10s", + "deploy": { + "replicas": 2 + }, + "healthcheck": { + "test": [ + "CMD", + "wget", + "-q", + "-O", + "/dev/null", + "http://127.0.0.1:8081/health/ready" + ], + "interval": "3s", + "timeout": "3s", + "retries": 30 + } + }, + "worker": { + "image": "mtusi-messenger:lab7", + "build": ".", + "restart": "unless-stopped", + "user": "65532:65532", + "cpus": 1, + "mem_limit": "1g", + "environment": { + "APPLICATION_ENVIRONMENT": "prod", + "APPLICATION_LOG_LEVEL": "info", + "APPLICATION_SHUTDOWN_TIMEOUT": "8s", + "APP_ENABLED": "true", + "APP_HOST": "0.0.0.0", + "APP_IDLE_TIMEOUT": "60s", + "APP_PORT": 8080, + "APP_READ_TIMEOUT": "125s", + "APP_SHUTDOWN_TIMEOUT": "8s", + "APP_WRITE_TIMEOUT": "125s", + "BROKER_URL_FILE": "/run/identity/broker_url", + "GOMEMLIMIT": "384MiB", + "JOB_LEASE": "90s", + "JOB_MAX_ATTEMPTS": "5", + "JOB_TIMEOUT": "60s", + "JWT_ACCESS_TOKEN_LIFETIME": "15m", + "MAX_IMAGE_PIXELS": "40000000", + "MAX_UPLOAD_BYTES": "10485760", + "OTEL_EXPORTER_OTLP_ENDPOINT": "https://otel-collector:9443", + "OTEL_EXPORTER_OTLP_PROTOCOL": "http/protobuf", + "OTEL_SDK_DISABLED": "false", + "POSTGRES_CONNECT_TIMEOUT": "500ms", + "POSTGRES_DATABASE": "messenger", + "POSTGRES_HOST": "postgres", + "POSTGRES_MAX_CONNECTIONS": "16", + "POSTGRES_PORT": 5432, + "POSTGRES_QUERY_TIMEOUT": "2s", + "POSTGRES_RUN_MIGRATIONS": "false", + "POSTGRES_USER": "api", + "POSTGRES_VERSION_TABLE": "versions", + "RECONCILE_GRACE_PERIOD": "1h", + "REDIS_DB": 0, + "REDIS_HOST": "redis", + "REDIS_PORT": "6380", + "REDIS_USERNAME": "api", + "S3_BUCKET": "messenger-attachments", + "S3_ENDPOINT": "https://s3:9443", + "S3_REGION": "us-east-1", + "SESSION_TTL_SECONDS": "3600", + "STAGING_RETENTION": "24h", + "UPLOAD_TIMEOUT": "120s", + "WORKER_CONCURRENCY": "2", + "WORKER_PREFETCH": "2", + "TLS_REQUIRED": "true", + "BAO_ADDR": "https://openbao:8200", + "SERVICE_ROLE": "worker", + "ARGON2_MEMORY_KIB": "65536", + "ARGON2_ITERATIONS": "3", + "ARGON2_PARALLELISM": "1" + }, + "volumes": [ + "./secrets/lab7/worker:/run/identity:ro" + ], + "networks": [ + "default" + ], + "read_only": true, + "cap_drop": [ + "ALL" + ], + "security_opt": [ + "no-new-privileges:true" + ], + "tmpfs": [ + "/tmp:size=256m,mode=1777", + "/run/credentials:uid=65532,gid=65532,mode=0700,size=1m" + ], + "stop_grace_period": "10s", + "command": [ + "worker" + ], + "healthcheck": { + "disable": true + } + }, + "dispatcher": { + "image": "mtusi-messenger:lab7", + "build": ".", + "restart": "unless-stopped", + "user": "65532:65532", + "cpus": 0.25, + "mem_limit": "512m", + "environment": { + "APPLICATION_ENVIRONMENT": "prod", + "APPLICATION_LOG_LEVEL": "info", + "APPLICATION_SHUTDOWN_TIMEOUT": "8s", + "APP_ENABLED": "true", + "APP_HOST": "0.0.0.0", + "APP_IDLE_TIMEOUT": "60s", + "APP_PORT": 8080, + "APP_READ_TIMEOUT": "125s", + "APP_SHUTDOWN_TIMEOUT": "8s", + "APP_WRITE_TIMEOUT": "125s", + "BROKER_URL_FILE": "/run/identity/broker_url", + "GOMEMLIMIT": "384MiB", + "JOB_LEASE": "90s", + "JOB_MAX_ATTEMPTS": "5", + "JOB_TIMEOUT": "60s", + "JWT_ACCESS_TOKEN_LIFETIME": "15m", + "MAX_IMAGE_PIXELS": "40000000", + "MAX_UPLOAD_BYTES": "10485760", + "OTEL_EXPORTER_OTLP_ENDPOINT": "https://otel-collector:9443", + "OTEL_EXPORTER_OTLP_PROTOCOL": "http/protobuf", + "OTEL_SDK_DISABLED": "false", + "POSTGRES_CONNECT_TIMEOUT": "500ms", + "POSTGRES_DATABASE": "messenger", + "POSTGRES_HOST": "postgres", + "POSTGRES_MAX_CONNECTIONS": "16", + "POSTGRES_PORT": 5432, + "POSTGRES_QUERY_TIMEOUT": "2s", + "POSTGRES_RUN_MIGRATIONS": "false", + "POSTGRES_USER": "api", + "POSTGRES_VERSION_TABLE": "versions", + "RECONCILE_GRACE_PERIOD": "1h", + "REDIS_DB": 0, + "REDIS_HOST": "redis", + "REDIS_PORT": "6380", + "REDIS_USERNAME": "api", + "S3_BUCKET": "messenger-attachments", + "S3_ENDPOINT": "https://s3:9443", + "S3_REGION": "us-east-1", + "SESSION_TTL_SECONDS": "3600", + "STAGING_RETENTION": "24h", + "UPLOAD_TIMEOUT": "120s", + "WORKER_CONCURRENCY": "2", + "WORKER_PREFETCH": "2", + "TLS_REQUIRED": "true", + "BAO_ADDR": "https://openbao:8200", + "SERVICE_ROLE": "dispatcher", + "ARGON2_MEMORY_KIB": "65536", + "ARGON2_ITERATIONS": "3", + "ARGON2_PARALLELISM": "1" + }, + "volumes": [ + "./secrets/lab7/dispatcher:/run/identity:ro" + ], + "networks": [ + "default" + ], + "read_only": true, + "cap_drop": [ + "ALL" + ], + "security_opt": [ + "no-new-privileges:true" + ], + "tmpfs": [ + "/tmp:size=256m,mode=1777", + "/run/credentials:uid=65532,gid=65532,mode=0700,size=1m" + ], + "stop_grace_period": "10s", + "command": [ + "dispatcher" + ], + "healthcheck": { + "disable": true + } + }, + "migrate": { + "image": "mtusi-messenger:lab7", + "build": ".", + "restart": "no", + "user": "65532:65532", + "cpus": 1, + "mem_limit": "512m", + "environment": { + "APPLICATION_ENVIRONMENT": "prod", + "APPLICATION_LOG_LEVEL": "info", + "APPLICATION_SHUTDOWN_TIMEOUT": "8s", + "APP_ENABLED": "true", + "APP_HOST": "0.0.0.0", + "APP_IDLE_TIMEOUT": "60s", + "APP_PORT": 8080, + "APP_READ_TIMEOUT": "125s", + "APP_SHUTDOWN_TIMEOUT": "8s", + "APP_WRITE_TIMEOUT": "125s", + "BROKER_URL_FILE": "/run/identity/broker_url", + "GOMEMLIMIT": "384MiB", + "JOB_LEASE": "90s", + "JOB_MAX_ATTEMPTS": "5", + "JOB_TIMEOUT": "60s", + "JWT_ACCESS_TOKEN_LIFETIME": "15m", + "MAX_IMAGE_PIXELS": "40000000", + "MAX_UPLOAD_BYTES": "10485760", + "OTEL_EXPORTER_OTLP_ENDPOINT": "https://otel-collector:9443", + "OTEL_EXPORTER_OTLP_PROTOCOL": "http/protobuf", + "OTEL_SDK_DISABLED": "true", + "POSTGRES_CONNECT_TIMEOUT": "500ms", + "POSTGRES_DATABASE": "messenger", + "POSTGRES_HOST": "postgres", + "POSTGRES_MAX_CONNECTIONS": "16", + "POSTGRES_PORT": 5432, + "POSTGRES_QUERY_TIMEOUT": "2s", + "POSTGRES_RUN_MIGRATIONS": "false", + "POSTGRES_USER": "messenger", + "POSTGRES_VERSION_TABLE": "versions", + "RECONCILE_GRACE_PERIOD": "1h", + "REDIS_DB": 0, + "REDIS_HOST": "redis", + "REDIS_PORT": "6380", + "REDIS_USERNAME": "api", + "S3_BUCKET": "messenger-attachments", + "S3_ENDPOINT": "https://s3:9443", + "S3_REGION": "us-east-1", + "SESSION_TTL_SECONDS": "3600", + "STAGING_RETENTION": "24h", + "UPLOAD_TIMEOUT": "120s", + "WORKER_CONCURRENCY": "2", + "WORKER_PREFETCH": "2", + "TLS_REQUIRED": "true", + "JWT_PRIVATE_KEY_PATH": "/run/identity/jwt.key", + "JWT_PUBLIC_KEY_PATH": "/run/identity/jwt.pub", + "BAO_ADDR": "", + "SERVICE_ROLE": "api", + "ARGON2_MEMORY_KIB": "65536", + "ARGON2_ITERATIONS": "3", + "ARGON2_PARALLELISM": "1", + "POSTGRES_PASSWORD_FILE": "/run/dbpassword" + }, + "volumes": [ + "./secrets/lab7/operator-client:/run/identity:ro", + "./secrets/lab7/postgres/password:/run/dbpassword:ro" + ], + "networks": [ + "default" + ], + "read_only": true, + "cap_drop": [ + "ALL" + ], + "security_opt": [ + "no-new-privileges:true" + ], + "tmpfs": [ + "/tmp:size=256m,mode=1777", + "/run/credentials:uid=65532,gid=65532,mode=0700,size=1m" + ], + "stop_grace_period": "10s", + "deploy": {}, + "healthcheck": { + "disable": true + }, + "command": [ + "migrate" + ] + }, + "operator": { + "image": "mtusi-messenger:lab7", + "build": ".", + "restart": "no", + "user": "65532:65532", + "cpus": 1, + "mem_limit": "512m", + "environment": { + "APPLICATION_ENVIRONMENT": "prod", + "APPLICATION_LOG_LEVEL": "info", + "APPLICATION_SHUTDOWN_TIMEOUT": "8s", + "APP_ENABLED": "true", + "APP_HOST": "0.0.0.0", + "APP_IDLE_TIMEOUT": "60s", + "APP_PORT": 8080, + "APP_READ_TIMEOUT": "125s", + "APP_SHUTDOWN_TIMEOUT": "8s", + "APP_WRITE_TIMEOUT": "125s", + "BROKER_URL_FILE": "/run/identity/broker_url", + "GOMEMLIMIT": "384MiB", + "JOB_LEASE": "90s", + "JOB_MAX_ATTEMPTS": "5", + "JOB_TIMEOUT": "60s", + "JWT_ACCESS_TOKEN_LIFETIME": "15m", + "MAX_IMAGE_PIXELS": "40000000", + "MAX_UPLOAD_BYTES": "10485760", + "OTEL_EXPORTER_OTLP_ENDPOINT": "https://otel-collector:9443", + "OTEL_EXPORTER_OTLP_PROTOCOL": "http/protobuf", + "OTEL_SDK_DISABLED": "true", + "POSTGRES_CONNECT_TIMEOUT": "500ms", + "POSTGRES_DATABASE": "messenger", + "POSTGRES_HOST": "postgres", + "POSTGRES_MAX_CONNECTIONS": "16", + "POSTGRES_PORT": 5432, + "POSTGRES_QUERY_TIMEOUT": "2s", + "POSTGRES_RUN_MIGRATIONS": "false", + "POSTGRES_USER": "messenger", + "POSTGRES_VERSION_TABLE": "versions", + "RECONCILE_GRACE_PERIOD": "1h", + "REDIS_DB": 0, + "REDIS_HOST": "redis", + "REDIS_PORT": "6380", + "REDIS_USERNAME": "api", + "S3_BUCKET": "messenger-attachments", + "S3_ENDPOINT": "https://s3:9443", + "S3_REGION": "us-east-1", + "SESSION_TTL_SECONDS": "3600", + "STAGING_RETENTION": "24h", + "UPLOAD_TIMEOUT": "120s", + "WORKER_CONCURRENCY": "2", + "WORKER_PREFETCH": "2", + "TLS_REQUIRED": "true", + "JWT_PRIVATE_KEY_PATH": "/run/identity/jwt.key", + "JWT_PUBLIC_KEY_PATH": "/run/identity/jwt.pub", + "BAO_ADDR": "", + "SERVICE_ROLE": "api", + "ARGON2_MEMORY_KIB": "65536", + "ARGON2_ITERATIONS": "3", + "ARGON2_PARALLELISM": "1", + "POSTGRES_PASSWORD_FILE": "/run/dbpassword", + "S3_ACCESS_KEY_ID_FILE": "/run/identity/s3_admin_access", + "S3_SECRET_ACCESS_KEY_FILE": "/run/identity/s3_admin_secret" + }, + "volumes": [ + "./secrets/lab7/operator-client:/run/identity:ro", + "./secrets/lab7/postgres/password:/run/dbpassword:ro", + "./tests/fixtures:/fixtures:ro" + ], + "networks": [ + "default" + ], + "read_only": true, + "cap_drop": [ + "ALL" + ], + "security_opt": [ + "no-new-privileges:true" + ], + "tmpfs": [ + "/tmp:size=256m,mode=1777", + "/run/credentials:uid=65532,gid=65532,mode=0700,size=1m" + ], + "stop_grace_period": "10s", + "deploy": {}, + "healthcheck": { + "disable": true + }, + "command": [ + "storage", + "reconcile" + ], + "profiles": [ + "tools" + ] + }, + "lb": { + "image": "haproxy:3.2-alpine@sha256:96ec3c75a9264ba94e733eeacb44051ed6e7b79bb77a73c2c36f236b6423f1f0", + "restart": "unless-stopped", + "cpus": 0.5, + "mem_limit": "128m", + "ports": [ + "127.0.0.1:8443:8443" + ], + "volumes": [ + "./infra/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro", + "./secrets/lab7/lb:/run/identity:ro" + ], + "read_only": true, + "tmpfs": [ + "/tmp:mode=1777" + ], + "cap_drop": [ + "ALL" + ], + "security_opt": [ + "no-new-privileges:true" + ], + "networks": [ + "default", + "ingress" + ] + }, + "postgres": { + "image": "postgres:17.11-alpine3.23@sha256:05aa77262db9277fde6374bb2f901c9604b8a9373029e1a7993440c6aae48736", + "restart": "unless-stopped", + "cpus": 2, + "mem_limit": "1g", + "environment": { + "POSTGRES_USER": "messenger", + "POSTGRES_DB": "messenger", + "POSTGRES_PASSWORD_FILE": "/run/identity/password" + }, + "volumes": [ + "postgres_data:/var/lib/postgresql/data", + "./secrets/lab7/postgres:/run/identity:ro", + "./secrets/lab7/postgres/init.sql:/docker-entrypoint-initdb.d/roles.sql:ro", + "./infra/postgres/pg_hba.conf:/etc/postgresql/pg_hba.conf:ro" + ], + "command": [ + "sh", + "-c", + "cp /run/identity/tls.key /tmp/server.key && chown postgres /tmp/server.key && chmod 600 /tmp/server.key && exec docker-entrypoint.sh postgres -c ssl=on -c ssl_cert_file=/run/identity/tls.crt -c ssl_key_file=/tmp/server.key -c hba_file=/etc/postgresql/pg_hba.conf" + ], + "healthcheck": { + "test": [ + "CMD", + "pg_isready", + "-U", + "messenger", + "-d", + "messenger" + ], + "interval": "2s", + "timeout": "2s", + "retries": 30 + }, + "security_opt": [ + "no-new-privileges:true" + ] + }, + "redis": { + "image": "redis:8.2.11-alpine3.22@sha256:489a769e0a54dc245896132a15c72d2bc7296719954da99398f43a3877e5f8eb", + "restart": "unless-stopped", + "cpus": 0.25, + "mem_limit": "128m", + "command": [ + "redis-server", + "--port", + "0", + "--tls-port", + "6380", + "--tls-cert-file", + "/run/identity/tls.crt", + "--tls-key-file", + "/run/identity/tls.key", + "--tls-ca-cert-file", + "/run/identity/ca.crt", + "--tls-auth-clients", + "no", + "--aclfile", + "/run/identity/users.acl", + "--appendonly", + "yes", + "--maxmemory", + "64mb", + "--maxmemory-policy", + "noeviction" + ], + "volumes": [ + "redis_data:/data", + "./secrets/lab7/redis:/run/identity:ro" + ], + "security_opt": [ + "no-new-privileges:true" + ] + }, + "broker": { + "image": "rabbitmq:4.2.0-alpine@sha256:32962eea9bf99928c2fc541a7defb78c7feca9df0434c5dfb7b539d61e1af524", + "hostname": "broker", + "restart": "unless-stopped", + "cpus": 0.5, + "mem_limit": "512m", + "volumes": [ + "broker_data:/var/lib/rabbitmq", + "./secrets/lab7/broker:/run/identity:ro", + "./infra/rabbitmq/rabbitmq.conf:/etc/rabbitmq/rabbitmq.conf:ro", + "./infra/rabbitmq/enabled_plugins:/etc/rabbitmq/enabled_plugins:ro", + "./infra/rabbitmq/definitions.json:/etc/rabbitmq/definitions.json:ro" + ], + "healthcheck": { + "test": [ + "CMD", + "rabbitmq-diagnostics", + "-q", + "check_running" + ], + "interval": "3s", + "timeout": "5s", + "retries": 30 + }, + "security_opt": [ + "no-new-privileges:true" + ] + }, + "s3": { + "image": "chrislusf/seaweedfs:4.52@sha256:57877e32bb90073b37dc109b438ba66e84fd5ed9ced5f5c04f56444ef243ce59", + "restart": "unless-stopped", + "cpus": 1, + "mem_limit": "512m", + "command": [ + "server", + "-dir=/data", + "-ip=127.0.0.1", + "-ip.bind=127.0.0.1", + "-volume.max=16", + "-master.volumeSizeLimitMB=1024", + "-filer", + "-s3", + "-s3.port=9000", + "-s3.config=/run/identity/s3.json" + ], + "volumes": [ + "s3_data:/data", + "./secrets/lab7/s3:/run/identity:ro" + ], + "security_opt": [ + "no-new-privileges:true" + ] + }, + "otel-collector": { + "command": [ + "--config=/etc/otelcol/config.yaml" + ], + "cpus": 0.5, + "image": "otel/opentelemetry-collector-contrib:0.140.0", + "mem_limit": "256m", + "restart": "unless-stopped", + "volumes": [ + "./secrets/lab7/otel-collector:/run/identity:ro", + "./infra/otel/collector.yaml:/etc/otelcol/config.yaml:ro" + ], + "security_opt": [ + "no-new-privileges:true" + ], + "cap_drop": [ + "ALL" + ], + "read_only": true, + "tmpfs": [ + "/tmp:mode=1777,size=64m" + ] + }, + "prometheus": { + "command": [ + "--config.file=/etc/prometheus/prometheus.yaml", + "--storage.tsdb.retention.time=7d", + "--storage.tsdb.retention.size=1GB", + "--web.listen-address=127.0.0.1:9090" + ], + "cpus": 0.5, + "image": "prom/prometheus:v3.16.0", + "mem_limit": "384m", + "restart": "unless-stopped", + "volumes": [ + { + "source": "prometheus_data", + "target": "/prometheus", + "type": "volume", + "volume": {} + }, + "./secrets/lab7/prometheus:/run/identity:ro", + "./infra/otel/prometheus.yaml:/etc/prometheus/prometheus.yaml:ro", + "./infra/otel/alerts.yaml:/etc/prometheus/alerts.yaml:ro" + ], + "security_opt": [ + "no-new-privileges:true" + ], + "cap_drop": [ + "ALL" + ], + "read_only": true, + "tmpfs": [ + "/tmp:mode=1777,size=64m" + ] + }, + "loki": { + "command": [ + "-config.file=/etc/loki/config.yaml" + ], + "cpus": 0.5, + "image": "grafana/loki:3.7.10", + "mem_limit": "384m", + "restart": "unless-stopped", + "volumes": [ + { + "source": "loki_data", + "target": "/loki", + "type": "volume", + "volume": {} + }, + "./secrets/lab7/loki:/run/identity:ro", + "./infra/otel/loki.yaml:/etc/loki/config.yaml:ro" + ], + "security_opt": [ + "no-new-privileges:true" + ], + "cap_drop": [ + "ALL" + ], + "read_only": true, + "tmpfs": [ + "/tmp:mode=1777,size=64m" + ] + }, + "tempo": { + "command": [ + "-config.file=/etc/tempo.yaml" + ], + "cpus": 0.5, + "image": "grafana/tempo:2.9.0", + "mem_limit": "384m", + "restart": "unless-stopped", + "volumes": [ + { + "source": "tempo_data", + "target": "/var/tempo", + "type": "volume", + "volume": {} + }, + "./secrets/lab7/tempo:/run/identity:ro", + "./infra/otel/tempo.yaml:/etc/tempo.yaml:ro" + ], + "security_opt": [ + "no-new-privileges:true" + ], + "cap_drop": [ + "ALL" + ], + "read_only": true, + "tmpfs": [ + "/tmp:mode=1777,size=64m" + ] + }, + "grafana": { + "cpus": 0.5, + "environment": { + "GF_ANALYTICS_CHECK_FOR_UPDATES": "false", + "GF_ANALYTICS_REPORTING_ENABLED": "false", + "GF_AUTH_ANONYMOUS_ENABLED": "false", + "GF_SECURITY_ADMIN_PASSWORD__FILE": "/run/identity/password", + "GF_SERVER_PROTOCOL": "https", + "GF_SERVER_CERT_FILE": "/run/identity/tls.crt", + "GF_SERVER_CERT_KEY": "/run/identity/tls.key" + }, + "image": "grafana/grafana:13.3.1", + "mem_limit": "384m", + "restart": "unless-stopped", + "volumes": [ + { + "source": "grafana_data", + "target": "/var/lib/grafana", + "type": "volume", + "volume": {} + }, + "./secrets/lab7/grafana:/run/identity:ro", + "./infra/otel/dashboards:/var/lib/grafana/dashboards:ro", + "./infra/otel/grafana/dashboards:/etc/grafana/provisioning/dashboards:ro", + "./secrets/lab7/grafana/datasources.yaml:/etc/grafana/provisioning/datasources/default.yaml:ro" + ], + "ports": [ + "127.0.0.1:13443:3000" + ], + "security_opt": [ + "no-new-privileges:true" + ], + "cap_drop": [ + "ALL" + ], + "networks": [ + "default", + "ingress" + ], + "read_only": true, + "tmpfs": [ + "/tmp:mode=1777,size=64m" + ] + }, + "s3-tls": { + "image": "envoyproxy/envoy:v1.36.4", + "restart": "unless-stopped", + "network_mode": "service:s3", + "depends_on": { + "s3": { + "condition": "service_started" + } + }, + "user": "65532:65532", + "cpus": 0.25, + "mem_limit": "128m", + "command": [ + "envoy", + "-c", + "/etc/envoy/config.json", + "--concurrency", + "1", + "--log-level", + "warn" + ], + "volumes": [ + "./secrets/lab7/s3:/run/identity:ro", + "./infra/envoy/s3.json:/etc/envoy/config.json:ro" + ], + "read_only": true, + "tmpfs": [ + "/tmp:mode=1777" + ], + "cap_drop": [ + "ALL" + ], + "security_opt": [ + "no-new-privileges:true" + ] + }, + "otel-collector-tls": { + "image": "envoyproxy/envoy:v1.36.4", + "restart": "unless-stopped", + "network_mode": "service:otel-collector", + "depends_on": { + "otel-collector": { + "condition": "service_started" + } + }, + "user": "65532:65532", + "cpus": 0.25, + "mem_limit": "128m", + "command": [ + "envoy", + "-c", + "/etc/envoy/config.json", + "--concurrency", + "1", + "--log-level", + "warn" + ], + "volumes": [ + "./secrets/lab7/otel-collector:/run/identity:ro", + "./infra/envoy/otel-collector.json:/etc/envoy/config.json:ro" + ], + "read_only": true, + "tmpfs": [ + "/tmp:mode=1777" + ], + "cap_drop": [ + "ALL" + ], + "security_opt": [ + "no-new-privileges:true" + ] + }, + "prometheus-tls": { + "image": "envoyproxy/envoy:v1.36.4", + "restart": "unless-stopped", + "network_mode": "service:prometheus", + "depends_on": { + "prometheus": { + "condition": "service_started" + } + }, + "user": "65532:65532", + "cpus": 0.25, + "mem_limit": "128m", + "command": [ + "envoy", + "-c", + "/etc/envoy/config.json", + "--concurrency", + "1", + "--log-level", + "warn" + ], + "volumes": [ + "./secrets/lab7/prometheus:/run/identity:ro", + "./infra/envoy/prometheus.json:/etc/envoy/config.json:ro" + ], + "read_only": true, + "tmpfs": [ + "/tmp:mode=1777" + ], + "cap_drop": [ + "ALL" + ], + "security_opt": [ + "no-new-privileges:true" + ] + }, + "loki-tls": { + "image": "envoyproxy/envoy:v1.36.4", + "restart": "unless-stopped", + "network_mode": "service:loki", + "depends_on": { + "loki": { + "condition": "service_started" + } + }, + "user": "65532:65532", + "cpus": 0.25, + "mem_limit": "128m", + "command": [ + "envoy", + "-c", + "/etc/envoy/config.json", + "--concurrency", + "1", + "--log-level", + "warn" + ], + "volumes": [ + "./secrets/lab7/loki:/run/identity:ro", + "./infra/envoy/loki.json:/etc/envoy/config.json:ro" + ], + "read_only": true, + "tmpfs": [ + "/tmp:mode=1777" + ], + "cap_drop": [ + "ALL" + ], + "security_opt": [ + "no-new-privileges:true" + ] + }, + "tempo-tls": { + "image": "envoyproxy/envoy:v1.36.4", + "restart": "unless-stopped", + "network_mode": "service:tempo", + "depends_on": { + "tempo": { + "condition": "service_started" + } + }, + "user": "65532:65532", + "cpus": 0.25, + "mem_limit": "128m", + "command": [ + "envoy", + "-c", + "/etc/envoy/config.json", + "--concurrency", + "1", + "--log-level", + "warn" + ], + "volumes": [ + "./secrets/lab7/tempo:/run/identity:ro", + "./infra/envoy/tempo.json:/etc/envoy/config.json:ro" + ], + "read_only": true, + "tmpfs": [ + "/tmp:mode=1777" + ], + "cap_drop": [ + "ALL" + ], + "security_opt": [ + "no-new-privileges:true" + ] + }, + "openbao": { + "image": "openbao/openbao:2.4.4", + "restart": "unless-stopped", + "user": "100:1000", + "cpus": 0.5, + "mem_limit": "256m", + "command": [ + "server", + "-config=/etc/openbao/server.hcl" + ], + "environment": { + "BAO_ADDR": "https://localhost:8200", + "BAO_CACERT": "/run/identity/ca.crt" + }, + "ports": [ + "127.0.0.1:18200:8200" + ], + "volumes": [ + "bao_data:/openbao/data", + "./secrets/lab7/openbao:/run/identity:ro", + "./infra/openbao/server.hcl:/etc/openbao/server.hcl:ro" + ], + "cap_drop": [ + "ALL" + ], + "security_opt": [ + "no-new-privileges:true" + ], + "entrypoint": [ + "bao" + ], + "networks": [ + "default", + "ingress" + ], + "read_only": true, + "tmpfs": [ + "/tmp:mode=1777,size=16m" + ] + }, + "bao-permissions": { + "image": "openbao/openbao:2.4.4", + "profiles": [ + "tools" + ], + "user": "0:0", + "entrypoint": [ + "chown" + ], + "command": [ + "-R", + "100:1000", + "/openbao/data" + ], + "volumes": [ + "bao_data:/openbao/data" + ], + "network_mode": "none", + "cap_drop": [ + "ALL" + ], + "cap_add": [ + "CHOWN", + "DAC_OVERRIDE" + ], + "security_opt": [ + "no-new-privileges:true" + ], + "cpus": 0.25, + "mem_limit": "32m", + "restart": "no" + } + }, + "volumes": { + "postgres_data": {}, + "redis_data": {}, + "broker_data": {}, + "s3_data": {}, + "bao_data": {}, + "prometheus_data": {}, + "loki_data": {}, + "tempo_data": {}, + "grafana_data": {} + }, + "networks": { + "default": { + "internal": true + }, + "ingress": {} + } +} diff --git a/contracts/trust-matrix.md b/contracts/trust-matrix.md index 0c2a977..f53f0ba 100644 --- a/contracts/trust-matrix.md +++ b/contracts/trust-matrix.md @@ -1,37 +1,30 @@ -# Матрица доверия — заполнить в решении lab7 +# Матрица доверия lab7 -Это обязательная матрица покрытия. Для каждой строки укажите реальный порт, endpoint, CA, service identity, проверку SAN/ACL, срок жизни credentials и команду положительного/отрицательного теста. TLS шифрует и подтверждает сервер; mTLS также проверяет клиента. При доверенном CA всё равно нужно ограничивать разрешённую identity. Plaintext hop допустим только внутри закрытого локального transport/proxy boundary, без опубликованного обхода; границу и rationale нужно доказать. +CA — локальная `lab7-local-ca`, создаётся `scripts/bootstrap.py`. SAN перечислены ниже; ключ CA и Shamir shares находятся в закрытом каталоге оператора, который не монтируется сервисам. Leaf certificates действуют 24 часа, CA — 365 дней. Клиентская и серверная EKU заданы явно. Все серверные проверки включают CA и hostname; app не включает `InsecureSkipVerify`. -| Связь | Минимум на 3 | Дополнительно на 4 | Доказательство / конфиг | -| --- | --- | --- | --- | -| Клиент → ingress/LB | TLS, CA + hostname, Bearer | Проверка expiry/revocation сессии и общего login rate limit | TODO | -| LB → API (все реплики) | mTLS + allowlist LB identity | Ротация leaf без обхода проверки | TODO | -| API/worker → PostgreSQL | TLS verify-full + раздельные ограниченные DB roles | Ротация credentials; mTLS — допустимое усиление | TODO | -| API/worker → S3 | TLS + разные scoped credentials/prefix rights | Проверка denied prefix и ротация | TODO | -| API/worker → broker | TLS + разные producer/consumer identities и ACL | mTLS + ограничение допустимых SAN/ролей | TODO | -| API/worker → Collector | TLS; сеть/аутентификация запрещает произвольный ingest | mTLS с раздельными identities | TODO | -| Collector → telemetry backends | TLS + scoped backend credentials или mTLS | Ротация и проверка least privilege | TODO | -| Grafana → data sources | TLS с проверкой сервера, ограниченная identity | Отрицательные проверки доступа | TODO | -| API/worker/agent → OpenBao | TLS + сервисный auth + scoped policy | Renewal/expiry/revocation | TODO | -| Оператор → OpenBao/Grafana/S3 admin | TLS + authentication, доступ локальный/через защищённый канал | Разделение operator/application roles | TODO | -| API → Redis/Valkey, если используется | TLS + ACL/пароль с минимальными правами | Ротация; mTLS по возможности | TODO | -| Replication/кластерные каналы, если добавлены | TLS/mTLS либо подтверждённый шифрованный transport | Проверка заявленной модели отказа | TODO | +| Связь | Endpoint / транспорт | Identity и авторизация | Lifecycle | Проверка | +| --- | --- | --- | --- | --- | +| Клиент → LB | localhost:8443, TLS | SAN localhost/127.0.0.1; бизнес-API требует Bearer | Сессия 3600 s, logout/expiry | `make test test-security`, `scripts/prove-auth.py` | +| LB → API ×2 | api:8080, mTLS | SAN api; клиент только DNS SAN lb, CRL; TLS tickets выключены | Leaf 24 h; `renew-leaf.py lb` | `scripts/security.py`, `scripts/renew-leaf.py lb` | +| API/worker/dispatcher → PostgreSQL | postgres:5432, TLS verify-full | Сервер SAN postgres; разные динамические LOGIN roles, наследующие api_role/worker_role/dispatcher_role | DB lease 600 s, max 1800 s, renewal; отзыв закрывает DB sessions | `securityprobe` db; `experiment-worker.py` | +| API/worker → S3 | s3:9443, mTLS + SigV4 | SAN s3; Envoy допускает SAN api/worker/operator-client; S3 отдельно проверяет scoped access key и prefix | KV key живёт до ротации; token grant ограничивает время его использования приложением | `securityprobe` s3; incident rotation | +| Dispatcher/worker → broker | broker:5671, mTLS + EXTERNAL | SAN broker; CN клиента → отдельный RabbitMQ user; producer и consumer permissions различаются | Leaf 24 h; при инциденте старый CN user удаляется вместе с соединением | `securityprobe` amqp/amqp-tls; worker experiment | +| API/worker/dispatcher → Collector | otel-collector:9443, mTLS | Только DNS SAN api/worker/dispatcher; CRL в Envoy | Leaf 24 h; обновление CRL требует пересоздания proxy | `scripts/security.py` | +| Collector → Loki/Tempo | loki:9443/otlp, tempo:9444, mTLS | SAN otel-collector; Loki разрешает только POST /otlp/v1/logs; Tempo ingest отделён от query | Leaf 24 h | `scripts/prove-telemetry.py`, Envoy RBAC | +| Prometheus → Collector metrics | otel-collector:9444 и :9445, mTLS | Только SAN prometheus; backend listeners loopback :8889/:8888 | Leaf 24 h | `up` в TLS query, `prove-telemetry.py` | +| Grafana → data sources | prometheus/loki/tempo:9443, mTLS | Только SAN grafana на query-пути; Loki write закрыт; Tempo ingest недоступен Grafana; Prometheus write API не включён | Generated provisioning вне Git; leaf 24 h | `prove-telemetry.py`, Envoy policies | +| API/worker/dispatcher → OpenBao | openbao:8200, TLS + cert auth | Каждая auth role доверяет конкретному leaf; policy разрешает только собственный KV и DB role | Token 300 s, max 1800 s; renewal/re-auth; DB lease отдельно | `security.py`, `prove-sealed.py`, `experiment-worker.py` | +| Оператор → OpenBao | localhost:18200, TLS + operator cert auth | SAN localhost; scoped operator policy; первоначальный root token отзывается после bootstrap | Operator token 300 s, max 1800 s; unseal — 2 из 3 shares | `bao.py`, `bao.py unseal` | +| Оператор → Grafana | localhost:13443, HTTPS + пароль | SAN localhost; anonymous выключен | Пароль генерируется локально, не хранится в Compose environment | HTTPS login; generated provisioning | +| Оператор → S3 | s3:9443 через Compose tools, mTLS + admin SigV4 | SAN operator-client; отдельный admin key; административного внешнего S3-порта нет | Operator-only files | `compose.sh run --rm -T operator storage bootstrap` | +| API → Redis | redis:6380, TLS + Redis ACL | SAN redis; user api, только chat:* и ratelimit:*; default off; admin commands запрещены | Секрет выдаётся API через OpenBao; noeviction и AOF | Общий rate limit в `security.py` | +| Replication / cluster | N/A | Реплик DB/broker/SeaweedFS между контейнерами нет | Один Docker host | Ограничение модели отказов | -Нет выбранного компонента — отметьте N/A с причиной, а не оставляйте пустую строку. Внутренний публичный health listener, если нужен, минимален и изолирован; через него нельзя обращаться к бизнес-API без mTLS. Отдельно задокументируйте scrape/health/bootstrap каналы: они тоже не должны создавать обход. +## Локальные границы и административные действия -## Обязательные отрицательные проверки - -1. HTTPS с правильным CA/hostname проходит; с пустым/чужим CA и неправильным hostname — ошибка верификации, не «сервер недоступен». -2. LB→API: без client cert, с cert чужого CA, с запрещённой identity от доверенного CA — отказ; с разрешённой LB identity — успех. -3. На 4 повторите набор для broker и Collector; ограничение на network policy не заменяет проверку identity. -4. Пользователь вне чата не читает metadata/content и не прикрепляет чужой файл; поддельный X-User-Id не меняет Bearer identity. -5. OpenBao app policy читает только свой secret/path; запись policy и чтение чужого path запрещены. Worker не получает admin credentials API/БД/S3. -6. На 4 отзовите secret/cert и дождитесь контролируемого эффекта с учётом действующих соединений, TLS resumption и cache/TTL. Покажите новое подключение, иначе тест отзыва может ничего не проверить. - -## Bootstrap и эксплуатация - -В репозитории только открытые конфиги, policy templates и scripts. CA private key, leaf keys, root token, unseal shares и SecretID не коммитятся, не запекаются в образ и не печатаются в отчёте. Для локального стенда оператор создаёт их вне Git и передаёт через files/secrets с ограниченными правами. Опишите, кто имеет доступ к CA/unseal, что нужно после рестарта OpenBao и как восстановить работу. - -OpenBao не использует dev-mode. Выберите persistent storage и auth метод (например AppRole с короткоживущим SecretID/response wrapping, cert auth или иная обоснованная identity). Не подменяйте root token постоянной переменной BAO_TOKEN в приложении. Рабочие tokens имеют policy, TTL и renewal/re-auth сценарий. Статические KV secrets на 3 допустимы при реальном чтении через сервисную identity; динамические DB credentials — хорошее расширение, но не обязательное название технологии. - -Разрешённый компромисс учебного стенда: ручной operator init/unseal, если он безопасно документирован и не нужен при каждом обычном рестарте API. Автоматическое хранение всех unseal shares рядом с OpenBao без защиты не считается решённым bootstrap. +- Envoy использует общий network namespace только со своим backend. Backend слушает loopback; его plaintext port с bridge недоступен. SeaweedFS master/filer/volume/S3 работают внутри одного контейнера и используют loopback. +- API health listener :8081 привязан к 127.0.0.1 и принимает только `/health/live` и `/health/ready`. Бизнес-маршруты на нём возвращают 404. LB проверяет readiness через основной mTLS listener. +- Bootstrap/migrations выполняются одноразовыми operator tools. PostgreSQL Unix socket доступен только внутри его контейнера; эти действия требуют доступа оператора к Docker host. +- После restart/recreate родителя Envoy пересоздаётся, чтобы войти в актуальный namespace. Обычный restart sidecar недостаточен для надёжного восстановления этой границы. +- Worker certificate CN получает новую версию при каждом отзыве. Старый RabbitMQ user удаляется; новый CN не восстанавливает права старому сертификату. Envoy использует CRL, OpenBao — привязку к новому leaf и отзыв выданных tokens/leases. +- Обычные рабочие процессы не получают root token, unseal shares, CA key, migrator password или чужую identity. Scoped KV credentials сами по себе не становятся автоматически истекающими: украденный S3 key необходимо отозвать на S3, что делает incident script. diff --git a/evidence/admin-run.txt b/evidence/admin-run.txt new file mode 100644 index 0000000..ce7c81a --- /dev/null +++ b/evidence/admin-run.txt @@ -0,0 +1 @@ +PASS: Grafana HTTPS operator authentication and anonymous denial diff --git a/evidence/admin.json b/evidence/admin.json new file mode 100644 index 0000000..c2b3113 --- /dev/null +++ b/evidence/admin.json @@ -0,0 +1,5 @@ +{ + "grafana_verified_https": true, + "anonymous_user_api_denied": true, + "operator_authenticated": true +} diff --git a/evidence/all-replicas-run.txt b/evidence/all-replicas-run.txt new file mode 100644 index 0000000..60002ca --- /dev/null +++ b/evidence/all-replicas-run.txt @@ -0,0 +1 @@ +PASS: each physical API container stopped sequentially; session/write/idempotency preserved diff --git a/evidence/all-replicas.json b/evidence/all-replicas.json new file mode 100644 index 0000000..8c66272 --- /dev/null +++ b/evidence/all-replicas.json @@ -0,0 +1,14 @@ +[ + { + "stopped_container": "1292b6fbbd2a01262923490e3dd18bd64e253c326aa0834f9db53ae247fb18a9", + "recovery_seconds": 0.13170712487772107, + "original_session_valid": true, + "no_duplicate": true + }, + { + "stopped_container": "55c1a619dba179163f075c9a50e72eaf37bb9a87bb4a2b4ae968621c13108b3f", + "recovery_seconds": 1.378536291886121, + "original_session_valid": true, + "no_duplicate": true + } +] diff --git a/evidence/auth-run.txt b/evidence/auth-run.txt new file mode 100644 index 0000000..fdb6f84 --- /dev/null +++ b/evidence/auth-run.txt @@ -0,0 +1 @@ +PASS: legacy rehash, KDF measurements and session expiry diff --git a/evidence/auth.json b/evidence/auth.json new file mode 100644 index 0000000..fdd33ca --- /dev/null +++ b/evidence/auth.json @@ -0,0 +1,26 @@ +{ + "rehash_lab6_profile_on_login": true, + "kdf": { + "iterations": 3, + "max_ms": 181.749, + "max_parallel_kdf": 2, + "memory_kib": 65536, + "p50_ms": 103.545, + "p95_ms": 181.749, + "parallelism": 1, + "samples": 12 + }, + "login_dos_budget": { + "requests": 24, + "concurrency": 8, + "duration_seconds": 1.490289583336562, + "statuses": { + "401": 24 + }, + "p95_all_ms": 536.9945000857115, + "api_memory_limit_mib": 512, + "max_kdf_memory_mib_per_process": 128, + "max_kdf_parallel_per_process": 2 + }, + "session_ttl_2_seconds_expires": true +} diff --git a/evidence/broker-persistence.json b/evidence/broker-persistence.json new file mode 100644 index 0000000..eb42e76 --- /dev/null +++ b/evidence/broker-persistence.json @@ -0,0 +1,10 @@ +{ + "result": "PASS", + "attachment_id": "e969c5c6-9e00-489e-b0c9-8fcfb739c8c5", + "broker_hostname": "broker", + "old_container_id": "3a55196a154c99581deaa314187bcefc09d31d6a01a9f80c0659cd43fae9ef00", + "new_container_id": "be47e533bc90852ec009a15c8f2e9d8bfaed3dfeaa73e80e8a03191e2e3370c1", + "message_present_before_and_after": true, + "outbox_published_before_replacement": true, + "dispatcher_stopped_during_replacement": true +} diff --git a/evidence/broker-run.txt b/evidence/broker-run.txt new file mode 100644 index 0000000..8c21f42 --- /dev/null +++ b/evidence/broker-run.txt @@ -0,0 +1 @@ +PASS: confirmed message survived broker container replacement with the same volume and stable node identity diff --git a/evidence/cert-expiry-run.txt b/evidence/cert-expiry-run.txt new file mode 100644 index 0000000..aadc0c8 --- /dev/null +++ b/evidence/cert-expiry-run.txt @@ -0,0 +1 @@ +PASS: expired leaf with trusted CA and allowed LB SAN rejected diff --git a/evidence/cert-expiry.json b/evidence/cert-expiry.json new file mode 100644 index 0000000..40d5002 --- /dev/null +++ b/evidence/cert-expiry.json @@ -0,0 +1,5 @@ +{ + "trusted_ca": true, + "allowed_lb_san": true, + "expired_leaf_rejected": true +} diff --git a/evidence/clean-clone-run.txt b/evidence/clean-clone-run.txt new file mode 100644 index 0000000..ca9ed9b --- /dev/null +++ b/evidence/clean-clone-run.txt @@ -0,0 +1 @@ +PASS: fresh source-only snapshot with new secrets/volumes passes HTTPS smoke diff --git a/evidence/clean-clone.json b/evidence/clean-clone.json new file mode 100644 index 0000000..3a41f14 --- /dev/null +++ b/evidence/clean-clone.json @@ -0,0 +1,9 @@ +{ + "fresh_source_snapshot": true, + "new_ca_and_secrets": true, + "new_project_and_volumes": true, + "separate_application_image_tag": true, + "no_lab6_access": true, + "verified_https_smoke": "PASS: lab7 public API scenario; infrastructure and grading evidence remain separate.", + "duration_seconds": 21.36097679194063 +} diff --git a/evidence/contracts.txt b/evidence/contracts.txt new file mode 100644 index 0000000..ed9cd47 --- /dev/null +++ b/evidence/contracts.txt @@ -0,0 +1,4 @@ +.venv/bin/python scripts/check_contract.py +PASS: offline skeleton structure, local refs and Python syntax; application NOT tested. +.venv/bin/python scripts/validate_openapi.py +PASS: full OpenAPI 3.1 and JSON Schema validation; application NOT tested. diff --git a/evidence/environment.json b/evidence/environment.json new file mode 100644 index 0000000..ff1bb89 --- /dev/null +++ b/evidence/environment.json @@ -0,0 +1,346 @@ +{ + "timestamp": "2026-11-17T14:23:20.326528+00:00", + "base_commit": "20afc2c790d3e72f6b7666376c9834c1ced1a405", + "working_tree_changes": true, + "system": "macOS-26.5.1-arm64-arm-64bit-Mach-O", + "architecture": "arm64", + "docker": "29.7.2", + "compose": "5.3.1", + "containers": [ + { + "name": "/mtusi2026lab7-api-1", + "image_id": "sha256:c207d41316af63e9dd59e8f6794c59664ea2257c120c12f4663699a6015f941f", + "started_at": "2026-11-17T14:21:21.899518257Z", + "user": "65532:65532", + "memory_limit": 536870912, + "nano_cpus": 1000000000, + "read_only": true, + "cap_drop": [ + "ALL" + ], + "security_opt": [ + "no-new-privileges:true" + ], + "port_bindings": {} + }, + { + "name": "/mtusi2026lab7-api-2", + "image_id": "sha256:c207d41316af63e9dd59e8f6794c59664ea2257c120c12f4663699a6015f941f", + "started_at": "2026-11-17T14:21:22.03496334Z", + "user": "65532:65532", + "memory_limit": 536870912, + "nano_cpus": 1000000000, + "read_only": true, + "cap_drop": [ + "ALL" + ], + "security_opt": [ + "no-new-privileges:true" + ], + "port_bindings": {} + }, + { + "name": "/mtusi2026lab7-broker-1", + "image_id": "sha256:32962eea9bf99928c2fc541a7defb78c7feca9df0434c5dfb7b539d61e1af524", + "started_at": "2026-11-17T14:09:35.819957347Z", + "user": "", + "memory_limit": 536870912, + "nano_cpus": 500000000, + "read_only": false, + "cap_drop": null, + "security_opt": [ + "no-new-privileges:true" + ], + "port_bindings": {} + }, + { + "name": "/mtusi2026lab7-dispatcher-1", + "image_id": "sha256:c207d41316af63e9dd59e8f6794c59664ea2257c120c12f4663699a6015f941f", + "started_at": "2026-11-17T14:21:21.895641548Z", + "user": "65532:65532", + "memory_limit": 536870912, + "nano_cpus": 250000000, + "read_only": true, + "cap_drop": [ + "ALL" + ], + "security_opt": [ + "no-new-privileges:true" + ], + "port_bindings": {} + }, + { + "name": "/mtusi2026lab7-grafana-1", + "image_id": "sha256:0615ec4fbd4072004b7d86b46ded63013a74b7d6eb1119273dd89e8fa25c13cf", + "started_at": "2026-11-17T14:09:32.193343262Z", + "user": "472", + "memory_limit": 402653184, + "nano_cpus": 500000000, + "read_only": true, + "cap_drop": [ + "ALL" + ], + "security_opt": [ + "no-new-privileges:true" + ], + "port_bindings": { + "3000/tcp": [ + { + "HostIp": "127.0.0.1", + "HostPort": "13443" + } + ] + } + }, + { + "name": "/mtusi2026lab7-lb-1", + "image_id": "sha256:96ec3c75a9264ba94e733eeacb44051ed6e7b79bb77a73c2c36f236b6423f1f0", + "started_at": "2026-11-17T14:09:38.225323208Z", + "user": "haproxy", + "memory_limit": 134217728, + "nano_cpus": 500000000, + "read_only": true, + "cap_drop": [ + "ALL" + ], + "security_opt": [ + "no-new-privileges:true" + ], + "port_bindings": { + "8443/tcp": [ + { + "HostIp": "127.0.0.1", + "HostPort": "8443" + } + ] + } + }, + { + "name": "/mtusi2026lab7-loki-1", + "image_id": "sha256:b43c257ccfc925d4a50b8717cac61a3b51df8c48d01847936057e5c4075a24a7", + "started_at": "2026-11-17T14:09:31.680025386Z", + "user": "10001", + "memory_limit": 402653184, + "nano_cpus": 500000000, + "read_only": true, + "cap_drop": [ + "ALL" + ], + "security_opt": [ + "no-new-privileges:true" + ], + "port_bindings": {} + }, + { + "name": "/mtusi2026lab7-loki-tls-1", + "image_id": "sha256:08acc53db6f4ac46e77e36b203c5254b4a2549869314bb54cf97aab81baa4566", + "started_at": "2026-11-17T14:09:37.32076875Z", + "user": "65532:65532", + "memory_limit": 134217728, + "nano_cpus": 250000000, + "read_only": true, + "cap_drop": [ + "ALL" + ], + "security_opt": [ + "no-new-privileges:true" + ], + "port_bindings": {} + }, + { + "name": "/mtusi2026lab7-openbao-1", + "image_id": "sha256:595c83b42614a4d2b044608e4593c05b019c5db25bc9c185d8fff3ac96c03ddd", + "started_at": "2026-11-17T14:09:31.746126136Z", + "user": "100:1000", + "memory_limit": 268435456, + "nano_cpus": 500000000, + "read_only": true, + "cap_drop": [ + "ALL" + ], + "security_opt": [ + "no-new-privileges:true" + ], + "port_bindings": { + "8200/tcp": [ + { + "HostIp": "127.0.0.1", + "HostPort": "18200" + } + ] + } + }, + { + "name": "/mtusi2026lab7-otel-collector-1", + "image_id": "sha256:ffc818ee108d0b934fd14207fd87ff247c9b64f344ed349d0b66c166c18a2312", + "started_at": "2026-11-17T14:09:31.295292678Z", + "user": "10001:10001", + "memory_limit": 268435456, + "nano_cpus": 500000000, + "read_only": true, + "cap_drop": [ + "ALL" + ], + "security_opt": [ + "no-new-privileges:true" + ], + "port_bindings": {} + }, + { + "name": "/mtusi2026lab7-otel-collector-tls-1", + "image_id": "sha256:08acc53db6f4ac46e77e36b203c5254b4a2549869314bb54cf97aab81baa4566", + "started_at": "2026-11-17T14:17:14.918988212Z", + "user": "65532:65532", + "memory_limit": 134217728, + "nano_cpus": 250000000, + "read_only": true, + "cap_drop": [ + "ALL" + ], + "security_opt": [ + "no-new-privileges:true" + ], + "port_bindings": {} + }, + { + "name": "/mtusi2026lab7-postgres-1", + "image_id": "sha256:05aa77262db9277fde6374bb2f901c9604b8a9373029e1a7993440c6aae48736", + "started_at": "2026-11-17T14:09:31.375696886Z", + "user": "", + "memory_limit": 1073741824, + "nano_cpus": 2000000000, + "read_only": false, + "cap_drop": null, + "security_opt": [ + "no-new-privileges:true" + ], + "port_bindings": {} + }, + { + "name": "/mtusi2026lab7-prometheus-1", + "image_id": "sha256:e04987641b57704f408e31f6373ba4138ecf27cb4a165d24aabb6c38fddd0f06", + "started_at": "2026-11-17T14:09:31.215812844Z", + "user": "nobody", + "memory_limit": 402653184, + "nano_cpus": 500000000, + "read_only": true, + "cap_drop": [ + "ALL" + ], + "security_opt": [ + "no-new-privileges:true" + ], + "port_bindings": {} + }, + { + "name": "/mtusi2026lab7-prometheus-tls-1", + "image_id": "sha256:08acc53db6f4ac46e77e36b203c5254b4a2549869314bb54cf97aab81baa4566", + "started_at": "2026-11-17T14:09:37.25840775Z", + "user": "65532:65532", + "memory_limit": 134217728, + "nano_cpus": 250000000, + "read_only": true, + "cap_drop": [ + "ALL" + ], + "security_opt": [ + "no-new-privileges:true" + ], + "port_bindings": {} + }, + { + "name": "/mtusi2026lab7-redis-1", + "image_id": "sha256:489a769e0a54dc245896132a15c72d2bc7296719954da99398f43a3877e5f8eb", + "started_at": "2026-11-17T14:09:31.388634636Z", + "user": "", + "memory_limit": 134217728, + "nano_cpus": 250000000, + "read_only": false, + "cap_drop": null, + "security_opt": [ + "no-new-privileges:true" + ], + "port_bindings": {} + }, + { + "name": "/mtusi2026lab7-s3-1", + "image_id": "sha256:57877e32bb90073b37dc109b438ba66e84fd5ed9ced5f5c04f56444ef243ce59", + "started_at": "2026-11-17T14:17:14.280459003Z", + "user": "", + "memory_limit": 536870912, + "nano_cpus": 1000000000, + "read_only": false, + "cap_drop": null, + "security_opt": [ + "no-new-privileges:true" + ], + "port_bindings": {} + }, + { + "name": "/mtusi2026lab7-s3-tls-1", + "image_id": "sha256:08acc53db6f4ac46e77e36b203c5254b4a2549869314bb54cf97aab81baa4566", + "started_at": "2026-11-17T14:17:14.883127462Z", + "user": "65532:65532", + "memory_limit": 134217728, + "nano_cpus": 250000000, + "read_only": true, + "cap_drop": [ + "ALL" + ], + "security_opt": [ + "no-new-privileges:true" + ], + "port_bindings": {} + }, + { + "name": "/mtusi2026lab7-tempo-1", + "image_id": "sha256:65a5789759435f1ef696f1953258b9bbdb18eb571d5ce711ff812d2e128288a4", + "started_at": "2026-11-17T14:09:31.447828553Z", + "user": "10001:10001", + "memory_limit": 402653184, + "nano_cpus": 500000000, + "read_only": true, + "cap_drop": [ + "ALL" + ], + "security_opt": [ + "no-new-privileges:true" + ], + "port_bindings": {} + }, + { + "name": "/mtusi2026lab7-tempo-tls-1", + "image_id": "sha256:08acc53db6f4ac46e77e36b203c5254b4a2549869314bb54cf97aab81baa4566", + "started_at": "2026-11-17T14:09:37.338702Z", + "user": "65532:65532", + "memory_limit": 134217728, + "nano_cpus": 250000000, + "read_only": true, + "cap_drop": [ + "ALL" + ], + "security_opt": [ + "no-new-privileges:true" + ], + "port_bindings": {} + }, + { + "name": "/mtusi2026lab7-worker-1", + "image_id": "sha256:c207d41316af63e9dd59e8f6794c59664ea2257c120c12f4663699a6015f941f", + "started_at": "2026-11-17T14:21:21.870546215Z", + "user": "65532:65532", + "memory_limit": 1073741824, + "nano_cpus": 1000000000, + "read_only": true, + "cap_drop": [ + "ALL" + ], + "security_opt": [ + "no-new-privileges:true" + ], + "port_bindings": {} + } + ], + "docker_cpus": 14, + "docker_memory_bytes": 8320954368 +} diff --git a/evidence/failover-first.txt b/evidence/failover-first.txt new file mode 100644 index 0000000..e66e973 --- /dev/null +++ b/evidence/failover-first.txt @@ -0,0 +1,4 @@ +python3 tests/resilience.py --mode failover --action-script "scripts/stop-one.sh" --base-url "https://localhost:8443" --ca-file "/Users/a7tech173/development/gitea.privcloud.tech/mtusi2026/mtusi2026lab7/secrets/lab7/lb/ca.crt" +Executing explicit failover hook; hook output suppressed to avoid leaking secrets. +{"observed_instances_before": 2, "successful_writes": 122, "failed_attempts": 1, "first_recovery_seconds": 1.804} +PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. diff --git a/evidence/failover-second.txt b/evidence/failover-second.txt new file mode 100644 index 0000000..c38a5ac --- /dev/null +++ b/evidence/failover-second.txt @@ -0,0 +1,4 @@ +python3 tests/resilience.py --mode failover --action-script "scripts/stop-one.sh" --base-url "https://localhost:8443" --ca-file "/Users/a7tech173/development/gitea.privcloud.tech/mtusi2026/mtusi2026lab7/secrets/lab7/lb/ca.crt" +Executing explicit failover hook; hook output suppressed to avoid leaking secrets. +{"observed_instances_before": 2, "successful_writes": 123, "failed_attempts": 1, "first_recovery_seconds": 1.338} +PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. diff --git a/evidence/failover-targets.json b/evidence/failover-targets.json new file mode 100644 index 0000000..b941fed --- /dev/null +++ b/evidence/failover-targets.json @@ -0,0 +1,10 @@ +[ + { + "instance_alias": "bca289fa-980d-4efa-b168-85176e3c952b", + "container_id": "d6fda7f70c3273d02dc57c920ab696da50e151e22af44ed8a285ea6913902da1" + }, + { + "instance_alias": "fb217a37-a62d-47f8-90a2-f6e5a7340541", + "container_id": "d6fda7f70c3273d02dc57c920ab696da50e151e22af44ed8a285ea6913902da1" + } +] diff --git a/evidence/final-smoke.txt b/evidence/final-smoke.txt new file mode 100644 index 0000000..6f8e92f --- /dev/null +++ b/evidence/final-smoke.txt @@ -0,0 +1,4 @@ +python3 tests/smoke.py --base-url "https://localhost:8443" --ca-file "/Users/a7tech173/development/gitea.privcloud.tech/mtusi2026/mtusi2026lab7/secrets/lab7/lb/ca.crt" +PASS: lab7 public API scenario; infrastructure and grading evidence remain separate. +python3 tests/security.py --base-url "https://localhost:8443" --ca-file "/Users/a7tech173/development/gitea.privcloud.tech/mtusi2026/mtusi2026lab7/secrets/lab7/lb/ca.crt" +PASS: trusted HTTPS works; untrusted CA rejected. This does NOT verify internal mTLS, OpenBao or hardening. diff --git a/evidence/go-checks.txt b/evidence/go-checks.txt new file mode 100644 index 0000000..8807fa4 --- /dev/null +++ b/evidence/go-checks.txt @@ -0,0 +1,39 @@ +cd service && go build ./... +cd service && go vet ./... +cd service && go test ./internal/... +? gitea.privcloud.tech/mtusi2026/internal/adapter/controller/http [no test files] +? gitea.privcloud.tech/mtusi2026/internal/adapter/repository/cache/outside [no test files] +ok gitea.privcloud.tech/mtusi2026/internal/adapter/repository/storage 0.478s +? gitea.privcloud.tech/mtusi2026/internal/config [no test files] +? gitea.privcloud.tech/mtusi2026/internal/dto [no test files] +? gitea.privcloud.tech/mtusi2026/internal/gen/contracts [no test files] +? gitea.privcloud.tech/mtusi2026/internal/pkg/apperr [no test files] +? gitea.privcloud.tech/mtusi2026/internal/pkg/auth/jwtprovider [no test files] +? gitea.privcloud.tech/mtusi2026/internal/pkg/broker [no test files] +? gitea.privcloud.tech/mtusi2026/internal/pkg/cache [no test files] +ok gitea.privcloud.tech/mtusi2026/internal/pkg/encrypting (cached) +? gitea.privcloud.tech/mtusi2026/internal/pkg/fault [no test files] +? gitea.privcloud.tech/mtusi2026/internal/pkg/imageprocessor [no test files] +? gitea.privcloud.tech/mtusi2026/internal/pkg/migrator [no test files] +? gitea.privcloud.tech/mtusi2026/internal/pkg/objectstore [no test files] +? gitea.privcloud.tech/mtusi2026/internal/pkg/openbao [no test files] +? gitea.privcloud.tech/mtusi2026/internal/pkg/postgres [no test files] +? gitea.privcloud.tech/mtusi2026/internal/pkg/postgres/txmanager [no test files] +? gitea.privcloud.tech/mtusi2026/internal/pkg/ratelimit [no test files] +? gitea.privcloud.tech/mtusi2026/internal/pkg/rest/framework [no test files] +? gitea.privcloud.tech/mtusi2026/internal/pkg/rest/headers [no test files] +ok gitea.privcloud.tech/mtusi2026/internal/pkg/rest/middleware (cached) +? gitea.privcloud.tech/mtusi2026/internal/pkg/rest/router [no test files] +? gitea.privcloud.tech/mtusi2026/internal/pkg/telemetry [no test files] +? gitea.privcloud.tech/mtusi2026/internal/pkg/transportsecurity [no test files] +? gitea.privcloud.tech/mtusi2026/internal/pkg/types [no test files] +? gitea.privcloud.tech/mtusi2026/internal/pkg/types/authprovider [no test files] +? gitea.privcloud.tech/mtusi2026/internal/pkg/types/environment [no test files] +? gitea.privcloud.tech/mtusi2026/internal/pkg/types/role [no test files] +? gitea.privcloud.tech/mtusi2026/internal/pkg/validator [no test files] +? gitea.privcloud.tech/mtusi2026/internal/service/attachment [no test files] +? gitea.privcloud.tech/mtusi2026/internal/service/auth [no test files] +? gitea.privcloud.tech/mtusi2026/internal/service/chat [no test files] +? gitea.privcloud.tech/mtusi2026/internal/service/observation [no test files] +? gitea.privcloud.tech/mtusi2026/internal/service/outbox [no test files] +? gitea.privcloud.tech/mtusi2026/internal/service/user [no test files] diff --git a/evidence/go-tests.txt b/evidence/go-tests.txt new file mode 100644 index 0000000..428618f --- /dev/null +++ b/evidence/go-tests.txt @@ -0,0 +1,36 @@ +? gitea.privcloud.tech/mtusi2026/internal/adapter/controller/http [no test files] +? gitea.privcloud.tech/mtusi2026/internal/adapter/repository/cache/outside [no test files] +ok gitea.privcloud.tech/mtusi2026/internal/adapter/repository/storage 0.790s +? gitea.privcloud.tech/mtusi2026/internal/config [no test files] +? gitea.privcloud.tech/mtusi2026/internal/dto [no test files] +? gitea.privcloud.tech/mtusi2026/internal/gen/contracts [no test files] +? gitea.privcloud.tech/mtusi2026/internal/pkg/apperr [no test files] +? gitea.privcloud.tech/mtusi2026/internal/pkg/auth/jwtprovider [no test files] +? gitea.privcloud.tech/mtusi2026/internal/pkg/broker [no test files] +? gitea.privcloud.tech/mtusi2026/internal/pkg/cache [no test files] +ok gitea.privcloud.tech/mtusi2026/internal/pkg/encrypting 1.049s +? gitea.privcloud.tech/mtusi2026/internal/pkg/fault [no test files] +? gitea.privcloud.tech/mtusi2026/internal/pkg/imageprocessor [no test files] +? gitea.privcloud.tech/mtusi2026/internal/pkg/migrator [no test files] +? gitea.privcloud.tech/mtusi2026/internal/pkg/objectstore [no test files] +? gitea.privcloud.tech/mtusi2026/internal/pkg/openbao [no test files] +? gitea.privcloud.tech/mtusi2026/internal/pkg/postgres [no test files] +? gitea.privcloud.tech/mtusi2026/internal/pkg/postgres/txmanager [no test files] +? gitea.privcloud.tech/mtusi2026/internal/pkg/ratelimit [no test files] +? gitea.privcloud.tech/mtusi2026/internal/pkg/rest/framework [no test files] +? gitea.privcloud.tech/mtusi2026/internal/pkg/rest/headers [no test files] +ok gitea.privcloud.tech/mtusi2026/internal/pkg/rest/middleware 0.437s +? gitea.privcloud.tech/mtusi2026/internal/pkg/rest/router [no test files] +? gitea.privcloud.tech/mtusi2026/internal/pkg/telemetry [no test files] +? gitea.privcloud.tech/mtusi2026/internal/pkg/transportsecurity [no test files] +? gitea.privcloud.tech/mtusi2026/internal/pkg/types [no test files] +? gitea.privcloud.tech/mtusi2026/internal/pkg/types/authprovider [no test files] +? gitea.privcloud.tech/mtusi2026/internal/pkg/types/environment [no test files] +? gitea.privcloud.tech/mtusi2026/internal/pkg/types/role [no test files] +? gitea.privcloud.tech/mtusi2026/internal/pkg/validator [no test files] +? gitea.privcloud.tech/mtusi2026/internal/service/attachment [no test files] +? gitea.privcloud.tech/mtusi2026/internal/service/auth [no test files] +? gitea.privcloud.tech/mtusi2026/internal/service/chat [no test files] +? gitea.privcloud.tech/mtusi2026/internal/service/observation [no test files] +? gitea.privcloud.tech/mtusi2026/internal/service/outbox [no test files] +? gitea.privcloud.tech/mtusi2026/internal/service/user [no test files] diff --git a/evidence/govulncheck-before.txt b/evidence/govulncheck-before.txt new file mode 100644 index 0000000..88605d1 --- /dev/null +++ b/evidence/govulncheck-before.txt @@ -0,0 +1,510 @@ +=== Symbol Results === + +Vulnerability #1: GO-2026-6859 + HTTP/1 server crash due to MaxBytesReader header race in net/http + More info: https://pkg.go.dev/vuln/GO-2026-6859 + Standard library + Found in: net/http@go1.27 + Fixed in: net/http@go1.27.4 + Example traces found: + #1: internal/pkg/telemetry/http.go:6:2: telemetry.init calls middleware.init, which calls http.CanonicalHeaderKey + #2: cmd/benchmark/main.go:78:22: benchmark.client.setup calls http.Client.Do + #3: cmd/app/storage.go:64:24: app.runStorage calls http.Client.Get + #4: internal/pkg/auth/jwtprovider/manager.go:42:8: jwtprovider.NewCookieManager calls http.Cookie.Valid + #5: internal/pkg/telemetry/http.go:6:2: telemetry.init calls middleware.init, which eventually calls http.HandleFunc + #6: internal/pkg/rest/router/router.go:73:15: router.router.ServeHTTP calls chi.Mux.ServeHTTP, which calls http.HandlerFunc.ServeHTTP + #7: internal/pkg/rest/framework/framework.go:49:15: framework.JSONResponse calls middleware.compressResponseWriter.WriteHeader, which calls http.Header.Add + #8: internal/pkg/rest/framework/framework.go:49:15: framework.JSONResponse calls middleware.compressResponseWriter.WriteHeader, which calls http.Header.Del + #9: internal/pkg/rest/framework/framework.go:220:29: framework.ParseToken calls http.Header.Get + #10: internal/adapter/controller/http/attachmentHandler.go:103:16: http.AttachmentHandler.Get calls http.Header.Set + #11: internal/adapter/controller/http/chat.go:184:30: http.ChatHandler.SendMessage calls http.Header.Values + #12: cmd/benchmark/main.go:76:27: benchmark.client.setup calls http.NewRequest + #13: internal/pkg/openbao/client.go:39:40: openbao.client.call calls http.NewRequestWithContext + #14: internal/pkg/telemetry/telemetry.go:93:112: telemetry.Init calls metric.NewPeriodicReader, which eventually calls http.ParseTime + #15: internal/pkg/objectstore/s3.go:66:27: objectstore.Store.Get calls s3.Client.GetObject, which eventually calls http.ReadRequest + #16: internal/service/attachment/reconcile.go:203:2: attachment.Service.Restore calls http2.requestBody.Close, which eventually calls http.ReadResponse + #17: internal/adapter/controller/http/auth.go:38:39: http.AuthHandler.Login calls http.Request.BasicAuth + #18: internal/pkg/auth/jwtprovider/manager.go:111:20: jwtprovider.manager.RefreshToken calls http.Request.Cookie + #19: cmd/benchmark/main.go:77:18: benchmark.client.setup calls http.Request.SetBasicAuth + #20: internal/service/attachment/reconcile.go:203:2: attachment.Service.Restore calls http2.requestBody.Close, which eventually calls http.Request.Write + #21: internal/pkg/objectstore/s3.go:66:27: objectstore.Store.Get calls s3.Client.GetObject, which eventually calls http.Response.Write + #22: internal/adapter/controller/http/controller.go:89:21: http.server.Stop calls http.Server.Close + #23: cmd/app/main.go:156:44: app.run calls http.Server.ListenAndServe + #24: internal/adapter/controller/http/controller.go:71:27: http.Start calls http.Server.Serve + #25: internal/adapter/controller/http/controller.go:87:29: http.server.Stop calls http.Server.Shutdown + #26: internal/pkg/auth/jwtprovider/manager.go:87:17: jwtprovider.manager.Clear calls http.SetCookie + #27: internal/pkg/telemetry/telemetry.go:84:28: telemetry.Init calls otlptracehttp.New, which eventually calls http.Transport.Clone + #28: internal/pkg/objectstore/s3.go:66:27: objectstore.Store.Get calls s3.Client.GetObject, which eventually calls http.Transport.CloseIdleConnections + #29: internal/pkg/objectstore/s3.go:66:27: objectstore.Store.Get calls s3.Client.GetObject, which eventually calls http.Transport.RoundTrip + #30: internal/service/attachment/reconcile.go:203:2: attachment.Service.Restore calls http.body.Close + #31: internal/service/attachment/reconcile.go:178:20: attachment.Service.Restore calls io.Copy, which eventually calls http.body.Read + #32: internal/service/attachment/reconcile.go:203:2: attachment.Service.Restore calls http.bodyEOFSignal.Close + #33: internal/service/attachment/reconcile.go:178:20: attachment.Service.Restore calls io.Copy, which eventually calls http.bodyEOFSignal.Read + #34: internal/service/attachment/reconcile.go:178:20: attachment.Service.Restore calls io.Copy, which eventually calls http.bodyLocked.Read + #35: internal/service/attachment/reconcile.go:178:20: attachment.Service.Restore calls io.Copy, which eventually calls http.bufioFlushWriter.Write + #36: internal/service/attachment/reconcile.go:203:2: attachment.Service.Restore calls http.cancelTimerBody.Close + #37: internal/service/attachment/reconcile.go:178:20: attachment.Service.Restore calls io.Copy, which eventually calls http.cancelTimerBody.Read + #38: internal/service/attachment/reconcile.go:203:2: attachment.Service.Restore calls http2.transportResponseBody.Close, which eventually calls http.checkConnErrorWriter.Write + #39: internal/service/attachment/reconcile.go:203:2: attachment.Service.Restore calls http2.transportResponseBody.Close, which eventually calls http.chunkWriter.Write + #40: internal/service/attachment/reconcile.go:178:20: attachment.Service.Restore calls io.Copy, which eventually calls http.connReader.Read + #41: internal/service/attachment/reconcile.go:203:2: attachment.Service.Restore calls http.expectContinueReader.Close + #42: internal/service/attachment/reconcile.go:178:20: attachment.Service.Restore calls io.Copy, which eventually calls http.expectContinueReader.Read + #43: internal/service/attachment/reconcile.go:203:2: attachment.Service.Restore calls http.gzipReader.Close + #44: internal/service/attachment/reconcile.go:178:20: attachment.Service.Restore calls io.Copy, which eventually calls http.gzipReader.Read + #45: internal/pkg/encrypting/argon2id.go:70:23: encrypting.encryptor.EncryptSecretUsingArgon2id calls rand.Read, which eventually calls http.loggingConn.Read + #46: internal/pkg/apperr/error.go:79:21: apperr.AppError.Error calls time.parseErrors.Error, which eventually calls http.loggingConn.Write + #47: internal/service/attachment/reconcile.go:203:2: attachment.Service.Restore calls http.maxBytesReader.Close + #48: internal/service/attachment/reconcile.go:178:20: attachment.Service.Restore calls io.Copy, which eventually calls http.maxBytesReader.Read + #49: internal/service/attachment/reconcile.go:203:2: attachment.Service.Restore calls http2.requestBody.Close, which eventually calls http.onceCloseListener.Close + #50: internal/service/attachment/reconcile.go:178:20: attachment.Service.Restore calls io.Copy, which eventually calls http.persistConn.Read + #51: internal/service/attachment/reconcile.go:178:20: attachment.Service.Restore calls io.Copy, which eventually calls http.persistConnWriter.ReadFrom + #52: internal/service/attachment/reconcile.go:203:2: attachment.Service.Restore calls http2.transportResponseBody.Close, which eventually calls http.persistConnWriter.Write + #53: internal/service/attachment/reconcile.go:203:2: attachment.Service.Restore calls http.readTrackingBody.Close + #54: internal/service/attachment/reconcile.go:178:20: attachment.Service.Restore calls io.Copy, which eventually calls http.readTrackingBody.Read + #55: internal/service/attachment/reconcile.go:178:20: attachment.Service.Restore calls io.Copy, which eventually calls http.readWriteCloserBody.Read + #56: internal/service/attachment/reconcile.go:178:20: attachment.Service.Restore calls io.Copy, which eventually calls http.response.ReadFrom + #57: cmd/benchmark/main.go:234:15: benchmark.main calls fmt.Fprintln, which calls http.response.Write + #58: internal/pkg/rest/framework/framework.go:49:15: framework.JSONResponse calls http.response.WriteHeader + #59: internal/pkg/ratelimit/redis.go:14:29: ratelimit.init calls redis.NewScript, which eventually calls http.response.WriteString + #60: internal/pkg/apperr/error.go:79:21: apperr.AppError.Error calls http.transportReadFromServerError.Error + +Vulnerability #2: GO-2026-6809 + Distrust compromised intermediate certificates in crypto/x509 + More info: https://pkg.go.dev/vuln/GO-2026-6809 + Standard library + Found in: crypto/x509@go1.27 + Fixed in: crypto/x509@go1.27.3 + Example traces found: + #1: internal/pkg/broker/rabbitmq.go:56:28: broker.Rabbit.connect calls amqp091.DialConfig, which eventually calls x509.Certificate.Verify + +Vulnerability #3: GO-2026-6700 + Pre-negotiation frame limit is not enforced to 4KB in + github.com/rabbitmq/amqp091-go + More info: https://pkg.go.dev/vuln/GO-2026-6700 + Module: github.com/rabbitmq/amqp091-go + Found in: github.com/rabbitmq/amqp091-go@v1.10.0 + Fixed in: github.com/rabbitmq/amqp091-go@v1.14.0 + Example traces found: + #1: internal/pkg/broker/rabbitmq.go:56:28: broker.Rabbit.connect calls amqp091.DialConfig + +Vulnerability #4: GO-2026-6617 + HTTP/2 server crash due to HPACK encoder race in net/http + More info: https://pkg.go.dev/vuln/GO-2026-6617 + Standard library + Found in: net/http/internal/http2@go1.27 + Fixed in: net/http/internal/http2@go1.27.2 + Example traces found: + #1: cmd/benchmark/main.go:78:22: benchmark.client.setup calls http.Client.Do, which eventually calls http2.NetHTTPClientConn.Close + #2: cmd/benchmark/main.go:78:22: benchmark.client.setup calls http.Client.Do, which eventually calls http2.NetHTTPClientConn.Close + #3: cmd/benchmark/main.go:78:22: benchmark.client.setup calls http.Client.Do, which eventually calls http2.NetHTTPClientConn.RoundTrip + #4: cmd/benchmark/main.go:78:22: benchmark.client.setup calls http.Client.Do, which eventually calls http2.NetHTTPClientConn.RoundTrip + #5: internal/adapter/controller/http/controller.go:87:29: http.server.Stop calls http.Server.Shutdown, which calls http2.Server.GracefulShutdown + #6: internal/adapter/controller/http/controller.go:87:29: http.server.Stop calls http.Server.Shutdown, which calls http2.Server.GracefulShutdown + #7: internal/adapter/controller/http/controller.go:71:27: http.Start calls http.Server.Serve, which eventually calls http2.Server.ServeConn + #8: internal/adapter/controller/http/controller.go:71:27: http.Start calls http.Server.Serve, which eventually calls http2.Server.ServeConn + #9: cmd/benchmark/main.go:78:22: benchmark.client.setup calls http.Client.Do, which eventually calls http2.Transport.AddConn + #10: cmd/benchmark/main.go:78:22: benchmark.client.setup calls http.Client.Do, which eventually calls http2.Transport.AddConn + #11: internal/pkg/objectstore/s3.go:66:27: objectstore.Store.Get calls s3.Client.GetObject, which eventually calls http2.Transport.CloseIdleConnections + #12: internal/pkg/objectstore/s3.go:66:27: objectstore.Store.Get calls s3.Client.GetObject, which eventually calls http2.Transport.CloseIdleConnections + #13: cmd/benchmark/main.go:78:22: benchmark.client.setup calls http.Client.Do, which eventually calls http2.Transport.NewClientConn + #14: cmd/benchmark/main.go:78:22: benchmark.client.setup calls http.Client.Do, which eventually calls http2.Transport.NewClientConn + #15: cmd/benchmark/main.go:78:22: benchmark.client.setup calls http.Client.Do, which eventually calls http2.Transport.RoundTrip + #16: cmd/benchmark/main.go:78:22: benchmark.client.setup calls http.Client.Do, which eventually calls http2.Transport.RoundTrip + #17: internal/pkg/rest/framework/framework.go:49:15: framework.JSONResponse calls http.response.WriteHeader, which eventually calls http2.bufferedWriterTimeoutWriter.Write + #18: internal/pkg/rest/framework/framework.go:49:15: framework.JSONResponse calls http.response.WriteHeader, which eventually calls http2.bufferedWriterTimeoutWriter.Write + #19: internal/pkg/rest/framework/framework.go:49:15: framework.JSONResponse calls http.response.WriteHeader, which eventually calls http2.chunkWriter.Write + #20: internal/pkg/rest/framework/framework.go:49:15: framework.JSONResponse calls http.response.WriteHeader, which eventually calls http2.chunkWriter.Write + #21: internal/service/attachment/reconcile.go:203:2: attachment.Service.Restore calls http2.gzipReader.Close + #22: internal/service/attachment/reconcile.go:203:2: attachment.Service.Restore calls http2.gzipReader.Close + #23: internal/service/attachment/reconcile.go:178:20: attachment.Service.Restore calls io.Copy, which eventually calls http2.gzipReader.Read + #24: internal/service/attachment/reconcile.go:178:20: attachment.Service.Restore calls io.Copy, which eventually calls http2.gzipReader.Read + #25: internal/service/attachment/reconcile.go:203:2: attachment.Service.Restore calls http2.requestBody.Close + #26: internal/service/attachment/reconcile.go:203:2: attachment.Service.Restore calls http2.requestBody.Close + #27: cmd/benchmark/main.go:234:15: benchmark.main calls fmt.Fprintln, which calls http2.responseWriter.Write + #28: cmd/benchmark/main.go:234:15: benchmark.main calls fmt.Fprintln, which calls http2.responseWriter.Write + #29: internal/pkg/rest/framework/framework.go:49:15: framework.JSONResponse calls http2.responseWriter.WriteHeader + #30: internal/pkg/rest/framework/framework.go:49:15: framework.JSONResponse calls http2.responseWriter.WriteHeader + #31: internal/pkg/ratelimit/redis.go:14:29: ratelimit.init calls redis.NewScript, which eventually calls http2.responseWriter.WriteString + #32: internal/pkg/ratelimit/redis.go:14:29: ratelimit.init calls redis.NewScript, which eventually calls http2.responseWriter.WriteString + #33: internal/pkg/rest/framework/framework.go:49:15: framework.JSONResponse calls http.response.WriteHeader, which eventually calls http2.stickyErrWriter.Write + #34: internal/pkg/rest/framework/framework.go:49:15: framework.JSONResponse calls http.response.WriteHeader, which eventually calls http2.stickyErrWriter.Write + #35: internal/service/attachment/reconcile.go:203:2: attachment.Service.Restore calls http2.transportResponseBody.Close + #36: internal/service/attachment/reconcile.go:203:2: attachment.Service.Restore calls http2.transportResponseBody.Close + #37: internal/service/attachment/reconcile.go:178:20: attachment.Service.Restore calls io.Copy, which eventually calls http2.transportResponseBody.Read + #38: internal/service/attachment/reconcile.go:178:20: attachment.Service.Restore calls io.Copy, which eventually calls http2.transportResponseBody.Read + +Vulnerability #5: GO-2026-6613 + HTTP/1 server connection desynchronization after 2xx CONNECT response in + net/http + More info: https://pkg.go.dev/vuln/GO-2026-6613 + Standard library + Found in: net/http@go1.27 + Fixed in: net/http@go1.27.2 + Example traces found: + #1: internal/pkg/telemetry/http.go:6:2: telemetry.init calls middleware.init, which calls http.CanonicalHeaderKey + #2: cmd/benchmark/main.go:78:22: benchmark.client.setup calls http.Client.Do + #3: cmd/app/storage.go:64:24: app.runStorage calls http.Client.Get + #4: internal/pkg/auth/jwtprovider/manager.go:42:8: jwtprovider.NewCookieManager calls http.Cookie.Valid + #5: internal/pkg/telemetry/http.go:6:2: telemetry.init calls middleware.init, which eventually calls http.HandleFunc + #6: internal/pkg/rest/router/router.go:73:15: router.router.ServeHTTP calls chi.Mux.ServeHTTP, which calls http.HandlerFunc.ServeHTTP + #7: internal/pkg/rest/framework/framework.go:49:15: framework.JSONResponse calls middleware.compressResponseWriter.WriteHeader, which calls http.Header.Add + #8: internal/pkg/rest/framework/framework.go:49:15: framework.JSONResponse calls middleware.compressResponseWriter.WriteHeader, which calls http.Header.Del + #9: internal/pkg/rest/framework/framework.go:220:29: framework.ParseToken calls http.Header.Get + #10: internal/adapter/controller/http/attachmentHandler.go:103:16: http.AttachmentHandler.Get calls http.Header.Set + #11: internal/adapter/controller/http/chat.go:184:30: http.ChatHandler.SendMessage calls http.Header.Values + #12: cmd/benchmark/main.go:76:27: benchmark.client.setup calls http.NewRequest + #13: internal/pkg/openbao/client.go:39:40: openbao.client.call calls http.NewRequestWithContext + #14: internal/pkg/telemetry/telemetry.go:93:112: telemetry.Init calls metric.NewPeriodicReader, which eventually calls http.ParseTime + #15: internal/pkg/objectstore/s3.go:66:27: objectstore.Store.Get calls s3.Client.GetObject, which eventually calls http.ReadRequest + #16: internal/service/attachment/reconcile.go:203:2: attachment.Service.Restore calls http2.requestBody.Close, which eventually calls http.ReadResponse + #17: internal/adapter/controller/http/auth.go:38:39: http.AuthHandler.Login calls http.Request.BasicAuth + #18: internal/pkg/auth/jwtprovider/manager.go:111:20: jwtprovider.manager.RefreshToken calls http.Request.Cookie + #19: cmd/benchmark/main.go:77:18: benchmark.client.setup calls http.Request.SetBasicAuth + #20: internal/service/attachment/reconcile.go:203:2: attachment.Service.Restore calls http2.requestBody.Close, which eventually calls http.Request.Write + #21: internal/pkg/objectstore/s3.go:66:27: objectstore.Store.Get calls s3.Client.GetObject, which eventually calls http.Response.Write + #22: internal/adapter/controller/http/controller.go:89:21: http.server.Stop calls http.Server.Close + #23: cmd/app/main.go:156:44: app.run calls http.Server.ListenAndServe + #24: internal/adapter/controller/http/controller.go:71:27: http.Start calls http.Server.Serve + #25: internal/adapter/controller/http/controller.go:87:29: http.server.Stop calls http.Server.Shutdown + #26: internal/pkg/auth/jwtprovider/manager.go:87:17: jwtprovider.manager.Clear calls http.SetCookie + #27: internal/pkg/telemetry/telemetry.go:84:28: telemetry.Init calls otlptracehttp.New, which eventually calls http.Transport.Clone + #28: internal/pkg/objectstore/s3.go:66:27: objectstore.Store.Get calls s3.Client.GetObject, which eventually calls http.Transport.CloseIdleConnections + #29: internal/pkg/objectstore/s3.go:66:27: objectstore.Store.Get calls s3.Client.GetObject, which eventually calls http.Transport.RoundTrip + #30: internal/service/attachment/reconcile.go:203:2: attachment.Service.Restore calls http.body.Close + #31: internal/service/attachment/reconcile.go:178:20: attachment.Service.Restore calls io.Copy, which eventually calls http.body.Read + #32: internal/service/attachment/reconcile.go:203:2: attachment.Service.Restore calls http.bodyEOFSignal.Close + #33: internal/service/attachment/reconcile.go:178:20: attachment.Service.Restore calls io.Copy, which eventually calls http.bodyEOFSignal.Read + #34: internal/service/attachment/reconcile.go:178:20: attachment.Service.Restore calls io.Copy, which eventually calls http.bodyLocked.Read + #35: internal/service/attachment/reconcile.go:178:20: attachment.Service.Restore calls io.Copy, which eventually calls http.bufioFlushWriter.Write + #36: internal/service/attachment/reconcile.go:203:2: attachment.Service.Restore calls http.cancelTimerBody.Close + #37: internal/service/attachment/reconcile.go:178:20: attachment.Service.Restore calls io.Copy, which eventually calls http.cancelTimerBody.Read + #38: internal/service/attachment/reconcile.go:203:2: attachment.Service.Restore calls http2.transportResponseBody.Close, which eventually calls http.checkConnErrorWriter.Write + #39: internal/service/attachment/reconcile.go:203:2: attachment.Service.Restore calls http2.transportResponseBody.Close, which eventually calls http.chunkWriter.Write + #40: internal/service/attachment/reconcile.go:178:20: attachment.Service.Restore calls io.Copy, which eventually calls http.connReader.Read + #41: internal/service/attachment/reconcile.go:203:2: attachment.Service.Restore calls http.expectContinueReader.Close + #42: internal/service/attachment/reconcile.go:178:20: attachment.Service.Restore calls io.Copy, which eventually calls http.expectContinueReader.Read + #43: internal/service/attachment/reconcile.go:203:2: attachment.Service.Restore calls http.gzipReader.Close + #44: internal/service/attachment/reconcile.go:178:20: attachment.Service.Restore calls io.Copy, which eventually calls http.gzipReader.Read + #45: internal/pkg/encrypting/argon2id.go:70:23: encrypting.encryptor.EncryptSecretUsingArgon2id calls rand.Read, which eventually calls http.loggingConn.Read + #46: internal/pkg/apperr/error.go:79:21: apperr.AppError.Error calls time.parseErrors.Error, which eventually calls http.loggingConn.Write + #47: internal/service/attachment/reconcile.go:203:2: attachment.Service.Restore calls http.maxBytesReader.Close + #48: internal/service/attachment/reconcile.go:178:20: attachment.Service.Restore calls io.Copy, which eventually calls http.maxBytesReader.Read + #49: internal/service/attachment/reconcile.go:203:2: attachment.Service.Restore calls http2.requestBody.Close, which eventually calls http.onceCloseListener.Close + #50: internal/service/attachment/reconcile.go:178:20: attachment.Service.Restore calls io.Copy, which eventually calls http.persistConn.Read + #51: internal/service/attachment/reconcile.go:178:20: attachment.Service.Restore calls io.Copy, which eventually calls http.persistConnWriter.ReadFrom + #52: internal/service/attachment/reconcile.go:203:2: attachment.Service.Restore calls http2.transportResponseBody.Close, which eventually calls http.persistConnWriter.Write + #53: internal/service/attachment/reconcile.go:203:2: attachment.Service.Restore calls http.readTrackingBody.Close + #54: internal/service/attachment/reconcile.go:178:20: attachment.Service.Restore calls io.Copy, which eventually calls http.readTrackingBody.Read + #55: internal/service/attachment/reconcile.go:178:20: attachment.Service.Restore calls io.Copy, which eventually calls http.readWriteCloserBody.Read + #56: internal/service/attachment/reconcile.go:178:20: attachment.Service.Restore calls io.Copy, which eventually calls http.response.ReadFrom + #57: cmd/benchmark/main.go:234:15: benchmark.main calls fmt.Fprintln, which calls http.response.Write + #58: internal/pkg/rest/framework/framework.go:49:15: framework.JSONResponse calls http.response.WriteHeader + #59: internal/pkg/ratelimit/redis.go:14:29: ratelimit.init calls redis.NewScript, which eventually calls http.response.WriteString + #60: internal/pkg/apperr/error.go:79:21: apperr.AppError.Error calls http.transportReadFromServerError.Error + +Vulnerability #6: GO-2026-6612 + Double flow control refund on HTTP/2 server streams in net/http + More info: https://pkg.go.dev/vuln/GO-2026-6612 + Standard library + Found in: net/http/internal/http2@go1.27 + Fixed in: net/http/internal/http2@go1.27.2 + Example traces found: + #1: internal/adapter/controller/http/controller.go:71:27: http.Start calls http.Server.Serve, which eventually calls http2.Server.ServeConn + #2: internal/adapter/controller/http/controller.go:71:27: http.Start calls http.Server.Serve, which eventually calls http2.Server.ServeConn + #3: internal/service/attachment/reconcile.go:203:2: attachment.Service.Restore calls http2.requestBody.Close + #4: internal/service/attachment/reconcile.go:203:2: attachment.Service.Restore calls http2.requestBody.Close + #5: internal/service/attachment/reconcile.go:178:20: attachment.Service.Restore calls io.Copy, which eventually calls http2.requestBody.Read + #6: internal/service/attachment/reconcile.go:178:20: attachment.Service.Restore calls io.Copy, which eventually calls http2.requestBody.Read + +Vulnerability #7: GO-2026-6611 + Excessive CPU consumption from repeated initial window changes in net/http + More info: https://pkg.go.dev/vuln/GO-2026-6611 + Standard library + Found in: net/http/internal/http2@go1.27 + Fixed in: net/http/internal/http2@go1.27.2 + Example traces found: + #1: cmd/benchmark/main.go:78:22: benchmark.client.setup calls http.Client.Do, which eventually calls http2.NetHTTPClientConn.Close + #2: cmd/benchmark/main.go:78:22: benchmark.client.setup calls http.Client.Do, which eventually calls http2.NetHTTPClientConn.Close + #3: cmd/benchmark/main.go:78:22: benchmark.client.setup calls http.Client.Do, which eventually calls http2.NetHTTPClientConn.RoundTrip + #4: cmd/benchmark/main.go:78:22: benchmark.client.setup calls http.Client.Do, which eventually calls http2.NetHTTPClientConn.RoundTrip + #5: internal/adapter/controller/http/controller.go:87:29: http.server.Stop calls http.Server.Shutdown, which calls http2.Server.GracefulShutdown + #6: internal/adapter/controller/http/controller.go:87:29: http.server.Stop calls http.Server.Shutdown, which calls http2.Server.GracefulShutdown + #7: internal/adapter/controller/http/controller.go:71:27: http.Start calls http.Server.Serve, which eventually calls http2.Server.ServeConn + #8: internal/adapter/controller/http/controller.go:71:27: http.Start calls http.Server.Serve, which eventually calls http2.Server.ServeConn + #9: cmd/benchmark/main.go:78:22: benchmark.client.setup calls http.Client.Do, which eventually calls http2.Transport.AddConn + #10: cmd/benchmark/main.go:78:22: benchmark.client.setup calls http.Client.Do, which eventually calls http2.Transport.AddConn + #11: internal/pkg/objectstore/s3.go:66:27: objectstore.Store.Get calls s3.Client.GetObject, which eventually calls http2.Transport.CloseIdleConnections + #12: internal/pkg/objectstore/s3.go:66:27: objectstore.Store.Get calls s3.Client.GetObject, which eventually calls http2.Transport.CloseIdleConnections + #13: cmd/benchmark/main.go:78:22: benchmark.client.setup calls http.Client.Do, which eventually calls http2.Transport.NewClientConn + #14: cmd/benchmark/main.go:78:22: benchmark.client.setup calls http.Client.Do, which eventually calls http2.Transport.NewClientConn + #15: cmd/benchmark/main.go:78:22: benchmark.client.setup calls http.Client.Do, which eventually calls http2.Transport.RoundTrip + #16: cmd/benchmark/main.go:78:22: benchmark.client.setup calls http.Client.Do, which eventually calls http2.Transport.RoundTrip + #17: internal/pkg/rest/framework/framework.go:49:15: framework.JSONResponse calls http.response.WriteHeader, which eventually calls http2.bufferedWriterTimeoutWriter.Write + #18: internal/pkg/rest/framework/framework.go:49:15: framework.JSONResponse calls http.response.WriteHeader, which eventually calls http2.bufferedWriterTimeoutWriter.Write + #19: internal/pkg/rest/framework/framework.go:49:15: framework.JSONResponse calls http.response.WriteHeader, which eventually calls http2.chunkWriter.Write + #20: internal/pkg/rest/framework/framework.go:49:15: framework.JSONResponse calls http.response.WriteHeader, which eventually calls http2.chunkWriter.Write + #21: internal/service/attachment/reconcile.go:203:2: attachment.Service.Restore calls http2.gzipReader.Close + #22: internal/service/attachment/reconcile.go:203:2: attachment.Service.Restore calls http2.gzipReader.Close + #23: internal/service/attachment/reconcile.go:178:20: attachment.Service.Restore calls io.Copy, which eventually calls http2.gzipReader.Read + #24: internal/service/attachment/reconcile.go:178:20: attachment.Service.Restore calls io.Copy, which eventually calls http2.gzipReader.Read + #25: internal/service/attachment/reconcile.go:203:2: attachment.Service.Restore calls http2.requestBody.Close + #26: internal/service/attachment/reconcile.go:203:2: attachment.Service.Restore calls http2.requestBody.Close + #27: cmd/benchmark/main.go:234:15: benchmark.main calls fmt.Fprintln, which calls http2.responseWriter.Write + #28: cmd/benchmark/main.go:234:15: benchmark.main calls fmt.Fprintln, which calls http2.responseWriter.Write + #29: internal/pkg/rest/framework/framework.go:49:15: framework.JSONResponse calls http2.responseWriter.WriteHeader + #30: internal/pkg/rest/framework/framework.go:49:15: framework.JSONResponse calls http2.responseWriter.WriteHeader + #31: internal/pkg/ratelimit/redis.go:14:29: ratelimit.init calls redis.NewScript, which eventually calls http2.responseWriter.WriteString + #32: internal/pkg/ratelimit/redis.go:14:29: ratelimit.init calls redis.NewScript, which eventually calls http2.responseWriter.WriteString + #33: internal/pkg/rest/framework/framework.go:49:15: framework.JSONResponse calls http.response.WriteHeader, which eventually calls http2.stickyErrWriter.Write + #34: internal/pkg/rest/framework/framework.go:49:15: framework.JSONResponse calls http.response.WriteHeader, which eventually calls http2.stickyErrWriter.Write + #35: internal/service/attachment/reconcile.go:203:2: attachment.Service.Restore calls http2.transportResponseBody.Close + #36: internal/service/attachment/reconcile.go:203:2: attachment.Service.Restore calls http2.transportResponseBody.Close + #37: internal/service/attachment/reconcile.go:178:20: attachment.Service.Restore calls io.Copy, which eventually calls http2.transportResponseBody.Read + #38: internal/service/attachment/reconcile.go:178:20: attachment.Service.Restore calls io.Copy, which eventually calls http2.transportResponseBody.Read + +Vulnerability #8: GO-2026-6610 + HTTP/2 transport accepts malformed framing-related headers in net/http + More info: https://pkg.go.dev/vuln/GO-2026-6610 + Standard library + Found in: net/http/internal/http2@go1.27 + Fixed in: net/http/internal/http2@go1.27.2 + Example traces found: + #1: cmd/benchmark/main.go:78:22: benchmark.client.setup calls http.Client.Do, which eventually calls http2.Transport.AddConn + #2: cmd/benchmark/main.go:78:22: benchmark.client.setup calls http.Client.Do, which eventually calls http2.Transport.AddConn + #3: cmd/benchmark/main.go:78:22: benchmark.client.setup calls http.Client.Do, which eventually calls http2.Transport.NewClientConn + #4: cmd/benchmark/main.go:78:22: benchmark.client.setup calls http.Client.Do, which eventually calls http2.Transport.NewClientConn + #5: cmd/benchmark/main.go:78:22: benchmark.client.setup calls http.Client.Do, which eventually calls http2.Transport.RoundTrip + #6: cmd/benchmark/main.go:78:22: benchmark.client.setup calls http.Client.Do, which eventually calls http2.Transport.RoundTrip + +Vulnerability #9: GO-2026-6608 + Memory limit bypass when parsing MIME headers in net/textproto, + mime/multipart + More info: https://pkg.go.dev/vuln/GO-2026-6608 + Standard library + Found in: net/textproto@go1.27 + Fixed in: net/textproto@go1.27.2 + Example traces found: + #1: internal/pkg/rest/router/router.go:73:15: router.router.ServeHTTP calls chi.Mux.ServeHTTP, which eventually calls multipart.Reader.ReadForm + #2: cmd/benchmark/main.go:78:22: benchmark.client.setup calls http.Client.Do, which eventually calls textproto.Reader.ReadMIMEHeader + +Vulnerability #10: GO-2026-6607 + Reject malformed ECH outer extension references in crypto/tls + More info: https://pkg.go.dev/vuln/GO-2026-6607 + Standard library + Found in: crypto/tls@go1.27 + Fixed in: crypto/tls@go1.27.2 + Example traces found: + #1: internal/pkg/broker/rabbitmq.go:56:28: broker.Rabbit.connect calls amqp091.DialConfig, which calls tls.Conn.Handshake + #2: internal/adapter/controller/http/controller.go:71:27: http.Start calls http.Server.Serve, which eventually calls tls.Conn.HandshakeContext + #3: internal/service/attachment/reconcile.go:178:20: attachment.Service.Restore calls io.Copy, which eventually calls tls.Conn.Read + #4: internal/service/attachment/reconcile.go:178:20: attachment.Service.Restore calls io.Copy, which eventually calls tls.Conn.Write + #5: internal/pkg/cache/redis.go:27:26: cache.New calls redis.NewClient, which eventually calls tls.DialWithDialer + #6: cmd/benchmark/main.go:78:22: benchmark.client.setup calls http.Client.Do, which eventually calls tls.Dialer.DialContext + +Vulnerability #11: GO-2026-6605 + HTTP/1 client connection desynchronization after CONNECT rejection in + net/http + More info: https://pkg.go.dev/vuln/GO-2026-6605 + Standard library + Found in: net/http@go1.27 + Fixed in: net/http@go1.27.2 + Example traces found: + #1: cmd/benchmark/main.go:78:22: benchmark.client.setup calls http.Client.Do + #2: cmd/app/storage.go:64:24: app.runStorage calls http.Client.Get + #3: internal/pkg/objectstore/s3.go:66:27: objectstore.Store.Get calls s3.Client.GetObject, which eventually calls http.Transport.CloseIdleConnections + #4: internal/pkg/objectstore/s3.go:66:27: objectstore.Store.Get calls s3.Client.GetObject, which eventually calls http.Transport.RoundTrip + #5: internal/pkg/objectstore/s3.go:66:27: objectstore.Store.Get calls s3.Client.GetObject, which eventually calls httputil.DumpRequestOut + +Vulnerability #12: GO-2026-6603 + HTTP/2 server memory exhaustion due to Trailer headers in net/http + More info: https://pkg.go.dev/vuln/GO-2026-6603 + Standard library + Found in: net/http/internal/http2@go1.27 + Fixed in: net/http/internal/http2@go1.27.2 + Example traces found: + #1: cmd/benchmark/main.go:78:22: benchmark.client.setup calls http.Client.Do, which eventually calls http2.NetHTTPClientConn.Close + #2: cmd/benchmark/main.go:78:22: benchmark.client.setup calls http.Client.Do, which eventually calls http2.NetHTTPClientConn.Close + #3: cmd/benchmark/main.go:78:22: benchmark.client.setup calls http.Client.Do, which eventually calls http2.NetHTTPClientConn.RoundTrip + #4: cmd/benchmark/main.go:78:22: benchmark.client.setup calls http.Client.Do, which eventually calls http2.NetHTTPClientConn.RoundTrip + #5: internal/adapter/controller/http/controller.go:87:29: http.server.Stop calls http.Server.Shutdown, which calls http2.Server.GracefulShutdown + #6: internal/adapter/controller/http/controller.go:87:29: http.server.Stop calls http.Server.Shutdown, which calls http2.Server.GracefulShutdown + #7: internal/adapter/controller/http/controller.go:71:27: http.Start calls http.Server.Serve, which eventually calls http2.Server.ServeConn + #8: internal/adapter/controller/http/controller.go:71:27: http.Start calls http.Server.Serve, which eventually calls http2.Server.ServeConn + #9: cmd/benchmark/main.go:78:22: benchmark.client.setup calls http.Client.Do, which eventually calls http2.Transport.AddConn + #10: cmd/benchmark/main.go:78:22: benchmark.client.setup calls http.Client.Do, which eventually calls http2.Transport.AddConn + #11: internal/pkg/objectstore/s3.go:66:27: objectstore.Store.Get calls s3.Client.GetObject, which eventually calls http2.Transport.CloseIdleConnections + #12: internal/pkg/objectstore/s3.go:66:27: objectstore.Store.Get calls s3.Client.GetObject, which eventually calls http2.Transport.CloseIdleConnections + #13: cmd/benchmark/main.go:78:22: benchmark.client.setup calls http.Client.Do, which eventually calls http2.Transport.NewClientConn + #14: cmd/benchmark/main.go:78:22: benchmark.client.setup calls http.Client.Do, which eventually calls http2.Transport.NewClientConn + #15: cmd/benchmark/main.go:78:22: benchmark.client.setup calls http.Client.Do, which eventually calls http2.Transport.RoundTrip + #16: cmd/benchmark/main.go:78:22: benchmark.client.setup calls http.Client.Do, which eventually calls http2.Transport.RoundTrip + #17: internal/pkg/rest/framework/framework.go:49:15: framework.JSONResponse calls http.response.WriteHeader, which eventually calls http2.bufferedWriterTimeoutWriter.Write + #18: internal/pkg/rest/framework/framework.go:49:15: framework.JSONResponse calls http.response.WriteHeader, which eventually calls http2.bufferedWriterTimeoutWriter.Write + #19: internal/pkg/rest/framework/framework.go:49:15: framework.JSONResponse calls http.response.WriteHeader, which eventually calls http2.chunkWriter.Write + #20: internal/pkg/rest/framework/framework.go:49:15: framework.JSONResponse calls http.response.WriteHeader, which eventually calls http2.chunkWriter.Write + #21: internal/service/attachment/reconcile.go:203:2: attachment.Service.Restore calls http2.gzipReader.Close + #22: internal/service/attachment/reconcile.go:203:2: attachment.Service.Restore calls http2.gzipReader.Close + #23: internal/service/attachment/reconcile.go:178:20: attachment.Service.Restore calls io.Copy, which eventually calls http2.gzipReader.Read + #24: internal/service/attachment/reconcile.go:178:20: attachment.Service.Restore calls io.Copy, which eventually calls http2.gzipReader.Read + #25: internal/service/attachment/reconcile.go:203:2: attachment.Service.Restore calls http2.requestBody.Close + #26: internal/service/attachment/reconcile.go:203:2: attachment.Service.Restore calls http2.requestBody.Close + #27: cmd/benchmark/main.go:234:15: benchmark.main calls fmt.Fprintln, which calls http2.responseWriter.Write + #28: cmd/benchmark/main.go:234:15: benchmark.main calls fmt.Fprintln, which calls http2.responseWriter.Write + #29: internal/pkg/rest/framework/framework.go:49:15: framework.JSONResponse calls http2.responseWriter.WriteHeader + #30: internal/pkg/rest/framework/framework.go:49:15: framework.JSONResponse calls http2.responseWriter.WriteHeader + #31: internal/pkg/ratelimit/redis.go:14:29: ratelimit.init calls redis.NewScript, which eventually calls http2.responseWriter.WriteString + #32: internal/pkg/ratelimit/redis.go:14:29: ratelimit.init calls redis.NewScript, which eventually calls http2.responseWriter.WriteString + #33: internal/pkg/rest/framework/framework.go:49:15: framework.JSONResponse calls http.response.WriteHeader, which eventually calls http2.stickyErrWriter.Write + #34: internal/pkg/rest/framework/framework.go:49:15: framework.JSONResponse calls http.response.WriteHeader, which eventually calls http2.stickyErrWriter.Write + #35: internal/service/attachment/reconcile.go:203:2: attachment.Service.Restore calls http2.transportResponseBody.Close + #36: internal/service/attachment/reconcile.go:203:2: attachment.Service.Restore calls http2.transportResponseBody.Close + #37: internal/service/attachment/reconcile.go:178:20: attachment.Service.Restore calls io.Copy, which eventually calls http2.transportResponseBody.Read + #38: internal/service/attachment/reconcile.go:178:20: attachment.Service.Restore calls io.Copy, which eventually calls http2.transportResponseBody.Read + +Vulnerability #13: GO-2026-6600 + Recognize yield as regexp preceder keyword in html/template + More info: https://pkg.go.dev/vuln/GO-2026-6600 + Standard library + Found in: html/template@go1.27 + Fixed in: html/template@go1.27.2 + Example traces found: + #1: internal/pkg/rest/router/router.go:73:15: router.router.ServeHTTP calls chi.Mux.ServeHTTP, which eventually calls template.Template.Execute + #2: internal/pkg/rest/router/router.go:73:15: router.router.ServeHTTP calls chi.Mux.ServeHTTP, which eventually calls template.Template.ExecuteTemplate + +Vulnerability #14: GO-2026-6599 + Reset context tracking on consecutive template expressions in html/template + More info: https://pkg.go.dev/vuln/GO-2026-6599 + Standard library + Found in: html/template@go1.27 + Fixed in: html/template@go1.27.2 + Example traces found: + #1: internal/pkg/rest/router/router.go:73:15: router.router.ServeHTTP calls chi.Mux.ServeHTTP, which eventually calls template.Template.Execute + #2: internal/pkg/rest/router/router.go:73:15: router.router.ServeHTTP calls chi.Mux.ServeHTTP, which eventually calls template.Template.ExecuteTemplate + +Vulnerability #15: GO-2026-6503 + RabbitMQ amqp091-go: Denial of Service via Sub-Spec Frame Size Negotiation + in github.com/rabbitmq/amqp091-go + More info: https://pkg.go.dev/vuln/GO-2026-6503 + Module: github.com/rabbitmq/amqp091-go + Found in: github.com/rabbitmq/amqp091-go@v1.10.0 + Fixed in: github.com/rabbitmq/amqp091-go@v1.13.0 + Example traces found: + #1: internal/pkg/broker/rabbitmq.go:56:28: broker.Rabbit.connect calls amqp091.DialConfig + +Vulnerability #16: GO-2026-6502 + RabbitMQ amqp091-go: Consumer Message Flooding via Signed-to-Unsigned + Integer Casting in Qos Configuration in github.com/rabbitmq/amqp091-go + More info: https://pkg.go.dev/vuln/GO-2026-6502 + Module: github.com/rabbitmq/amqp091-go + Found in: github.com/rabbitmq/amqp091-go@v1.10.0 + Fixed in: github.com/rabbitmq/amqp091-go@v1.13.0 + Example traces found: + #1: internal/pkg/broker/rabbitmq.go:120:15: broker.Rabbit.Consume calls amqp091.Channel.Qos + +Vulnerability #17: GO-2026-6500 + RabbitMQ amqp091-go: Resource Exhaustion (OOM) via Unbounded Body Buffer + Allocation in github.com/rabbitmq/amqp091-go + More info: https://pkg.go.dev/vuln/GO-2026-6500 + Module: github.com/rabbitmq/amqp091-go + Found in: github.com/rabbitmq/amqp091-go@v1.10.0 + Fixed in: github.com/rabbitmq/amqp091-go@v1.13.0 + Example traces found: + #1: internal/pkg/broker/rabbitmq.go:56:28: broker.Rabbit.connect calls amqp091.DialConfig + +Vulnerability #18: GO-2026-6498 + RabbitMQ amqp091-go: Silent Data Truncation and State Corruption via + Shortstr Integer Overflow in github.com/rabbitmq/amqp091-go + More info: https://pkg.go.dev/vuln/GO-2026-6498 + Module: github.com/rabbitmq/amqp091-go + Found in: github.com/rabbitmq/amqp091-go@v1.10.0 + Fixed in: github.com/rabbitmq/amqp091-go@v1.13.0 + Example traces found: + #1: internal/pkg/broker/rabbitmq.go:190:2: broker.Rabbit.Depth calls amqp091.Channel.Close + #2: internal/pkg/broker/rabbitmq.go:86:19: broker.Rabbit.Publish calls amqp091.Channel.Confirm + #3: internal/pkg/broker/rabbitmq.go:123:29: broker.Rabbit.Consume calls amqp091.Channel.Consume + #4: internal/pkg/broker/rabbitmq.go:92:30: broker.Rabbit.Publish calls amqp091.Channel.PublishWithContext + #5: internal/pkg/broker/rabbitmq.go:120:15: broker.Rabbit.Consume calls amqp091.Channel.Qos + #6: cmd/securityprobe/main.go:154:25: securityprobe.run calls amqp091.Channel.QueueDeclare + #7: internal/pkg/broker/rabbitmq.go:191:25: broker.Rabbit.Depth calls amqp091.Channel.QueueInspect + #8: cmd/securityprobe/main.go:153:23: securityprobe.run calls amqp091.Connection.Channel + #9: internal/pkg/broker/rabbitmq.go:189:2: broker.Rabbit.Depth calls amqp091.Connection.Close + #10: internal/pkg/broker/rabbitmq.go:164:14: broker.Consume calls amqp091.Delivery.Ack + #11: internal/pkg/broker/rabbitmq.go:160:16: broker.Consume calls amqp091.Delivery.Nack + #12: internal/pkg/broker/rabbitmq.go:147:18: broker.Consume calls amqp091.Delivery.Reject + #13: internal/pkg/broker/rabbitmq.go:56:28: broker.Rabbit.connect calls amqp091.DialConfig + +Vulnerability #19: GO-2026-6496 + RabbitMQ amqp091-go: Protocol Desynchronization and Frame Injection via + Integer Overflow in readLongstr in github.com/rabbitmq/amqp091-go + More info: https://pkg.go.dev/vuln/GO-2026-6496 + Module: github.com/rabbitmq/amqp091-go + Found in: github.com/rabbitmq/amqp091-go@v1.10.0 + Fixed in: github.com/rabbitmq/amqp091-go@v1.13.0 + Example traces found: + #1: internal/pkg/broker/rabbitmq.go:56:28: broker.Rabbit.connect calls amqp091.DialConfig + +Vulnerability #20: GO-2026-6494 + RabbitMQ amqp091-go: Denial of Service via Malicious Field Length in AMQP + Client in github.com/rabbitmq/amqp091-go + More info: https://pkg.go.dev/vuln/GO-2026-6494 + Module: github.com/rabbitmq/amqp091-go + Found in: github.com/rabbitmq/amqp091-go@v1.10.0 + Fixed in: github.com/rabbitmq/amqp091-go@v1.13.0 + Example traces found: + #1: internal/pkg/broker/rabbitmq.go:56:28: broker.Rabbit.connect calls amqp091.DialConfig + +Vulnerability #21: GO-2026-6492 + RabbitMQ amqp091-go: Missing Explicit TLS Minimum Version Configuration In + URI Parser in github.com/rabbitmq/amqp091-go + More info: https://pkg.go.dev/vuln/GO-2026-6492 + Module: github.com/rabbitmq/amqp091-go + Found in: github.com/rabbitmq/amqp091-go@v1.10.0 + Fixed in: github.com/rabbitmq/amqp091-go@v1.13.0 + Example traces found: + #1: internal/pkg/broker/rabbitmq.go:56:28: broker.Rabbit.connect calls amqp091.DialConfig + +Vulnerability #22: GO-2026-6491 + RabbitMQ amqp091-go: Plaintext Credential Exposure via Exported PLAIN + Authentication Struct Fields in github.com/rabbitmq/amqp091-go + More info: https://pkg.go.dev/vuln/GO-2026-6491 + Module: github.com/rabbitmq/amqp091-go + Found in: github.com/rabbitmq/amqp091-go@v1.10.0 + Fixed in: github.com/rabbitmq/amqp091-go@v1.13.0 + Example traces found: + #1: internal/pkg/broker/rabbitmq.go:56:28: broker.Rabbit.connect calls amqp091.DialConfig + +Vulnerability #23: GO-2026-6372 + Potential Memory Exhaustion/Protocol Violation via Broker-Controlled + Oversized Payload in github.com/rabbitmq/amqp091-go + More info: https://pkg.go.dev/vuln/GO-2026-6372 + Module: github.com/rabbitmq/amqp091-go + Found in: github.com/rabbitmq/amqp091-go@v1.10.0 + Fixed in: github.com/rabbitmq/amqp091-go@v1.13.0 + Example traces found: + #1: internal/pkg/broker/rabbitmq.go:190:2: broker.Rabbit.Depth calls amqp091.Channel.Close + #2: internal/pkg/broker/rabbitmq.go:86:19: broker.Rabbit.Publish calls amqp091.Channel.Confirm + #3: internal/pkg/broker/rabbitmq.go:123:29: broker.Rabbit.Consume calls amqp091.Channel.Consume + #4: cmd/securityprobe/main.go:161:29: securityprobe.run calls amqp091.Channel.IsClosed + #5: internal/pkg/broker/rabbitmq.go:89:30: broker.Rabbit.Publish calls amqp091.Channel.NotifyPublish + #6: internal/pkg/broker/rabbitmq.go:90:28: broker.Rabbit.Publish calls amqp091.Channel.NotifyReturn + #7: internal/pkg/broker/rabbitmq.go:92:30: broker.Rabbit.Publish calls amqp091.Channel.PublishWithContext + #8: internal/pkg/broker/rabbitmq.go:120:15: broker.Rabbit.Consume calls amqp091.Channel.Qos + #9: cmd/securityprobe/main.go:154:25: securityprobe.run calls amqp091.Channel.QueueDeclare + #10: internal/pkg/broker/rabbitmq.go:191:25: broker.Rabbit.Depth calls amqp091.Channel.QueueInspect + #11: cmd/securityprobe/main.go:153:23: securityprobe.run calls amqp091.Connection.Channel + #12: internal/pkg/broker/rabbitmq.go:189:2: broker.Rabbit.Depth calls amqp091.Connection.Close + #13: internal/pkg/broker/rabbitmq.go:30:26: broker.Rabbit.connect calls amqp091.DefaultDial + #14: internal/pkg/broker/rabbitmq.go:164:14: broker.Consume calls amqp091.Delivery.Ack + #15: internal/pkg/broker/rabbitmq.go:160:16: broker.Consume calls amqp091.Delivery.Nack + #16: internal/pkg/broker/rabbitmq.go:147:18: broker.Consume calls amqp091.Delivery.Reject + #17: internal/pkg/broker/rabbitmq.go:56:28: broker.Rabbit.connect calls amqp091.DialConfig + #18: internal/pkg/apperr/error.go:79:21: apperr.AppError.Error calls amqp091.Error.Error + #19: internal/pkg/broker/rabbitmq.go:14:2: broker.init calls amqp091.init + #20: internal/service/attachment/reconcile.go:203:2: attachment.Service.Restore calls http2.requestBody.Close, which eventually calls amqp091.shutdown + #21: internal/service/attachment/reconcile.go:203:2: attachment.Service.Restore calls http2.requestBody.Close, which eventually calls amqp091.shutdown + +Your code is affected by 23 vulnerabilities from 1 module and the Go standard library. +This scan also found 3 vulnerabilities in packages you import and 3 +vulnerabilities in modules you require, but your code doesn't appear to call +these vulnerabilities. +Use '-show verbose' for more details. diff --git a/evidence/govulncheck.txt b/evidence/govulncheck.txt new file mode 100644 index 0000000..e758922 --- /dev/null +++ b/evidence/govulncheck.txt @@ -0,0 +1 @@ +No vulnerabilities found. diff --git a/evidence/image-equivalence.json b/evidence/image-equivalence.json new file mode 100644 index 0000000..6693809 --- /dev/null +++ b/evidence/image-equivalence.json @@ -0,0 +1,9 @@ +{ + "running_container_image": "sha256:50791f115882c3476e7a9ee5b113a60c4ff5c4b2e8f5a7d8820f18695c4f02d2", + "tag_after_independent_build": "sha256:c207d41316af63e9dd59e8f6794c59664ea2257c120c12f4663699a6015f941f", + "binary_sha256_identical": true, + "binaries": [ + "c8147e057feae2b500866e4f9de7b9e585d8d42a153c26215ecb59893c837dd2 /usr/local/bin/messenger", + "8dda90c95a12292a85a52464ba46c6a826c14eff80369874b0ed62e778fbf785 /usr/local/bin/securityprobe" + ] +} diff --git a/evidence/image-scan.json b/evidence/image-scan.json new file mode 100644 index 0000000..2454466 --- /dev/null +++ b/evidence/image-scan.json @@ -0,0 +1,21 @@ +{ + "scanner": "aquasec/trivy:0.67.2", + "image_id": "sha256:c207d41316af63e9dd59e8f6794c59664ea2257c120c12f4663699a6015f941f", + "results": [ + { + "target": "/scan/image.tar (alpine 3.23.6)", + "counts": {}, + "findings": [] + }, + { + "target": "usr/local/bin/messenger", + "counts": {}, + "findings": [] + }, + { + "target": "usr/local/bin/securityprobe", + "counts": {}, + "findings": [] + } + ] +} diff --git a/evidence/images-run.txt b/evidence/images-run.txt new file mode 100644 index 0000000..347c8fc --- /dev/null +++ b/evidence/images-run.txt @@ -0,0 +1 @@ +PASS: image geometry, EXIF, alpha, real formats, checksums, ACL and negative inputs diff --git a/evidence/images.json b/evidence/images.json new file mode 100644 index 0000000..717ac91 --- /dev/null +++ b/evidence/images.json @@ -0,0 +1,53 @@ +{ + "results": [ + { + "case": "small", + "result": "PASS", + "thumbnail": [ + 16, + 8 + ] + }, + { + "case": "floor", + "result": "PASS", + "thumbnail": [ + 256, + 76 + ] + }, + { + "case": "alpha", + "result": "PASS", + "thumbnail": [ + 40, + 20 + ] + }, + { + "case": "orientation", + "result": "PASS", + "thumbnail": [ + 128, + 256 + ] + }, + { + "case": "thin", + "result": "PASS", + "thumbnail": [ + 1, + 256 + ] + }, + { + "case": "center_crop", + "result": "PASS", + "thumbnail": [ + 256, + 85 + ] + } + ], + "negative_cases": 4 +} diff --git a/evidence/leaf-rotation.json b/evidence/leaf-rotation.json new file mode 100644 index 0000000..9f232cd --- /dev/null +++ b/evidence/leaf-rotation.json @@ -0,0 +1,6 @@ +{ + "image_unchanged": true, + "old_leaf_rejected": true, + "new_leaf_accepted": true, + "verified_https_ready": true +} diff --git a/evidence/leaf-run.txt b/evidence/leaf-run.txt new file mode 100644 index 0000000..41ba818 --- /dev/null +++ b/evidence/leaf-run.txt @@ -0,0 +1,4 @@ +Local lab7 CA, identities and synthetic secrets prepared. No credentials printed. +Leaf replaced and old serial revoked without rebuilding the image: lb +PASS: verified HTTPS readiness +PASS: leaf rotation without image rebuild and old-serial revocation diff --git a/evidence/persistence-run.txt b/evidence/persistence-run.txt new file mode 100644 index 0000000..c4fb3ad --- /dev/null +++ b/evidence/persistence-run.txt @@ -0,0 +1,3 @@ +python3 tests/resilience.py --mode persistence --action-script "scripts/restart.sh" --base-url "https://localhost:8443" --ca-file "/Users/a7tech173/development/gitea.privcloud.tech/mtusi2026/mtusi2026lab7/secrets/lab7/lb/ca.crt" +Executing explicit persistence hook; hook output suppressed to avoid leaking secrets. +PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence. diff --git a/evidence/public-checks.txt b/evidence/public-checks.txt new file mode 100644 index 0000000..6f8e92f --- /dev/null +++ b/evidence/public-checks.txt @@ -0,0 +1,4 @@ +python3 tests/smoke.py --base-url "https://localhost:8443" --ca-file "/Users/a7tech173/development/gitea.privcloud.tech/mtusi2026/mtusi2026lab7/secrets/lab7/lb/ca.crt" +PASS: lab7 public API scenario; infrastructure and grading evidence remain separate. +python3 tests/security.py --base-url "https://localhost:8443" --ca-file "/Users/a7tech173/development/gitea.privcloud.tech/mtusi2026/mtusi2026lab7/secrets/lab7/lb/ca.crt" +PASS: trusted HTTPS works; untrusted CA rejected. This does NOT verify internal mTLS, OpenBao or hardening. diff --git a/evidence/restart-evidence.json b/evidence/restart-evidence.json new file mode 100644 index 0000000..fe7673a --- /dev/null +++ b/evidence/restart-evidence.json @@ -0,0 +1,313 @@ +{ + "before": { + "/mtusi2026lab7-api-1": { + "id": "d6fda7f70c3273d02dc57c920ab696da50e151e22af44ed8a285ea6913902da1", + "started_at": "2026-11-17T14:04:13.474204128Z", + "volumes": [] + }, + "/mtusi2026lab7-api-2": { + "id": "54a760b9c39b7f9b65c217df4b5ea83a4350a578579cd001de82a50604c648d4", + "started_at": "2026-11-17T14:04:13.475004044Z", + "volumes": [] + }, + "/mtusi2026lab7-broker-1": { + "id": "be47e533bc90852ec009a15c8f2e9d8bfaed3dfeaa73e80e8a03191e2e3370c1", + "started_at": "2026-11-17T14:09:15.50756267Z", + "volumes": [ + { + "name": "mtusi2026lab7_broker_data", + "destination": "/var/lib/rabbitmq" + } + ] + }, + "/mtusi2026lab7-dispatcher-1": { + "id": "dc137b22d3e6c75277f348537103a5cef6b8099cca45f2a2f2c11115fae202df", + "started_at": "2026-11-17T14:09:28.751998385Z", + "volumes": [] + }, + "/mtusi2026lab7-grafana-1": { + "id": "adeb6ec904706c78545c39ed621c7484b74a1b277a02d6f88e989b5100a66262", + "started_at": "2026-11-17T10:51:01.37820772Z", + "volumes": [ + { + "name": "mtusi2026lab7_grafana_data", + "destination": "/var/lib/grafana" + } + ] + }, + "/mtusi2026lab7-lb-1": { + "id": "bcdeea3ec62786e76b0226e58fd1184fe120855a02732090f0b05771a352a7d6", + "started_at": "2026-11-17T14:03:42.774457377Z", + "volumes": [] + }, + "/mtusi2026lab7-loki-1": { + "id": "e4e27b3ae65375241ad7f192aa0f82759e7929702cf9cff8082c3560d0366252", + "started_at": "2026-11-17T14:06:30.454375136Z", + "volumes": [ + { + "name": "mtusi2026lab7_loki_data", + "destination": "/loki" + } + ] + }, + "/mtusi2026lab7-loki-tls-1": { + "id": "3fdda65b7eb7c05f278e2ae68428189d707957e7bdf8b62271c51682c9292c4e", + "started_at": "2026-11-17T14:06:31.257750678Z", + "volumes": [] + }, + "/mtusi2026lab7-openbao-1": { + "id": "56dc536779db92bf8a1841c4b7e235a028c85a1a3b98f6227c28e2d8d847f6a7", + "started_at": "2026-11-17T10:51:01.376433887Z", + "volumes": [ + { + "name": "1d1ea9e4a43d16c79452b101c377c8b20382d0b104219dd18ed19518510c7236", + "destination": "/openbao/file" + }, + { + "name": "a4fdd6b41d15d5198e1577cb10314d870ef1e3898b26c4dfe0912278760ed48b", + "destination": "/openbao/logs" + }, + { + "name": "mtusi2026lab7_bao_data", + "destination": "/openbao/data" + } + ] + }, + "/mtusi2026lab7-otel-collector-1": { + "id": "29e8dc4c7eec171bcd572b91d52653fb10dd398f0693ec3903a14c02371cb88f", + "started_at": "2026-11-17T10:51:01.29778072Z", + "volumes": [] + }, + "/mtusi2026lab7-otel-collector-tls-1": { + "id": "915b6ea1291e3d8f2e7cda13f3d92eb99fb6222b199f40650673e8d0134cb2c9", + "started_at": "2026-11-17T10:51:02.868115263Z", + "volumes": [] + }, + "/mtusi2026lab7-postgres-1": { + "id": "ef7ea1912bec58bd1f62c767c4321e5ef98180a479ef9d0d5bd7288517ddcc3a", + "started_at": "2026-11-17T10:33:32.361978804Z", + "volumes": [ + { + "name": "mtusi2026lab7_postgres_data", + "destination": "/var/lib/postgresql/data" + } + ] + }, + "/mtusi2026lab7-prometheus-1": { + "id": "26d14907a8d04095b62ae9b92073e9743526ad3d94e7fe58b89d2480b0a6baa8", + "started_at": "2026-11-17T10:51:01.373130512Z", + "volumes": [ + { + "name": "mtusi2026lab7_prometheus_data", + "destination": "/prometheus" + } + ] + }, + "/mtusi2026lab7-prometheus-tls-1": { + "id": "9a71876af1662d245be27ac72ca89cdf1d1fe87604cff6bc9364edd30497939f", + "started_at": "2026-11-17T10:51:02.816800138Z", + "volumes": [] + }, + "/mtusi2026lab7-redis-1": { + "id": "e5835ee95c53224aed3870c331c7af04d11b209944a12a8694524b20b72acca5", + "started_at": "2026-11-17T10:33:32.355671846Z", + "volumes": [ + { + "name": "mtusi2026lab7_redis_data", + "destination": "/data" + } + ] + }, + "/mtusi2026lab7-s3-1": { + "id": "79232cd23eb7cb0d401caffc771bdddf9d26925e069682c6fd89ec434b97b146", + "started_at": "2026-11-17T13:58:48.96820938Z", + "volumes": [ + { + "name": "mtusi2026lab7_s3_data", + "destination": "/data" + } + ] + }, + "/mtusi2026lab7-s3-tls-1": { + "id": "72b8d5056992bcb5537c05c79a892d6a251e31952309b27b216e88c34d68a1c8", + "started_at": "2026-11-17T13:58:49.722560547Z", + "volumes": [] + }, + "/mtusi2026lab7-tempo-1": { + "id": "debd0f0fd4f810ed8edc27351a8d9c6926e8c2e036ed622e8eab21d7bdca0cfd", + "started_at": "2026-11-17T14:06:28.474408426Z", + "volumes": [ + { + "name": "mtusi2026lab7_tempo_data", + "destination": "/var/tempo" + } + ] + }, + "/mtusi2026lab7-tempo-tls-1": { + "id": "cef5ce161fb4a5cae103c48bfbe47c1956a56c97362e5fe265eab0a521810da5", + "started_at": "2026-11-17T14:06:31.279364886Z", + "volumes": [] + }, + "/mtusi2026lab7-worker-1": { + "id": "a8027a54c01fff295bda985e12c10570612becc90b7120f45a49ce7b5f766839", + "started_at": "2026-11-17T14:09:27.92889876Z", + "volumes": [] + } + }, + "after": { + "/mtusi2026lab7-api-1": { + "id": "d6fda7f70c3273d02dc57c920ab696da50e151e22af44ed8a285ea6913902da1", + "started_at": "2026-11-17T14:09:37.9784245Z", + "volumes": [] + }, + "/mtusi2026lab7-api-2": { + "id": "54a760b9c39b7f9b65c217df4b5ea83a4350a578579cd001de82a50604c648d4", + "started_at": "2026-11-17T14:09:38.034661625Z", + "volumes": [] + }, + "/mtusi2026lab7-broker-1": { + "id": "be47e533bc90852ec009a15c8f2e9d8bfaed3dfeaa73e80e8a03191e2e3370c1", + "started_at": "2026-11-17T14:09:35.819957347Z", + "volumes": [ + { + "name": "mtusi2026lab7_broker_data", + "destination": "/var/lib/rabbitmq" + } + ] + }, + "/mtusi2026lab7-dispatcher-1": { + "id": "dc137b22d3e6c75277f348537103a5cef6b8099cca45f2a2f2c11115fae202df", + "started_at": "2026-11-17T14:09:38.093570667Z", + "volumes": [] + }, + "/mtusi2026lab7-grafana-1": { + "id": "adeb6ec904706c78545c39ed621c7484b74a1b277a02d6f88e989b5100a66262", + "started_at": "2026-11-17T14:09:32.193343262Z", + "volumes": [ + { + "name": "mtusi2026lab7_grafana_data", + "destination": "/var/lib/grafana" + } + ] + }, + "/mtusi2026lab7-lb-1": { + "id": "bcdeea3ec62786e76b0226e58fd1184fe120855a02732090f0b05771a352a7d6", + "started_at": "2026-11-17T14:09:38.225323208Z", + "volumes": [] + }, + "/mtusi2026lab7-loki-1": { + "id": "e4e27b3ae65375241ad7f192aa0f82759e7929702cf9cff8082c3560d0366252", + "started_at": "2026-11-17T14:09:31.680025386Z", + "volumes": [ + { + "name": "mtusi2026lab7_loki_data", + "destination": "/loki" + } + ] + }, + "/mtusi2026lab7-loki-tls-1": { + "id": "9d289263bdf85dd8527aa2f2e5f712e288224d43cf263f933dc5b4b0ea7767c5", + "started_at": "2026-11-17T14:09:37.32076875Z", + "volumes": [] + }, + "/mtusi2026lab7-openbao-1": { + "id": "56dc536779db92bf8a1841c4b7e235a028c85a1a3b98f6227c28e2d8d847f6a7", + "started_at": "2026-11-17T14:09:31.746126136Z", + "volumes": [ + { + "name": "mtusi2026lab7_bao_data", + "destination": "/openbao/data" + }, + { + "name": "1d1ea9e4a43d16c79452b101c377c8b20382d0b104219dd18ed19518510c7236", + "destination": "/openbao/file" + }, + { + "name": "a4fdd6b41d15d5198e1577cb10314d870ef1e3898b26c4dfe0912278760ed48b", + "destination": "/openbao/logs" + } + ] + }, + "/mtusi2026lab7-otel-collector-1": { + "id": "29e8dc4c7eec171bcd572b91d52653fb10dd398f0693ec3903a14c02371cb88f", + "started_at": "2026-11-17T14:09:31.295292678Z", + "volumes": [] + }, + "/mtusi2026lab7-otel-collector-tls-1": { + "id": "c47eae066f34b75500db188715818ce55b3e89c2bca4fc25e83bba3a6f0aa79c", + "started_at": "2026-11-17T14:09:37.33655425Z", + "volumes": [] + }, + "/mtusi2026lab7-postgres-1": { + "id": "ef7ea1912bec58bd1f62c767c4321e5ef98180a479ef9d0d5bd7288517ddcc3a", + "started_at": "2026-11-17T14:09:31.375696886Z", + "volumes": [ + { + "name": "mtusi2026lab7_postgres_data", + "destination": "/var/lib/postgresql/data" + } + ] + }, + "/mtusi2026lab7-prometheus-1": { + "id": "26d14907a8d04095b62ae9b92073e9743526ad3d94e7fe58b89d2480b0a6baa8", + "started_at": "2026-11-17T14:09:31.215812844Z", + "volumes": [ + { + "name": "mtusi2026lab7_prometheus_data", + "destination": "/prometheus" + } + ] + }, + "/mtusi2026lab7-prometheus-tls-1": { + "id": "6ce0529bded56716bb6f3df214263b6cc6bf62bdf0defeba7e1f31d5b798e261", + "started_at": "2026-11-17T14:09:37.25840775Z", + "volumes": [] + }, + "/mtusi2026lab7-redis-1": { + "id": "e5835ee95c53224aed3870c331c7af04d11b209944a12a8694524b20b72acca5", + "started_at": "2026-11-17T14:09:31.388634636Z", + "volumes": [ + { + "name": "mtusi2026lab7_redis_data", + "destination": "/data" + } + ] + }, + "/mtusi2026lab7-s3-1": { + "id": "79232cd23eb7cb0d401caffc771bdddf9d26925e069682c6fd89ec434b97b146", + "started_at": "2026-11-17T14:09:34.173379429Z", + "volumes": [ + { + "name": "mtusi2026lab7_s3_data", + "destination": "/data" + } + ] + }, + "/mtusi2026lab7-s3-tls-1": { + "id": "e69293ccadc868d3d2b6d63e55d7b4a9d15df9b08b95688cd809802deb703335", + "started_at": "2026-11-17T14:09:37.325704166Z", + "volumes": [] + }, + "/mtusi2026lab7-tempo-1": { + "id": "debd0f0fd4f810ed8edc27351a8d9c6926e8c2e036ed622e8eab21d7bdca0cfd", + "started_at": "2026-11-17T14:09:31.447828553Z", + "volumes": [ + { + "name": "mtusi2026lab7_tempo_data", + "destination": "/var/tempo" + } + ] + }, + "/mtusi2026lab7-tempo-tls-1": { + "id": "08cf52574facd688fb2497bdd32dcc3cb55c83b9814ea215e0f3a2cd50170b5a", + "started_at": "2026-11-17T14:09:37.338702Z", + "volumes": [] + }, + "/mtusi2026lab7-worker-1": { + "id": "a8027a54c01fff295bda985e12c10570612becc90b7120f45a49ce7b5f766839", + "started_at": "2026-11-17T14:09:37.89173875Z", + "volumes": [] + } + }, + "all_stores_restarted": true +} diff --git a/evidence/sealed-run.txt b/evidence/sealed-run.txt new file mode 100644 index 0000000..c9b9adf --- /dev/null +++ b/evidence/sealed-run.txt @@ -0,0 +1 @@ +PASS: sealed OpenBao produces bounded fail-closed and recovers after operator unseal diff --git a/evidence/sealed.json b/evidence/sealed.json new file mode 100644 index 0000000..85e839a --- /dev/null +++ b/evidence/sealed.json @@ -0,0 +1,6 @@ +{ + "short_token_ttl_seconds": 30, + "fail_closed_seconds": 28.5457634171471, + "recovered_after_operator_unseal": true, + "plaintext_fallback": false +} diff --git a/evidence/secret-audit.json b/evidence/secret-audit.json new file mode 100644 index 0000000..68ebb16 --- /dev/null +++ b/evidence/secret-audit.json @@ -0,0 +1,8 @@ +{ + "versionable_files_checked": 285, + "generated_secrets_checked": 42, + "matches": 0, + "private_material_ignored": true, + "build_context_excludes_secrets": true, + "initial_root_token_removed": true +} diff --git a/evidence/security-run.txt b/evidence/security-run.txt new file mode 100644 index 0000000..4b98fbe --- /dev/null +++ b/evidence/security-run.txt @@ -0,0 +1,50 @@ +PASS: API replica 1 accepts LB +PASS: API replica 1 rejects no certificate +PASS: API replica 1 rejects same CA wrong identity +PASS: API replica 1 rejects foreign CA +PASS: API replica 2 accepts LB +PASS: API replica 2 rejects no certificate +PASS: API replica 2 rejects same CA wrong identity +PASS: API replica 2 rejects foreign CA +PASS: API server hostname checked +PASS: DB TLS verify-full +PASS: DB rejects wrong hostname +PASS: DB rejects untrusted server CA +PASS: Collector accepts api +PASS: Collector accepts worker +PASS: Collector accepts dispatcher +PASS: Collector denies stranger +PASS: Collector denies foreign +PASS: Collector denies worker no-cert +PASS: Broker denies stranger +PASS: Broker denies foreign +PASS: Broker denies worker no-cert +PASS: S3 denies unauthenticated identity stranger +PASS: S3 denies unauthenticated identity foreign +PASS: S3 denies unauthenticated identity worker +PASS: Worker least privilege db +PASS: Worker least privilege s3 +PASS: Worker least privilege amqp +PASS: Redis scoped ACL +PASS: Matrix allowed https://otel-collector:9444/metrics +PASS: Matrix rejects unknown identity https://otel-collector:9444/metrics +PASS: Matrix allowed https://otel-collector:9445/metrics +PASS: Matrix rejects unknown identity https://otel-collector:9445/metrics +PASS: Matrix allowed https://prometheus:9443/api/v1/query?query=up +PASS: Matrix rejects unknown identity https://prometheus:9443/api/v1/query?query=up +PASS: Matrix allowed https://loki:9443/ready +PASS: Matrix rejects unknown identity https://loki:9443/ready +PASS: Matrix allowed https://tempo:9443/ready +PASS: Matrix rejects unknown identity https://tempo:9443/ready +PASS: Matrix allowed https://loki:9443/otlp/v1/logs +PASS: Matrix rejects unknown identity https://loki:9443/otlp/v1/logs +PASS: Matrix allowed https://tempo:9444/v1/traces +PASS: Matrix rejects unknown identity https://tempo:9444/v1/traces +PASS: Grafana cannot ingest Loki logs +PASS: Grafana cannot use Tempo ingest +PASS: Collector cannot query Loki +PASS: OpenBao worker policy allow/deny +PASS: Shared login rate limit returns 429 Retry-After +PASS: JSON body limit +PASS: Logout revocation +PASS: Plaintext backend bypasses unreachable diff --git a/evidence/security.json b/evidence/security.json new file mode 100644 index 0000000..501d7d4 --- /dev/null +++ b/evidence/security.json @@ -0,0 +1,443 @@ +{ + "timestamp": "2026-11-17T14:18:27Z", + "results": [ + { + "scenario": "API replica 1 accepts LB", + "passed": true, + "details": { + "expect": "allow", + "matched": true, + "mode": "tls" + } + }, + { + "scenario": "API replica 1 rejects no certificate", + "passed": true, + "details": { + "expect": "deny", + "matched": true, + "mode": "tls" + } + }, + { + "scenario": "API replica 1 rejects same CA wrong identity", + "passed": true, + "details": { + "expect": "deny", + "matched": true, + "mode": "tls" + } + }, + { + "scenario": "API replica 1 rejects foreign CA", + "passed": true, + "details": { + "expect": "deny", + "matched": true, + "mode": "tls" + } + }, + { + "scenario": "API replica 2 accepts LB", + "passed": true, + "details": { + "expect": "allow", + "matched": true, + "mode": "tls" + } + }, + { + "scenario": "API replica 2 rejects no certificate", + "passed": true, + "details": { + "expect": "deny", + "matched": true, + "mode": "tls" + } + }, + { + "scenario": "API replica 2 rejects same CA wrong identity", + "passed": true, + "details": { + "expect": "deny", + "matched": true, + "mode": "tls" + } + }, + { + "scenario": "API replica 2 rejects foreign CA", + "passed": true, + "details": { + "expect": "deny", + "matched": true, + "mode": "tls" + } + }, + { + "scenario": "API server hostname checked", + "passed": true, + "details": { + "expect": "deny", + "matched": true, + "mode": "tls" + } + }, + { + "scenario": "DB TLS verify-full", + "passed": true, + "details": { + "matched": true, + "mode": "db-tls" + } + }, + { + "scenario": "DB rejects wrong hostname", + "passed": true, + "details": { + "matched": true, + "mode": "db-tls" + } + }, + { + "scenario": "DB rejects untrusted server CA", + "passed": true, + "details": { + "matched": true, + "mode": "db-tls" + } + }, + { + "scenario": "Collector accepts api", + "passed": true, + "details": { + "expect": "allow", + "matched": true, + "mode": "tls" + } + }, + { + "scenario": "Collector accepts worker", + "passed": true, + "details": { + "expect": "allow", + "matched": true, + "mode": "tls" + } + }, + { + "scenario": "Collector accepts dispatcher", + "passed": true, + "details": { + "expect": "allow", + "matched": true, + "mode": "tls" + } + }, + { + "scenario": "Collector denies stranger", + "passed": true, + "details": { + "expect": "deny", + "matched": true, + "mode": "tls" + } + }, + { + "scenario": "Collector denies foreign", + "passed": true, + "details": { + "expect": "deny", + "matched": true, + "mode": "tls" + } + }, + { + "scenario": "Collector denies worker no-cert", + "passed": true, + "details": { + "expect": "deny", + "matched": true, + "mode": "tls" + } + }, + { + "scenario": "Broker denies stranger", + "passed": true, + "details": { + "expect": "deny", + "matched": true, + "mode": "amqp-tls" + } + }, + { + "scenario": "Broker denies foreign", + "passed": true, + "details": { + "expect": "deny", + "matched": true, + "mode": "amqp-tls" + } + }, + { + "scenario": "Broker denies worker no-cert", + "passed": true, + "details": { + "expect": "deny", + "matched": true, + "mode": "amqp-tls" + } + }, + { + "scenario": "S3 denies unauthenticated identity stranger", + "passed": true, + "details": { + "expect": "deny", + "matched": true, + "mode": "tls" + } + }, + { + "scenario": "S3 denies unauthenticated identity foreign", + "passed": true, + "details": { + "expect": "deny", + "matched": true, + "mode": "tls" + } + }, + { + "scenario": "S3 denies unauthenticated identity worker", + "passed": true, + "details": { + "expect": "deny", + "matched": true, + "mode": "tls" + } + }, + { + "scenario": "Worker least privilege db", + "passed": true, + "details": { + "ddl_denied": true, + "jobs_read": true, + "messages_read_denied": true, + "mode": "db", + "password_read_denied": true, + "role": "worker", + "session_read_denied": true + } + }, + { + "scenario": "Worker least privilege s3", + "passed": true, + "details": { + "forbidden/security-denied": true, + "mode": "s3", + "originals/security-denied": true, + "results_read": true, + "results_write": true, + "role": "worker" + } + }, + { + "scenario": "Worker least privilege amqp", + "passed": true, + "details": { + "configure_profile": true, + "mode": "amqp", + "publish_profile": true, + "queue_read": true, + "role": "worker" + } + }, + { + "scenario": "Redis scoped ACL", + "passed": true, + "details": { + "config_denied": true, + "foreign_key_denied": true, + "mode": "redis", + "ping": true, + "role": "api" + } + }, + { + "scenario": "Matrix allowed https://otel-collector:9444/metrics", + "passed": true, + "details": { + "expect": "allow", + "matched": true, + "mode": "tls" + } + }, + { + "scenario": "Matrix rejects unknown identity https://otel-collector:9444/metrics", + "passed": true, + "details": { + "expect": "deny", + "matched": true, + "mode": "tls" + } + }, + { + "scenario": "Matrix allowed https://otel-collector:9445/metrics", + "passed": true, + "details": { + "expect": "allow", + "matched": true, + "mode": "tls" + } + }, + { + "scenario": "Matrix rejects unknown identity https://otel-collector:9445/metrics", + "passed": true, + "details": { + "expect": "deny", + "matched": true, + "mode": "tls" + } + }, + { + "scenario": "Matrix allowed https://prometheus:9443/api/v1/query?query=up", + "passed": true, + "details": { + "expect": "allow", + "matched": true, + "mode": "tls" + } + }, + { + "scenario": "Matrix rejects unknown identity https://prometheus:9443/api/v1/query?query=up", + "passed": true, + "details": { + "expect": "deny", + "matched": true, + "mode": "tls" + } + }, + { + "scenario": "Matrix allowed https://loki:9443/ready", + "passed": true, + "details": { + "expect": "allow", + "matched": true, + "mode": "tls" + } + }, + { + "scenario": "Matrix rejects unknown identity https://loki:9443/ready", + "passed": true, + "details": { + "expect": "deny", + "matched": true, + "mode": "tls" + } + }, + { + "scenario": "Matrix allowed https://tempo:9443/ready", + "passed": true, + "details": { + "expect": "allow", + "matched": true, + "mode": "tls" + } + }, + { + "scenario": "Matrix rejects unknown identity https://tempo:9443/ready", + "passed": true, + "details": { + "expect": "deny", + "matched": true, + "mode": "tls" + } + }, + { + "scenario": "Matrix allowed https://loki:9443/otlp/v1/logs", + "passed": true, + "details": { + "expect": "allow", + "matched": true, + "mode": "tls" + } + }, + { + "scenario": "Matrix rejects unknown identity https://loki:9443/otlp/v1/logs", + "passed": true, + "details": { + "expect": "deny", + "matched": true, + "mode": "tls" + } + }, + { + "scenario": "Matrix allowed https://tempo:9444/v1/traces", + "passed": true, + "details": { + "expect": "allow", + "matched": true, + "mode": "tls" + } + }, + { + "scenario": "Matrix rejects unknown identity https://tempo:9444/v1/traces", + "passed": true, + "details": { + "expect": "deny", + "matched": true, + "mode": "tls" + } + }, + { + "scenario": "Grafana cannot ingest Loki logs", + "passed": true, + "details": { + "expect": "deny", + "matched": true, + "mode": "tls" + } + }, + { + "scenario": "Grafana cannot use Tempo ingest", + "passed": true, + "details": { + "expect": "deny", + "matched": true, + "mode": "tls" + } + }, + { + "scenario": "Collector cannot query Loki", + "passed": true, + "details": { + "expect": "deny", + "matched": true, + "mode": "tls" + } + }, + { + "scenario": "OpenBao worker policy allow/deny", + "passed": true, + "details": true + }, + { + "scenario": "Shared login rate limit returns 429 Retry-After", + "passed": true, + "details": true + }, + { + "scenario": "JSON body limit", + "passed": true, + "details": true + }, + { + "scenario": "Logout revocation", + "passed": true, + "details": true + }, + { + "scenario": "Plaintext backend bypasses unreachable", + "passed": true, + "details": true + } + ] +} diff --git a/evidence/source-manifest.json b/evidence/source-manifest.json new file mode 100644 index 0000000..3742db8 --- /dev/null +++ b/evidence/source-manifest.json @@ -0,0 +1,242 @@ +{ + ".dockerignore": "b97272d9afabe60343197865f06df5797e324ecbac6f9abc896680adc63efef8", + ".env.example": "2b82450b1a8fb4e8ae8b2665bc66060e83780985d5d54b235b9c22773a22638f", + ".github/pull_request_template.md": "43cf3184751513eac7fb36e17e42ba9c2f897b8b8bd1f98910d4622b513121d1", + ".github/workflows/application.yml": "3fd928cbda2e51bc82129cde79da7cd686ab50e0e0099a78a42da62292d7befd", + ".github/workflows/contracts.yml": "a45d7932ef14130aff8020bce79f043cccaf9692d697088a754d309a03f7d3e8", + ".gitignore": "a8e1b288a5369fd02317c7fc6e67858c7341ebc62c665092a501e590bbc38136", + "CONTRIBUTING.md": "2c01dd698cc9653dece49a94c9fd3f58d36e84764aaa643b46e26d7e599dc12e", + "COURSE.md": "c34d53cd082d194aa4b58f504c4e0348d25b6afec3d6a9c0963dd01c9c81e971", + "Dockerfile": "52309424c0eb41f904e1e45e6b848add12c8a2e1414b31718f4efc133c71cfcf", + "Dockerfile.example": "fd7ac9491c0c3c663b083c41a762fb38bc2d212fa4a2ed3e6a0702af72108709", + "MAINTAINERS.md": "6e10c41ab1f287d3ef93cdeadbb22d49651a53e8b9d2578b279abed3931b20d1", + "Makefile": "b4bb788f68af4f12ed3b360bcd8a6e02b9c2c995e5679b9099dae10621a0943b", + "README.md": "7fd1e422bb069bb00351dfb7ebca14c3b112b1872f106f81c61a621c5ac35a39", + "REPORT.md": "6143098476828fec2c315a8110c65d2b59a86f700a7b1e1c0725f825d825d2a5", + "compose.example.yaml": "abcd9e7cfb57d3854a8362550cf36a0030cd417eac2a342abb5cb6c72c0e1896", + "compose.faults.yaml": "3a045ad5532c8f7e5ddb10e15562f83c69e884d2e5a8418de4571ede31356278", + "compose.yaml": "7294aba715e2f4b130e3cf14c84aab3babf590ce05fa1d50cf4b6d633c2635fa", + "contracts/README.md": "72df27a559f88ffa3fd8c3c79d3c32f6db2c5986a0602677c51994c434a3fc4f", + "contracts/attachment-states.md": "cf0519e137410e4b34ab632aadb8ad8a381d4b73033e27b354c58eeacbdf86d6", + "contracts/examples.json": "381cc18c03789e935b391d007fb5843964be9cfd9f7530cb6b1996ed4a1c4c37", + "contracts/infrastructure.json": "90b706e85ee2fcaa54e1113e98206555e0cd678caf03204c9ac363a6f1323d30", + "contracts/jobs.schema.json": "9ccf9b11466c0c3a26a941520ac5ff543241fc4935e3a6eebb01ef303779b10d", + "contracts/observability.md": "882c120c0e153a06c9cc824a802e317c58eb6590bc3d3a576f2e0a96f8597b62", + "contracts/openapi.json": "51dd0b8199aee7be81437056144c5fca9d82c2b4387dae62585cadf230ee749a", + "contracts/trust-matrix.md": "3402b5ff28d1fa559d1b960423fb264a0552a74da0725f63eeaf94a9295f7d80", + "examples/requests.http": "e69880f740f1e346354b519f14487d2bfcf1a41cbaa8cbedc61d258e7592fd3a", + "infra/README.md": "436e16ec36e58ff53fa10c3e3d0e40cc0133d909e417ca35cf88a27cd2e5f6a5", + "infra/envoy/loki.json": "f4995cbd4bb5f192c771d7e85ef76b60cd54a16a3e7755c18abf672db04216bc", + "infra/envoy/otel-collector.json": "e294440309c7a46f6e721dbaed411525f9542f553e0e4bf21414d791b4fb3a1e", + "infra/envoy/prometheus.json": "9a5ec99fc64672271967e1c886d0fd9ca4a780dfe0a69e22acd8430f711a44f2", + "infra/envoy/s3.json": "5b1eddd549bb193ec2e6c3c020b78efd4d52075af75fc95ccd1fe86c500a259c", + "infra/envoy/tempo.json": "2c0761b93c1132460e2592daf416fb7e6803af324a49134b02a5bf827be99558", + "infra/generate.py": "43d10607e754a0ec8d816d857f74606765cc9ec865dac650be5445cd6c482aec", + "infra/haproxy.cfg": "759ab33fcaedf0fe1ea7119bf583b5286c25aed22f846654326828e0cad81be3", + "infra/imagemagick-policy.xml": "c362e5c794f9e700e429a39f734371fcbc79aef529f8a59384f602d36fdd37c8", + "infra/openbao/README.md": "394889bdd5fc06069d362eddea88bdf177bac898f30d7625adc9ddabffe0f21a", + "infra/openbao/api.hcl": "7c5fc792543ebe72c98aa45a337ed6ec9808783f6cc398a3bdd78ba1a3a14da3", + "infra/openbao/dispatcher.hcl": "a4af4534de533bdd3c10e0b8169d01aae42e264ae5f38582565bf86ef518f981", + "infra/openbao/server.hcl": "89de14644ba8579324e532a0fb9822c1f87d88604512063c721bc686932b699c", + "infra/openbao/worker.hcl": "72418eb4c7c2e779745115b4adf5c078bcf76c542e9fcd239a5c9c228d2ab43f", + "infra/otel/README.md": "7c9a128d7e025b11c76687343d237706a5d9ae90a9eff0e4d1d859bebe3602e0", + "infra/otel/RUNBOOK.md": "18d9ef9d55629aea30baca379ad5496f809ba641ad5e8cf9a9abaf9414973628", + "infra/otel/alerts.yaml": "f737f1188a2a52247003bf62a434a020e0b5e5d51705f7f8ca1c2c99a41372fd", + "infra/otel/collector.yaml": "0fb3eed23283bd317a28046f7327c0baf962e74fd6a949f52c696a3df8045fcc", + "infra/otel/dashboards/messenger.json": "886e31505dd2a4fd0c4b12c4e2889b848855605afe6c87daabe72a3f951bc25f", + "infra/otel/grafana/dashboards/default.yaml": "6d9f3f36f8fb66d875642da4aad154ce1cbddbb47c728c15e491cbbba94ea562", + "infra/otel/loki.yaml": "e187d31180b8239ba5a330e97452dc82f42b1352a2a6a475acf7ec170c36d56d", + "infra/otel/prometheus.yaml": "18d0020cdadfa931ed3808509af7f79cef203f54a012fbd0cb43e175f4df2309", + "infra/otel/tempo.yaml": "c38f3bbab532e919ad0f2eb6373efac05bd79c0c77b692932e2e0d059dc60958", + "infra/postgres/grants.sql": "a4470970f9decf757c36327eafc673f1bc81fab95d815ab717e62c8764144e67", + "infra/postgres/pg_hba.conf": "5e189df0d2143d5a4c18d21c6d6d1aabfb5cbc4903adee3c80945099b5f04f70", + "infra/rabbitmq/definitions.json": "b1dd7b5c1cc196ecc833b696fbd80671841c02789182888adbe71fc8150a85ea", + "infra/rabbitmq/enabled_plugins": "2bed3e041f079e6daef14bf6dea9d9e3e12465922f2d1ca6eeaf973c4fa0c211", + "infra/rabbitmq/rabbitmq.conf": "53f78364aa921977af08a7cb9e8d445342420e4d8ea61ecfbd71ceb814777f46", + "lab.json": "f4cc620370d5c84be3d7a4422c48c05398cb28aa89185d4141b534b58e4b51c4", + "requirements-dev.txt": "4cefdb67007caf026d1613be7c519c01b4c8343db7c4f4b618677e01cf113525", + "scripts/audit-secrets.py": "d2cb73887d06b5866268dee9f4045b2a25bfdf80c3a06b2f4201a339730f4925", + "scripts/bao.py": "1d2b18456a6184e00c1ede2ddf701a9a0836f9232aedbcc0081d98547a6ef332", + "scripts/bootstrap.py": "47ba4cfda9b420974526c211f49c5b720c4b44c8a081235d4f9e673d2cebe040", + "scripts/build-image.py": "565f13519fd8852ccd2795d24ed8dd20e112ac7c0c687e2f278ae1619650f16e", + "scripts/capture-environment.py": "d49094d291cfbb380835e2a3255d1c9f86c4670adefd28a40cebab8eed6fb68a", + "scripts/check_contract.py": "7c4402c2a6ab80d3b4676e1866d6ecc87b6daea14c080bcb9be9a7e8b959d671", + "scripts/compose.sh": "7f4abd7f232ab7096482d4f25d410229b707d2a98e666393b23c68e4b3de45f7", + "scripts/experiment-worker.py": "aa0f21de0bfbccb41dfd40b157ef3314ade64168e3d651dd6a51055d85cac669", + "scripts/final-checks.sh": "c4a00b9287b1a9ff9ac5e8a43d10618d1040d89c44337c54770b2a40c0f898ec", + "scripts/generate-types.py": "9e88eda11718e6df8f9816ea2c643a4e7d6c34fa4869fae854c47b2882fc3798", + "scripts/prove-admin.py": "198c88c95bc793d44ab233a45ba6c5b306c66c5950ffb72cd1a86d4d2776f864", + "scripts/prove-all-replicas.py": "38b9fdb1516693f8108a3d2a48c9130c1aa17f4867668ed6608dfbbe18897ec8", + "scripts/prove-auth.py": "a50f516e426a04ca1f133408e5968b1e324fdfcd6ba68c453fa25947fb5e4ec8", + "scripts/prove-cert-expiry.py": "b55d1e60b6ce3571b365de6d7b1e94ac775487419e31d6b2a6cae2a6da1fe650", + "scripts/prove-clean-clone.py": "72edaff9f2461bdfedf00713ea8de4c990bbbbf24ab956de6f23402afff5d66d", + "scripts/prove-leaf.py": "c44ed204efd52c774f7d2069552c8517935eb94961c7d94c0dec9c5930d3775d", + "scripts/prove-sealed.py": "cde24884a75daa4885fa1b6fbb4aa1b4988834610511cc8873440bbca624819a", + "scripts/prove-telemetry.py": "e3f5aed0934f044aedf09c5eaf8afb5be445e3650aba66f3739ae4a5bbc01064", + "scripts/renew-leaf.py": "603024c424990c558387fc3a9eb0a5b18958cef5fa9907e192eb03c1ddcc124a", + "scripts/restart-evidence.py": "bb75b354a13305c5dd0ad6b0c6533025434715fdd3f30b560c46dfa5c09104c0", + "scripts/restart.example.sh": "0328246ba75defbced992104e986fde973f63e3830d83251eced5953263e944b", + "scripts/restart.sh": "dbb5117429f0386741d60bc4096c6835949c0cc853e55e18d777709599681675", + "scripts/rotate-worker.py": "ff544fe9c504025465f01df8c197c993581deade778c79fa01501387e419e5cc", + "scripts/scan-images.py": "f53604bb699417f8c317dd700fed0ad918f79bfadb1122613b609dfa3e281843", + "scripts/security.py": "27d6f6fd22b2c1fa5dced045ac9154035fba87b1a8ff6df8109d20b98622bc59", + "scripts/stop-one.example.sh": "1bcf34394520b89d95b7605ec56675be799eea920e83cfbba306d5dc0a6d9811", + "scripts/stop-one.py": "a061d00d2dde7d9190bce7e83b701ce6994a0341b5c449899b9142a02d340356", + "scripts/stop-one.sh": "969f832212f8e3399446dad8fecd033267ec6676ef2111c5726c365698c5c206", + "scripts/summarize-evidence.py": "fdef2efc800ce81631dcca7aae974abf435d19a800d184d544d64ddf773b73fc", + "scripts/test-storage.sh": "71225896d8283d4bdcc9be4105abfe98d6d6000b85b67539075cbed80a1a596e", + "scripts/up.sh": "4f220b7f058ecd343a03f2106955872a59055a7c2fa798bc21a58ae7db79e499", + "scripts/validate_openapi.py": "4e981acbfc10dac58817bf922d5896e996798551fedc1b2385130e1cddb13d54", + "scripts/wait-ready.py": "5b8c61077acc07fa58786732f114b6691d619da4205cb5dd8e3a05078a81dbd3", + "security/SCAN_TRIAGE.md": "a66c2a11f81a186dad424496f2f7ea47a46db0818446420894fbdb51b1860df8", + "security/THREAT_MODEL.md": "7ec25a768b9ee2d7acb7e8a07576a413e881c3240a0909c957e7c8101c9d13d4", + "service/cmd/app/main.go": "dd04e7b22f8426b0b57b40e5cd4c455fff23cfc9d26cc42addd52ed81585cc15", + "service/cmd/app/pipeline.go": "136355cb8b52a391631068f214405a82f83e7875dda704a16f4a966e95376051", + "service/cmd/app/storage.go": "c21c04e969c835f593890c4c33706ab1dc9b501526a05d9d158f235971e2579d", + "service/cmd/benchmark/main.go": "26227a4eedb4e80de5fb011637a9df3022fea5d56a5cb45722101ffc5524bd1d", + "service/cmd/benchmark/main_test.go": "7304e7056582f8f0ae4130f1fcee0d453d3426036e8c5990581dbf09bc61f01a", + "service/cmd/securityprobe/main.go": "e90761cb1511c85ac4bc4a5253896483c915e498449fb413adeb580b903da6de", + "service/go.mod": "2ef766ef7afdab1ec443e57b39e06cbfdfe2bcdf723cf43169eebd2570d698c7", + "service/go.sum": "c1cf821074b60d627c45be58496f118d51bd6c8017350ba623dce556c8f23fba", + "service/internal/adapter/controller/http/attachment.go": "829cb066e16841f872c8d8067a74b091c2430a3d1eca9365aea6d8b36b40e7fb", + "service/internal/adapter/controller/http/attachmentHandler.go": "83673b5f0733be117df90c34e9b7e5b6cab2d18e6481edd7a0b006f5ef10f31e", + "service/internal/adapter/controller/http/auth.go": "43b3f24df2bf76cf26449a9a0b2d4bb1ad377ef03f94fea979b10c6b5a1da8dc", + "service/internal/adapter/controller/http/chat.go": "c3cf09fd828c18a0e7c46549ed4e3160b8b22118057f46c63ff78d0df3704b2b", + "service/internal/adapter/controller/http/controller.go": "fdd74a6b1ff5c621ef9d0dca459cb6e951e26fdb044270abc6a5ceef6ce28d8b", + "service/internal/adapter/controller/http/handler.go": "1d6007f299ef85c0dec94b35e67a1dc37ebcba94d5322bbb6fe19c1f7c9fccc7", + "service/internal/adapter/controller/http/health.go": "25cd15594bc634ab8f4cee51588763c057f22cdc140a984abc42af60cafa7a73", + "service/internal/adapter/controller/http/router.go": "829cb066e16841f872c8d8067a74b091c2430a3d1eca9365aea6d8b36b40e7fb", + "service/internal/adapter/controller/http/user.go": "25d26b1ab24eb474e21eae547e2a4cd4fef8b95570bca016d9639aea307d46c1", + "service/internal/adapter/repository/cache/outside/chat.go": "d9f3f1677703d62e5a6a9cb28cb5264ff8b9cda25c944be736c8afd211d4abfb", + "service/internal/adapter/repository/storage/attachment_scan.go": "d2c9ba746141b939524298051dffd4edb1ded55e3fb2bc47725a5566a9fe47a9", + "service/internal/adapter/repository/storage/attachments.go": "fe80b68d72eca5f776580fd56875f9f045faf57eeacdc1fbf857f1e2b7a31ef5", + "service/internal/adapter/repository/storage/chats.go": "d7090a47013a3d207646e0afa8a13a49f7b4387de3e04f8a8bae7427c116bbcd", + "service/internal/adapter/repository/storage/idempotency_test.go": "f075082c25e667e3929b17dfa67860c8690808c607e224bdf3f9b8569c0f8bbc", + "service/internal/adapter/repository/storage/jobs.go": "627bb00a98d835143c5d437ab221dfdf809784bc44f58cb987cec6502d08987e", + "service/internal/adapter/repository/storage/messages.go": "3bbe58606b7e989b5c1e5b1a913c59880276dd259e4f2392c38e5bc48f3f69ff", + "service/internal/adapter/repository/storage/observation.go": "5feb70ce2de6abec4d10a9d7d3cfdb2277230de78bd8d5b69bede959e4d07d58", + "service/internal/adapter/repository/storage/outbox.go": "09f5b1242ab6c0729f060bc27011dc8af78114e9a4e4dcdfa865da7fa3461748", + "service/internal/adapter/repository/storage/refresh_tokens.go": "30ad6ed103b9f6c2e551df481a27c0a41dd2ab12ba9380b75a65fa5909cbaaad", + "service/internal/adapter/repository/storage/users.go": "8f54120a68e7050385a3b16d6e41e208d81eb711c62cd4d2bd9e726802c7d07a", + "service/internal/config/abstractions.go": "47a6d954d5789640bbc1cab93fd8581cee33d0268beaf40e05f1bdfa0714f781", + "service/internal/config/application.go": "7709c2f37d3c1b1d9c97b30d0a4940e6f34878e7ee475fb9b99fa6f898f65bfd", + "service/internal/config/attachment.go": "9f1705873d7945f5510e4fd9159e65f08629ac329c631576b3c05fda048fe28e", + "service/internal/config/config.go": "b2f8655c78e922b1b87e0ab279f6c545dd9bbe4244a3d6ed36bd6246dfba52bd", + "service/internal/config/cookies.go": "93955e1f5d6734afb079772f12e003ac297c7dfd8f1a9bf5d43ca0dfa4a6bbf6", + "service/internal/config/jwt.go": "10a57daacf3e41c4ac1ae6b5d5a1b476962873207bd256e154dc5aaa34ac64f0", + "service/internal/config/pipeline.go": "22948e39770d8bea102bab20f653557cb864b97edfc1e69520e2f88d27bbc07a", + "service/internal/config/postgres.go": "80389e45cce562ddd1e359774ea1a433f6d2ac2011a11b15576580609030ede9", + "service/internal/config/redis.go": "f7cd8c4785f39e5e4478231617b935a4879ac290e98202734a712273a72dc037", + "service/internal/config/server.go": "c3203ed1105edbf3a3197951e628e375035b599fa4bbe7429bdb0fb9f11198b1", + "service/internal/config/validation.go": "d8115ca776021cdd8d44573740c2d5fdf7f379c6909d7abd39d72f31d0f9ef98", + "service/internal/dto/attachment.go": "33df44b9944cb0e2e377bd241e7c0cc459634f7379607f244b140f1ce68dd6e8", + "service/internal/dto/auth.go": "1b50ee60aaa0ccd4c62ed4e4380390fdf9c08365cbfa2d0c324c7c376397b1c9", + "service/internal/dto/chat.go": "29e54d7b24235a580247b934bb7659a112539046d49046517cbeb64f2b9954bc", + "service/internal/dto/message.go": "d4ce6456453f2f3ec66d8456dbdf7828de44452df58b2497c37e1e281a88697a", + "service/internal/dto/page.go": "819dfe56a078cf7032036a7e5942043a5f5ecdb77a10dcffafc65905b4cd55c6", + "service/internal/dto/user.go": "ddc9ad138746e7b932a2bdfb72a1b51e26f1aa14f5dd8a71a201456edc43f4f0", + "service/internal/gen/contracts/types.gen.go": "c9c09568311a214d7f1416e59739bcd7b7d860defc1881398b02512807eb484f", + "service/internal/pkg/apperr/code.go": "d882d85353494f7f1f28f8d7afe0031ed066e1e0eb9a146251448275305c4302", + "service/internal/pkg/apperr/code_string.go": "48651daf286010f8e2609c3601a44b2593c81235b4eb659cb17c954255a5255c", + "service/internal/pkg/apperr/error.go": "f56a5a6e636d1c71adc1886e91caef924517c478e68c398d3c679658acfbc483", + "service/internal/pkg/apperr/meta.go": "e9c37b9d40bd072d39c19a0fed947f8c798cf22ca8234b2dd3aef83c937b1d00", + "service/internal/pkg/apperr/metakey.go": "cb68597145dafa8053f77d438394afefbdadb070ec4f1cecd5b468c0123b809c", + "service/internal/pkg/apperr/postgres.go": "556d77a2c95a3cc3c9019d3a5f6bfb5d1c7df8b2c8f125d5f0b519eea09b4fab", + "service/internal/pkg/apperr/redis.go": "45a8f0f1c2f5984593999170333b8c7d6f5a6b5d49f7f60c0ca4546149dc1d45", + "service/internal/pkg/auth/jwtprovider/claims.go": "ba12f8aed6a5c34f2ad157eca980c9030c1a7f97ebc848fd0f3ba9d01a1cf5c4", + "service/internal/pkg/auth/jwtprovider/cookie.go": "4aac299144847976ea62e491eacc84723e3c9f0a1b1d7f34ccad8f938f830559", + "service/internal/pkg/auth/jwtprovider/error.go": "6c5903a4b1a35ba7841e57c1f160364de1778095bd37385a8f7579b11d586e89", + "service/internal/pkg/auth/jwtprovider/interface.go": "d88e563c9e785710d2e38a4ef8f6d94dbf8c4c60fff7306ec4d91f5445da3e19", + "service/internal/pkg/auth/jwtprovider/manager.go": "f1db7ad279b6d3e1900453bbdeae03ff405356403c1ab63629e92cee690b5644", + "service/internal/pkg/auth/jwtprovider/provider.go": "1606d09846f676d84d6930b1f849de36a65b8ffbb00fb29adefa43edb9afe8e8", + "service/internal/pkg/auth/jwtprovider/readers.go": "d7064e7be222be43e9927e407429d3e3136186fdb3e410f09fcc142999fcc9b2", + "service/internal/pkg/broker/rabbitmq.go": "806be6250487de277e270e96f2641b644cbf6f0084248cb774408e497b6fc812", + "service/internal/pkg/cache/redis.go": "61e21e44c41628d4c8aa923b901a29398d7e9c41ffbec71e9a310964bf25b1e4", + "service/internal/pkg/encrypting/argon2id.go": "6ef700f03b84c52b186435b9be244bd03c37474d4d90a8b100f4c3b545c2260e", + "service/internal/pkg/encrypting/encryptor.go": "b07128f65ddca6897260d9bfcf232eafafeab5ca01b141c4f80b4c0966d2c260", + "service/internal/pkg/encrypting/error.go": "13a893c0f7ec390497561df22decac9bc1f3142ca4def69818cd79fe8c659f5b", + "service/internal/pkg/encrypting/interface.go": "ca25defd6c7bfabf24009149e3d6e4975484a76d1c6182b78e6abf3238d1aa62", + "service/internal/pkg/encrypting/token_test.go": "44769661e8fa8ac116462e300ed518c2f4743c7914b7c32de0fb691f4dc28734", + "service/internal/pkg/fault/fault.go": "bfd659aa7ef2906a182ab0714d73ad63a90c6dd9ffb535af69a0c101b0e629ba", + "service/internal/pkg/imageprocessor/image.go": "7854c17607848c000ffd3dc7179f568b27cce03d3782464cce5d5445ddf563f5", + "service/internal/pkg/migrator/tern.go": "eb2e704c1aecac44a789e8938fcb12b9cf98d8ec66d54477ea48513ad698daab", + "service/internal/pkg/objectstore/s3.go": "98e39fd6a259d9fc5474783b7b60a110a59efeacea5714d54bd8f7273879fb91", + "service/internal/pkg/openbao/client.go": "7d2a1344b3cb42aad62187534a19af3c56192c5c44bf49dd46e5d82ff4994489", + "service/internal/pkg/postgres/after-connect.go": "638764a8143e038e51dc894bda268048eab6354e9f865bacfeea0480f10b99bf", + "service/internal/pkg/postgres/config.go": "25120b03517c8e933e10a3a78bbc46567d273ad59c7108d1abfeb4f298080af2", + "service/internal/pkg/postgres/err.go": "da004b6167345f88139839f5d1720effd5bf92210b9c00a9e6cd5b3a8a1ac11a", + "service/internal/pkg/postgres/interfaces.go": "d216fe34cdfb0448fcd644f9f2262a4541676e0aa89275faf3760099bc52e388", + "service/internal/pkg/postgres/postgres.go": "53eeada1a98924b1fbab216d2027261b8fd91844965ad464c0a1226d3c8bf030", + "service/internal/pkg/postgres/txmanager/tx.go": "9f3ed43fffb91dc46525e583464ec04b773258a255bfe7580fa93eb4668bc312", + "service/internal/pkg/ratelimit/redis.go": "d005d19f6eb60d58f15ed631bab624424b8fb7b043bdbd6d78ecc325702c64e4", + "service/internal/pkg/rest/framework/framework.go": "2cffaaa3b92ae25b2413f2c99ae4af247e8e623dcc1eddd81eb2923c7ba4803e", + "service/internal/pkg/rest/headers/headers.go": "077b18cebe3627cb4c9b57e5c9e8ed77798ebfb3245dce5df64ea3b89525e705", + "service/internal/pkg/rest/middleware/auth.go": "50784cbcce1450ea86ea9542031a9731b46f58287817045cf30c3b7009862243", + "service/internal/pkg/rest/middleware/auth_test.go": "e01e16e8fd997e36836099a4f650c06a6b12c8eb7839bb3bea9d52e86fa01046", + "service/internal/pkg/rest/middleware/instance.go": "8e9877aa38b0c5ea4778e524354513ebea83036fca57d2658cbc94397058aeac", + "service/internal/pkg/rest/middleware/logging.go": "5a2a3a0a83e0aeccc0fd71d9814df8bd66cd3c3528c3fa0485b016b673862231", + "service/internal/pkg/rest/middleware/middleware.go": "b8b06a2dccca8d794f6200b6ef7dbad438182cbebc39566f828adc2578996010", + "service/internal/pkg/rest/middleware/request-id.go": "c901675387cfcb4c3446614e0cc7227148f8a6ffc09f9084e3a3ea9137af1acf", + "service/internal/pkg/rest/middleware/timeout.go": "bae9f97b85ac37331cb9e66f47961d7cff0e605aabbe068bc268456c43460c0a", + "service/internal/pkg/rest/router/router.go": "345ef7619886fffc6d0c421135a047eef2cd74bf7e594c3a0b25e71f6e811d4a", + "service/internal/pkg/telemetry/http.go": "f210cea7f7fe6b9716522b452a1226eeec396fa5a893d453e0c008679733c586", + "service/internal/pkg/telemetry/observation.go": "10137f5cced7c4fbe1794102e73938f227598e04bda82722dd4e1875b43781a6", + "service/internal/pkg/telemetry/process.go": "c455581bbf8142dc8518cee7698f03f41033a3a3ba217b79bd784fcb0a98f0e7", + "service/internal/pkg/telemetry/telemetry.go": "eb3b69beaf199daa29ae8d38de87fe688328577a912092e8257e30755fce96da", + "service/internal/pkg/transportsecurity/tls.go": "f7032135845c70d5e84b42f49e2389c9d34efb32c0df03bd2c6343d0ab5cd5b6", + "service/internal/pkg/types/authprovider/provider.go": "4fb06d1e114a99eafb4da34c8d7ad53594d55b438c80e5afed3fe1ccf3f979b6", + "service/internal/pkg/types/environment/environment.go": "c6cfd877a2c492123ce23d0b34432d6e951d606cce161c65d924027aa61b2954", + "service/internal/pkg/types/error.go": "6fee4c45b6e08854d21966bb89091dfb1c50b4e577257156c56dffa5f7347390", + "service/internal/pkg/types/role/role.go": "611d29787a97aaabc8906fe3cb99729ace95129910daa7ae273623395e27112d", + "service/internal/pkg/validator/validator.go": "7c6557925b86247aa5e82466d340c2f99a67499bff57b61233a014c2187dc6a4", + "service/internal/service/attachment/interface.go": "38bfff542af5abfc40ded1c5dd3abd7b08786662468849ab3f897783fa4f48a0", + "service/internal/service/attachment/pipeline.go": "c44faac7a73c7a69819472b990cf659705be9d71e6eb695e2c790e5b0988c44b", + "service/internal/service/attachment/reconcile.go": "130bf1199b532ef52eede78a4eea4448f19602bb013f6268d4af30ab70f68eb2", + "service/internal/service/attachment/service.go": "ee6d08f47eaaf52dd93dfaaaee0d517bbda9ce0b494a9e1b73bd52122495247a", + "service/internal/service/auth/api.go": "65b6656a2549a6f6c5b69217da57126395648c420d9bbb9c78c432c194ea8f0d", + "service/internal/service/auth/interface.go": "0266c72369881fdbcdd330a48c52cb8745214725ae02c42f12bdf5fe016503da", + "service/internal/service/auth/svc.go": "d26cadcabcb3b930296f6b5f7597a16a516bf3db867c5804d03dd072e1899173", + "service/internal/service/chat/api.go": "490448c738c3e27d09476f3530a5bbd3edc7872a7b9116195987d73285f8e6cf", + "service/internal/service/chat/idempotency.go": "d9e188147141c068685d58ecf3b2d6969bc35e841bb7451ec30b8a9d60bd0b1f", + "service/internal/service/chat/interface.go": "3a0562b5907da2f642a7b630d15aea3a6be954428d09873862b3b37678e93614", + "service/internal/service/chat/svc.go": "d1b325f58fb4b1b0efe27a3dc0d988219effd84ae1a18e9450182a5cdbc845bf", + "service/internal/service/observation/service.go": "014154616843ad4fcc4d1fb46da11a2cdaa7cd8b71171b61e88fb0146a530c0f", + "service/internal/service/outbox/api.go": "6a1088d0219f32f92b49fd270d38f1bb396af8eb82856612e4359088c52b514c", + "service/internal/service/outbox/models.go": "2a28401f1a9ff36fe0a852269717479f599243131b24f76fba287aaa6601865d", + "service/internal/service/outbox/status.go": "40ac7300e89dab55a9d5f7c71a128507ef9c28483c0a42f3953242eef9ad0205", + "service/internal/service/outbox/svc.go": "747d208f628b4320ab00a7b9f9867d5e9b80d0e540ea5cbd0d20d5ea95872f82", + "service/internal/service/outbox/transport.go": "79c58cddf2ebcc78b4ed3048cdb0bee0d57e1be266f1742cf76f77f92855cc5d", + "service/internal/service/user/api.go": "a702cf6c410eb57006543dbc341a0eb6faa2fa300bd256532ac150a00eb205f8", + "service/internal/service/user/interface.go": "ed8905f680e533e7f13fd09225f908d46e78945b06720b080c48c7302a4fb5fb", + "service/internal/service/user/svc.go": "0dbbca7ee6fda228364ae29f40d4a94e063737a97e630b3de48174e4feb56005", + "service/migrations/001_create_users_table.sql": "1b471cc1999a4074e85f313d614a16d4ca29cbfa41b364205b6716bdfe6faf91", + "service/migrations/002_create_refresh_tokens_table.sql": "a413f0e480c3eb0ca1c6b0f2f4e9c2888451534a48fd244de208b5cf2686a656", + "service/migrations/003_create_chats_table.sql": "5eb5754b38115ba993063eee16b5ea4a771157b4fcd2c2e1d98b00615084d388", + "service/migrations/004_create_chat_members_table.sql": "d6d6fc46a4ffea85495d2273fef45c0685353e639a9d251ece87f83a9f127c29", + "service/migrations/005_create_messages_table.sql": "b34ef70a31c577ccfe932c324bd4e7306c35863fe0a6722fd24134a1d07beb33", + "service/migrations/006_message_idempotency.sql": "b330000aac4ad29135ae2616ca4081a2bd79570f86e1167bcc2ad29002a787cb", + "service/migrations/007_attachments.sql": "a6c7b7a6211dd74ffd67e768792c67bf0d1e0f168de157f9cda166244fc39fcf", + "service/migrations/008_async_pipeline.sql": "587d2a5d473821b8aa9df31f1bf8dbfb93161c5164d38f0619330c6e4021a71b", + "service/migrations/009_variant_manifest.sql": "5f2a884981cc6c4ef42a17a7b9e9bf9f0fe7556c463c877b7e50fc0c67e48f12", + "service/migrations/010_observability.sql": "767bfc0f44c87d9f9146bc2f67f30f91f0007514abefcccd1945eae243e6b027", + "service/migrations/011_terminal_slo.sql": "c06fef77f3cb9694df1c1f0c8a771860d669bfd2ecad4635a3a1839989c52ccb", + "service/migrations/embed.go": "8dc35112dc91bb3b453c1032887c77462c027ce63e1e7eb432dfd03e2dcf5709", + "service/tests/api_test.go": "556486d1109cb54ad045b4f53843ae1edb0f47029dc838ee4b5635a2242d0a2b", + "service/tests/attachments_test.go": "9b928104a2157d5ae84b2e09e915d0468634086937da4885380b29f2e4402e84", + "service/tests/concurrency_test.go": "73632d5d4bf5566c209c3640eb82d4dd2c931c2a225708bcb2a0325641a3d2d2", + "service/tests/idempotency_test.go": "9d83697d5cdbc717b0d69762e8dcc823a81f515761a5f9c7819613a51aa687e5", + "student-tests/README.md": "6030f68e6e0c966f31f80709ef1b1504512d534066ef684ede54e7d8b0597b32", + "student-tests/broker_persistence.py": "db738001ed1f55f0153e81069a3c8a28f49b8fac59598b90f21800c430ed4fdf", + "student-tests/images.py": "f0b9e9fa28a472d88751bf9210823a9fdec7e400c973de1ddfbb6d8e73be2b7c", + "student-tests/pipeline.py": "5224fd0c3bc16da77e0f49e156ff9197b075bdf3a5b2a4c79191dd73c6583f46", + "student-tests/storage.py": "3efeba78a45d0a579e4725f7ae24a623fc70d12a225e0137a429caedbebde60e", + "tests/README.md": "e0bfa7e196797f6a5bf6e44d8ad2ffeb82651927ff5d8484eb1c2dda1ec7dec5", + "tests/client.py": "8d50976fc7c7bbb81e88a152f613c61ab25eaf53dba33c6a9dfab19f868d546d", + "tests/fixtures/README.md": "c001783760b4f87a7a8441a7ba55f5d080ff48e1397d3581d72a88d7691f96c2", + "tests/fixtures/sample.png": "99ea7533c441f79cd630d15ce4d58d1ec4e06f6f0efb3b77a5c088305cd1bd75", + "tests/image_probe.py": "a9a4f89181c8e0fda6ce42e1a1af32fb42d8e3ae519e314207127c9b9b448b13", + "tests/load.py": "6d3511ea719c7a7e841083e73d435c86dcbfd9f4e5c2a07d62eb9614b01d9933", + "tests/resilience.py": "1f96663a174b3978d24eff66f2741f5f8c9b788d3242261d37a972720c52788b", + "tests/security.py": "6560a56cc22981c9f06043709bfabe7a47fa7eab57e62741e95534ac5ddff8a7", + "tests/smoke.py": "52ac38eb343ff7d43e7543588e1da58ca06c9fce74b2b0d0bf0e4982bc23c1db" +} diff --git a/evidence/storage-run.txt b/evidence/storage-run.txt new file mode 100644 index 0000000..b26fb4a --- /dev/null +++ b/evidence/storage-run.txt @@ -0,0 +1 @@ +PASS: active source protected, ready staging removed, failed retention enforced, restore/replay verified diff --git a/evidence/storage.json b/evidence/storage.json new file mode 100644 index 0000000..a0e75fb --- /dev/null +++ b/evidence/storage.json @@ -0,0 +1,24 @@ +{ + "results": [ + { + "scenario": "queued_staging_protected", + "result": "PASS", + "attachment_id": "e9227a4d-862e-4c2c-8617-3853671794fa" + }, + { + "scenario": "staging_deleted_after_ready", + "result": "PASS", + "attachment_id": "e9227a4d-862e-4c2c-8617-3853671794fa" + }, + { + "scenario": "failed_retention_and_missing_source_replay", + "result": "PASS", + "age_simulated_hours": 25 + }, + { + "scenario": "verified_restore_then_replay", + "result": "PASS", + "job_id": "cba3b401-9375-42bf-8a39-821b0854d0c4" + } + ] +} diff --git a/evidence/telemetry-run.txt b/evidence/telemetry-run.txt new file mode 100644 index 0000000..5970116 --- /dev/null +++ b/evidence/telemetry-run.txt @@ -0,0 +1 @@ +PASS: metrics, worker logs and real trace queried over TLS; secret/body canaries absent diff --git a/evidence/telemetry.json b/evidence/telemetry.json new file mode 100644 index 0000000..150e5ea --- /dev/null +++ b/evidence/telemetry.json @@ -0,0 +1,1309 @@ +{ + "signals": { + "logs": { + "status": "success", + "data": { + "resultType": "streams", + "result": [ + { + "stream": { + "attempt": "1", + "deployment_environment_name": "lab", + "detected_level": "info", + "event": "job_received", + "flags": "1", + "job_id": "[123,76,116,230,206,123,64,198,190,50,60,242,91,207,1,197]", + "observed_timestamp": "1794925075550259175", + "scope_name": "messenger", + "service_instance_id": "7f5ed92c-becf-4805-9019-4e83885c2f91", + "service_name": "messenger-worker", + "service_version": "lab7", + "severity_number": "9", + "severity_text": "INFO", + "span_id": "e64482573b2c384c", + "trace_id": "88257285326bc8e7d5987f59ddc784e3" + }, + "values": [ + [ + "1794925075550241675", + "job_received" + ] + ] + }, + { + "stream": { + "attempt": "1", + "deployment_environment_name": "lab", + "detected_level": "info", + "event": "job_received", + "flags": "1", + "job_id": "[195,111,10,219,170,39,74,108,175,203,104,104,2,66,77,183]", + "observed_timestamp": "1794925108181130260", + "scope_name": "messenger", + "service_instance_id": "7f5ed92c-becf-4805-9019-4e83885c2f91", + "service_name": "messenger-worker", + "service_version": "lab7", + "severity_number": "9", + "severity_text": "INFO", + "span_id": "51447056258814d5", + "trace_id": "c039a0aec96327d8540867de25c37ce6" + }, + "values": [ + [ + "1794925108181118760", + "job_received" + ] + ] + }, + { + "stream": { + "deployment_environment_name": "lab", + "detected_level": "info", + "event": "job_ready", + "flags": "1", + "job_id": "[123,76,116,230,206,123,64,198,190,50,60,242,91,207,1,197]", + "observed_timestamp": "1794925075605667425", + "scope_name": "messenger", + "service_instance_id": "7f5ed92c-becf-4805-9019-4e83885c2f91", + "service_name": "messenger-worker", + "service_version": "lab7", + "severity_number": "9", + "severity_text": "INFO", + "span_id": "2440a75faf7ab0a6", + "trace_id": "88257285326bc8e7d5987f59ddc784e3" + }, + "values": [ + [ + "1794925075605651925", + "job_ready" + ] + ] + }, + { + "stream": { + "deployment_environment_name": "lab", + "detected_level": "info", + "event": "job_ready", + "flags": "1", + "job_id": "[195,111,10,219,170,39,74,108,175,203,104,104,2,66,77,183]", + "observed_timestamp": "1794925108227828760", + "scope_name": "messenger", + "service_instance_id": "7f5ed92c-becf-4805-9019-4e83885c2f91", + "service_name": "messenger-worker", + "service_version": "lab7", + "severity_number": "9", + "severity_text": "INFO", + "span_id": "3b0d184558c715c9", + "trace_id": "c039a0aec96327d8540867de25c37ce6" + }, + "values": [ + [ + "1794925108227816343", + "job_ready" + ] + ] + }, + { + "stream": { + "deployment_environment_name": "lab", + "detected_level": "info", + "event": "job_ready", + "flags": "1", + "job_id": "[35,73,135,228,67,58,73,172,135,165,167,39,144,20,86,185]", + "observed_timestamp": "1794925071790645757", + "scope_name": "messenger", + "service_instance_id": "7f5ed92c-becf-4805-9019-4e83885c2f91", + "service_name": "messenger-worker", + "service_version": "lab7", + "severity_number": "9", + "severity_text": "INFO", + "span_id": "e13e2aabebe90311", + "trace_id": "c5eac85c4e184be3156aa03cc0d11b86" + }, + "values": [ + [ + "1794925071790622673", + "job_ready" + ] + ] + } + ], + "stats": { + "summary": { + "bytesProcessedPerSecond": 154981, + "linesProcessedPerSecond": 1298, + "totalBytesProcessed": 10506, + "totalLinesProcessed": 88, + "execTime": 0.067789, + "queueTime": 0.000193, + "subqueries": 0, + "totalEntriesReturned": 5, + "splits": 1, + "shards": 1, + "totalPostFilterLines": 88, + "totalStructuredMetadataBytesProcessed": 9176 + }, + "querier": { + "store": { + "totalChunksRef": 0, + "totalChunksDownloaded": 0, + "chunksDownloadTime": 0, + "queryReferencedStructuredMetadata": false, + "queryUsedV2Engine": false, + "chunk": { + "headChunkBytes": 0, + "headChunkLines": 0, + "decompressedBytes": 0, + "decompressedLines": 0, + "compressedBytes": 0, + "totalDuplicates": 0, + "postFilterLines": 0, + "headChunkStructuredMetadataBytes": 0, + "decompressedStructuredMetadataBytes": 0 + }, + "chunkRefsFetchTime": 0, + "congestionControlLatency": 0, + "pipelineWrapperFilteredLines": 0, + "dataobj": { + "prePredicateDecompressedRows": 0, + "prePredicateDecompressedBytes": 0, + "prePredicateDecompressedStructuredMetadataBytes": 0, + "postPredicateRows": 0, + "postPredicateDecompressedBytes": 0, + "postPredicateStructuredMetadataBytes": 0, + "postFilterRows": 0, + "pagesScanned": 0, + "pagesDownloaded": 0, + "pagesDownloadedBytes": 0, + "pageBatches": 0, + "totalRowsAvailable": 0, + "totalPageDownloadTime": 0 + } + }, + "querierExecTime": 0.024873 + }, + "ingester": { + "totalReached": 1, + "totalChunksMatched": 1, + "totalBatches": 1, + "totalLinesSent": 5, + "store": { + "totalChunksRef": 0, + "totalChunksDownloaded": 0, + "chunksDownloadTime": 0, + "queryReferencedStructuredMetadata": false, + "queryUsedV2Engine": false, + "chunk": { + "headChunkBytes": 10506, + "headChunkLines": 88, + "decompressedBytes": 0, + "decompressedLines": 0, + "compressedBytes": 0, + "totalDuplicates": 0, + "postFilterLines": 88, + "headChunkStructuredMetadataBytes": 9176, + "decompressedStructuredMetadataBytes": 0 + }, + "chunkRefsFetchTime": 185916, + "congestionControlLatency": 0, + "pipelineWrapperFilteredLines": 0, + "dataobj": { + "prePredicateDecompressedRows": 0, + "prePredicateDecompressedBytes": 0, + "prePredicateDecompressedStructuredMetadataBytes": 0, + "postPredicateRows": 0, + "postPredicateDecompressedBytes": 0, + "postPredicateStructuredMetadataBytes": 0, + "postFilterRows": 0, + "pagesScanned": 0, + "pagesDownloaded": 0, + "pagesDownloadedBytes": 0, + "pageBatches": 0, + "totalRowsAvailable": 0, + "totalPageDownloadTime": 0 + } + }, + "recvWaitTime": 0.005071 + }, + "cache": { + "chunk": { + "entriesFound": 0, + "entriesRequested": 0, + "entriesStored": 0, + "bytesReceived": 0, + "bytesSent": 0, + "requests": 0, + "downloadTime": 0, + "queryLengthServed": 0 + }, + "index": { + "entriesFound": 0, + "entriesRequested": 0, + "entriesStored": 0, + "bytesReceived": 0, + "bytesSent": 0, + "requests": 0, + "downloadTime": 0, + "queryLengthServed": 0 + }, + "result": { + "entriesFound": 0, + "entriesRequested": 0, + "entriesStored": 0, + "bytesReceived": 0, + "bytesSent": 0, + "requests": 0, + "downloadTime": 0, + "queryLengthServed": 0 + }, + "statsResult": { + "entriesFound": 0, + "entriesRequested": 0, + "entriesStored": 0, + "bytesReceived": 0, + "bytesSent": 0, + "requests": 0, + "downloadTime": 0, + "queryLengthServed": 0 + }, + "volumeResult": { + "entriesFound": 0, + "entriesRequested": 0, + "entriesStored": 0, + "bytesReceived": 0, + "bytesSent": 0, + "requests": 0, + "downloadTime": 0, + "queryLengthServed": 0 + }, + "seriesResult": { + "entriesFound": 0, + "entriesRequested": 0, + "entriesStored": 0, + "bytesReceived": 0, + "bytesSent": 0, + "requests": 0, + "downloadTime": 0, + "queryLengthServed": 0 + }, + "labelResult": { + "entriesFound": 0, + "entriesRequested": 0, + "entriesStored": 0, + "bytesReceived": 0, + "bytesSent": 0, + "requests": 0, + "downloadTime": 0, + "queryLengthServed": 0 + }, + "instantMetricResult": { + "entriesFound": 0, + "entriesRequested": 0, + "entriesStored": 0, + "bytesReceived": 0, + "bytesSent": 0, + "requests": 0, + "downloadTime": 0, + "queryLengthServed": 0 + } + }, + "index": { + "totalChunks": 0, + "postFilterChunks": 0, + "shardsDuration": 0, + "usedBloomFilters": false, + "totalStreams": 0, + "chunkRefsLookupTime": 0, + "bloomFilterTime": 0 + } + } + } + }, + "metrics": { + "status": "success", + "data": { + "resultType": "vector", + "result": [ + { + "metric": { + "__name__": "up", + "instance": "otel-collector:9444", + "job": "messenger" + }, + "value": [ + 1794925117.589, + "1" + ] + }, + { + "metric": { + "__name__": "up", + "instance": "otel-collector:9445", + "job": "collector" + }, + "value": [ + 1794925117.589, + "1" + ] + }, + { + "metric": { + "__name__": "up", + "instance": "localhost:9090", + "job": "prometheus" + }, + "value": [ + 1794925117.589, + "1" + ] + } + ] + } + }, + "traces": { + "batches": [ + { + "resource": { + "attributes": [ + { + "key": "deployment.environment.name", + "value": { + "stringValue": "lab" + } + }, + { + "key": "service.instance.id", + "value": { + "stringValue": "b1c4f083-bb13-4a42-9714-5b21bf429e3a" + } + }, + { + "key": "service.name", + "value": { + "stringValue": "messenger-api" + } + }, + { + "key": "service.version", + "value": { + "stringValue": "lab7" + } + } + ] + }, + "scopeSpans": [ + { + "scope": { + "name": "messenger" + }, + "spans": [ + { + "traceId": "wDmgrsljJ9hUCGfeJcN85g==", + "spanId": "1htAla3Cp/E=", + "flags": 257, + "name": "POST /api/v1/chats/{chat_id}/attachments", + "kind": "SPAN_KIND_SERVER", + "startTimeUnixNano": "1794925108083949468", + "endTimeUnixNano": "1794925108096444968", + "attributes": [ + { + "key": "http.route", + "value": { + "stringValue": "/api/v1/chats/{chat_id}/attachments" + } + }, + { + "key": "http.request.method", + "value": { + "stringValue": "POST" + } + }, + { + "key": "http.response.status_code", + "value": { + "intValue": "202" + } + } + ], + "status": {} + }, + { + "traceId": "wDmgrsljJ9hUCGfeJcN85g==", + "spanId": "HmWhdZtsy+Y=", + "parentSpanId": "1htAla3Cp/E=", + "flags": 257, + "name": "auth.VerifySession", + "kind": "SPAN_KIND_INTERNAL", + "startTimeUnixNano": "1794925108083960301", + "endTimeUnixNano": "1794925108084193676", + "status": {} + }, + { + "traceId": "wDmgrsljJ9hUCGfeJcN85g==", + "spanId": "SkzPza6mDvs=", + "parentSpanId": "HmWhdZtsy+Y=", + "flags": 257, + "name": "postgresql SELECT", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794925108084025510", + "endTimeUnixNano": "1794925108084183010", + "attributes": [ + { + "key": "db.system.name", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "SELECT" + } + } + ], + "status": {} + }, + { + "traceId": "wDmgrsljJ9hUCGfeJcN85g==", + "spanId": "ZsTPGzzx7ek=", + "parentSpanId": "HmWhdZtsy+Y=", + "flags": 257, + "name": "auth.validateStoredSession", + "kind": "SPAN_KIND_INTERNAL", + "startTimeUnixNano": "1794925108084191010", + "endTimeUnixNano": "1794925108084193343", + "status": {} + }, + { + "traceId": "wDmgrsljJ9hUCGfeJcN85g==", + "spanId": "X11Gd+ad0oc=", + "parentSpanId": "1htAla3Cp/E=", + "flags": 257, + "name": "attachment.upload", + "kind": "SPAN_KIND_INTERNAL", + "startTimeUnixNano": "1794925108084203135", + "endTimeUnixNano": "1794925108096379635", + "status": {} + }, + { + "traceId": "wDmgrsljJ9hUCGfeJcN85g==", + "spanId": "sAebZTTXs0A=", + "parentSpanId": "X11Gd+ad0oc=", + "flags": 257, + "name": "chat.GetForMember", + "kind": "SPAN_KIND_INTERNAL", + "startTimeUnixNano": "1794925108084203676", + "endTimeUnixNano": "1794925108084391010", + "status": {} + }, + { + "traceId": "wDmgrsljJ9hUCGfeJcN85g==", + "spanId": "Kic+mK5fin4=", + "parentSpanId": "sAebZTTXs0A=", + "flags": 257, + "name": "chat.cachedChat", + "kind": "SPAN_KIND_INTERNAL", + "startTimeUnixNano": "1794925108084205218", + "endTimeUnixNano": "1794925108084388801", + "status": {} + }, + { + "traceId": "wDmgrsljJ9hUCGfeJcN85g==", + "spanId": "0EvEVUOxl6c=", + "parentSpanId": "X11Gd+ad0oc=", + "flags": 257, + "name": "postgresql SELECT", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794925108090567635", + "endTimeUnixNano": "1794925108090755926", + "attributes": [ + { + "key": "db.system.name", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "SELECT" + } + } + ], + "status": {} + }, + { + "traceId": "wDmgrsljJ9hUCGfeJcN85g==", + "spanId": "cOTdNQFAf18=", + "parentSpanId": "X11Gd+ad0oc=", + "flags": 257, + "name": "s3.put", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794925108090770676", + "endTimeUnixNano": "1794925108093825176", + "status": {} + }, + { + "traceId": "wDmgrsljJ9hUCGfeJcN85g==", + "spanId": "plw/vW+pLwY=", + "parentSpanId": "X11Gd+ad0oc=", + "flags": 257, + "name": "postgresql BEGIN", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794925108094133426", + "endTimeUnixNano": "1794925108094205676", + "attributes": [ + { + "key": "db.system.name", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "BEGIN" + } + } + ], + "status": {} + }, + { + "traceId": "wDmgrsljJ9hUCGfeJcN85g==", + "spanId": "wX6mjOKefBo=", + "parentSpanId": "X11Gd+ad0oc=", + "flags": 257, + "name": "postgresql INSERT", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794925108094215343", + "endTimeUnixNano": "1794925108094877718", + "attributes": [ + { + "key": "db.system.name", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "INSERT" + } + } + ], + "status": {} + }, + { + "traceId": "wDmgrsljJ9hUCGfeJcN85g==", + "spanId": "3qw8T5QG5+k=", + "parentSpanId": "X11Gd+ad0oc=", + "flags": 257, + "name": "postgresql INSERT", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794925108094891593", + "endTimeUnixNano": "1794925108095329093", + "attributes": [ + { + "key": "db.system.name", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "INSERT" + } + } + ], + "status": {} + }, + { + "traceId": "wDmgrsljJ9hUCGfeJcN85g==", + "spanId": "QrhGNdUBdwA=", + "parentSpanId": "X11Gd+ad0oc=", + "flags": 257, + "name": "postgresql INSERT", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794925108095338218", + "endTimeUnixNano": "1794925108095660176", + "attributes": [ + { + "key": "db.system.name", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "INSERT" + } + } + ], + "status": {} + }, + { + "traceId": "wDmgrsljJ9hUCGfeJcN85g==", + "spanId": "8GWEpOPlYUQ=", + "parentSpanId": "X11Gd+ad0oc=", + "flags": 257, + "name": "postgresql INSERT", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794925108095701718", + "endTimeUnixNano": "1794925108095930468", + "attributes": [ + { + "key": "db.system.name", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "INSERT" + } + } + ], + "status": {} + }, + { + "traceId": "wDmgrsljJ9hUCGfeJcN85g==", + "spanId": "fSVuCdDuK64=", + "parentSpanId": "X11Gd+ad0oc=", + "flags": 257, + "name": "postgresql COMMIT", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794925108095934468", + "endTimeUnixNano": "1794925108096266134", + "attributes": [ + { + "key": "db.system.name", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "COMMIT" + } + } + ], + "status": {} + } + ] + } + ] + }, + { + "resource": { + "attributes": [ + { + "key": "deployment.environment.name", + "value": { + "stringValue": "lab" + } + }, + { + "key": "service.instance.id", + "value": { + "stringValue": "54b193c5-ee76-42ee-9d6b-c0583f3dacae" + } + }, + { + "key": "service.name", + "value": { + "stringValue": "messenger-dispatcher" + } + }, + { + "key": "service.version", + "value": { + "stringValue": "lab7" + } + } + ] + }, + "scopeSpans": [ + { + "scope": { + "name": "messenger" + }, + "spans": [ + { + "traceId": "wDmgrsljJ9hUCGfeJcN85g==", + "spanId": "RXLqNo8EHr8=", + "parentSpanId": "X11Gd+ad0oc=", + "flags": 769, + "name": "attachments publish", + "kind": "SPAN_KIND_PRODUCER", + "startTimeUnixNano": "1794925108173212260", + "endTimeUnixNano": "1794925108181776218", + "status": {} + } + ] + } + ] + }, + { + "resource": { + "attributes": [ + { + "key": "deployment.environment.name", + "value": { + "stringValue": "lab" + } + }, + { + "key": "service.instance.id", + "value": { + "stringValue": "7f5ed92c-becf-4805-9019-4e83885c2f91" + } + }, + { + "key": "service.name", + "value": { + "stringValue": "messenger-worker" + } + }, + { + "key": "service.version", + "value": { + "stringValue": "lab7" + } + } + ] + }, + "scopeSpans": [ + { + "scope": { + "name": "messenger" + }, + "spans": [ + { + "traceId": "wDmgrsljJ9hUCGfeJcN85g==", + "spanId": "UURwViWIFNU=", + "parentSpanId": "RXLqNo8EHr8=", + "flags": 769, + "name": "attachments consume", + "kind": "SPAN_KIND_CONSUMER", + "startTimeUnixNano": "1794925108181112718", + "endTimeUnixNano": "1794925108227860134", + "status": {} + }, + { + "traceId": "wDmgrsljJ9hUCGfeJcN85g==", + "spanId": "Ow0YRVjHFck=", + "parentSpanId": "UURwViWIFNU=", + "flags": 257, + "name": "attachment.process", + "kind": "SPAN_KIND_INTERNAL", + "startTimeUnixNano": "1794925108181196343", + "endTimeUnixNano": "1794925108227850593", + "status": {} + }, + { + "traceId": "wDmgrsljJ9hUCGfeJcN85g==", + "spanId": "Uld5roIcm58=", + "parentSpanId": "Ow0YRVjHFck=", + "flags": 257, + "name": "postgresql BEGIN", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794925108181224676", + "endTimeUnixNano": "1794925108181420176", + "attributes": [ + { + "key": "db.system.name", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "BEGIN" + } + } + ], + "status": {} + }, + { + "traceId": "wDmgrsljJ9hUCGfeJcN85g==", + "spanId": "S60uqf376GI=", + "parentSpanId": "Ow0YRVjHFck=", + "flags": 257, + "name": "postgresql SELECT", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794925108181431676", + "endTimeUnixNano": "1794925108181796384", + "attributes": [ + { + "key": "db.system.name", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "SELECT" + } + } + ], + "status": {} + }, + { + "traceId": "wDmgrsljJ9hUCGfeJcN85g==", + "spanId": "Hi7Bx7P0S0A=", + "parentSpanId": "Ow0YRVjHFck=", + "flags": 257, + "name": "postgresql SELECT", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794925108181807968", + "endTimeUnixNano": "1794925108181984843", + "attributes": [ + { + "key": "db.system.name", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "SELECT" + } + } + ], + "status": {} + }, + { + "traceId": "wDmgrsljJ9hUCGfeJcN85g==", + "spanId": "/iHj++cUmgU=", + "parentSpanId": "Ow0YRVjHFck=", + "flags": 257, + "name": "postgresql UPDATE", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794925108182001093", + "endTimeUnixNano": "1794925108182228384", + "attributes": [ + { + "key": "db.system.name", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "UPDATE" + } + } + ], + "status": {} + }, + { + "traceId": "wDmgrsljJ9hUCGfeJcN85g==", + "spanId": "CHEYEzXIraU=", + "parentSpanId": "Ow0YRVjHFck=", + "flags": 257, + "name": "postgresql UPDATE", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794925108182240760", + "endTimeUnixNano": "1794925108182414802", + "attributes": [ + { + "key": "db.system.name", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "UPDATE" + } + } + ], + "status": {} + }, + { + "traceId": "wDmgrsljJ9hUCGfeJcN85g==", + "spanId": "ZpIVoh9mKPk=", + "parentSpanId": "Ow0YRVjHFck=", + "flags": 257, + "name": "postgresql COMMIT", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794925108182421343", + "endTimeUnixNano": "1794925108182647301", + "attributes": [ + { + "key": "db.system.name", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "COMMIT" + } + } + ], + "status": {} + }, + { + "traceId": "wDmgrsljJ9hUCGfeJcN85g==", + "spanId": "lJA8jfhz3oI=", + "parentSpanId": "Ow0YRVjHFck=", + "flags": 257, + "name": "s3.get", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794925108182666843", + "endTimeUnixNano": "1794925108184783968", + "status": {} + }, + { + "traceId": "wDmgrsljJ9hUCGfeJcN85g==", + "spanId": "mIgXvJgf+hU=", + "parentSpanId": "Ow0YRVjHFck=", + "flags": 257, + "name": "image.transform", + "kind": "SPAN_KIND_INTERNAL", + "startTimeUnixNano": "1794925108188598426", + "endTimeUnixNano": "1794925108209044968", + "status": {} + }, + { + "traceId": "wDmgrsljJ9hUCGfeJcN85g==", + "spanId": "3nWI5h4ER+Q=", + "parentSpanId": "Ow0YRVjHFck=", + "flags": 257, + "name": "s3.put", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794925108209058926", + "endTimeUnixNano": "1794925108212312009", + "status": {} + }, + { + "traceId": "wDmgrsljJ9hUCGfeJcN85g==", + "spanId": "/niDu77HuvI=", + "parentSpanId": "Ow0YRVjHFck=", + "flags": 257, + "name": "s3.put", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794925108212604718", + "endTimeUnixNano": "1794925108215457468", + "status": {} + }, + { + "traceId": "wDmgrsljJ9hUCGfeJcN85g==", + "spanId": "rZWhtHtHZEY=", + "parentSpanId": "Ow0YRVjHFck=", + "flags": 257, + "name": "s3.put", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794925108215547468", + "endTimeUnixNano": "1794925108222717135", + "status": {} + }, + { + "traceId": "wDmgrsljJ9hUCGfeJcN85g==", + "spanId": "wIER+SCeGpU=", + "parentSpanId": "Ow0YRVjHFck=", + "flags": 257, + "name": "s3.put", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794925108222746968", + "endTimeUnixNano": "1794925108224558968", + "status": {} + }, + { + "traceId": "wDmgrsljJ9hUCGfeJcN85g==", + "spanId": "4umzcAMWYBE=", + "parentSpanId": "Ow0YRVjHFck=", + "flags": 257, + "name": "postgresql BEGIN", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794925108224683010", + "endTimeUnixNano": "1794925108224821969", + "attributes": [ + { + "key": "db.system.name", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "BEGIN" + } + } + ], + "status": {} + }, + { + "traceId": "wDmgrsljJ9hUCGfeJcN85g==", + "spanId": "ECYRTrP0e2A=", + "parentSpanId": "Ow0YRVjHFck=", + "flags": 257, + "name": "postgresql SELECT", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794925108224870301", + "endTimeUnixNano": "1794925108225069884", + "attributes": [ + { + "key": "db.system.name", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "SELECT" + } + } + ], + "status": {} + }, + { + "traceId": "wDmgrsljJ9hUCGfeJcN85g==", + "spanId": "ai3k7//XfoE=", + "parentSpanId": "Ow0YRVjHFck=", + "flags": 257, + "name": "postgresql SELECT", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794925108225077385", + "endTimeUnixNano": "1794925108225192551", + "attributes": [ + { + "key": "db.system.name", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "SELECT" + } + } + ], + "status": {} + }, + { + "traceId": "wDmgrsljJ9hUCGfeJcN85g==", + "spanId": "BjPuNHtI4k4=", + "parentSpanId": "Ow0YRVjHFck=", + "flags": 257, + "name": "postgresql INSERT", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794925108225207093", + "endTimeUnixNano": "1794925108226042760", + "attributes": [ + { + "key": "db.system.name", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "INSERT" + } + } + ], + "status": {} + }, + { + "traceId": "wDmgrsljJ9hUCGfeJcN85g==", + "spanId": "LzO9fZzeCnk=", + "parentSpanId": "Ow0YRVjHFck=", + "flags": 257, + "name": "postgresql INSERT", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794925108226049093", + "endTimeUnixNano": "1794925108226173051", + "attributes": [ + { + "key": "db.system.name", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "INSERT" + } + } + ], + "status": {} + }, + { + "traceId": "wDmgrsljJ9hUCGfeJcN85g==", + "spanId": "QESK6PFch3o=", + "parentSpanId": "Ow0YRVjHFck=", + "flags": 257, + "name": "postgresql INSERT", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794925108226199510", + "endTimeUnixNano": "1794925108226324052", + "attributes": [ + { + "key": "db.system.name", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "INSERT" + } + } + ], + "status": {} + }, + { + "traceId": "wDmgrsljJ9hUCGfeJcN85g==", + "spanId": "fJNic5IJqXQ=", + "parentSpanId": "Ow0YRVjHFck=", + "flags": 257, + "name": "postgresql INSERT", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794925108226337135", + "endTimeUnixNano": "1794925108226551343", + "attributes": [ + { + "key": "db.system.name", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "INSERT" + } + } + ], + "status": {} + }, + { + "traceId": "wDmgrsljJ9hUCGfeJcN85g==", + "spanId": "ISyzeL7dphc=", + "parentSpanId": "Ow0YRVjHFck=", + "flags": 257, + "name": "postgresql UPDATE", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794925108226560218", + "endTimeUnixNano": "1794925108226706718", + "attributes": [ + { + "key": "db.system.name", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "UPDATE" + } + } + ], + "status": {} + }, + { + "traceId": "wDmgrsljJ9hUCGfeJcN85g==", + "spanId": "KNBcopXRzvw=", + "parentSpanId": "Ow0YRVjHFck=", + "flags": 257, + "name": "postgresql UPDATE", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794925108226732051", + "endTimeUnixNano": "1794925108226901801", + "attributes": [ + { + "key": "db.system.name", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "UPDATE" + } + } + ], + "status": {} + }, + { + "traceId": "wDmgrsljJ9hUCGfeJcN85g==", + "spanId": "azwXS++4KyA=", + "parentSpanId": "Ow0YRVjHFck=", + "flags": 257, + "name": "postgresql UPDATE", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794925108226908010", + "endTimeUnixNano": "1794925108227345385", + "attributes": [ + { + "key": "db.system.name", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "UPDATE" + } + } + ], + "status": {} + }, + { + "traceId": "wDmgrsljJ9hUCGfeJcN85g==", + "spanId": "Ei41qQouwlw=", + "parentSpanId": "Ow0YRVjHFck=", + "flags": 257, + "name": "postgresql COMMIT", + "kind": "SPAN_KIND_CLIENT", + "startTimeUnixNano": "1794925108227352843", + "endTimeUnixNano": "1794925108227787176", + "attributes": [ + { + "key": "db.system.name", + "value": { + "stringValue": "postgresql" + } + }, + { + "key": "db.operation.name", + "value": { + "stringValue": "COMMIT" + } + } + ], + "status": {} + } + ] + } + ] + } + ] + } + }, + "secret_canaries_absent": true, + "checked_container_logs": true, + "checked_backend_sample": true, + "limitations": "Checks cover captured logs and sampled backend signals, not a mathematical proof for every historical span." +} diff --git a/evidence/verification-summary.md b/evidence/verification-summary.md new file mode 100644 index 0000000..dd5f2bc --- /dev/null +++ b/evidence/verification-summary.md @@ -0,0 +1,36 @@ +# Фактическая итоговая проверка + +2026-11-17T14:23:21.443062+00:00 + +Результаты относятся к synthetic lab7 stand. Source snapshot: `source-manifest.json`; environment/image IDs: `environment.json` и `image-scan.json`. + +| Проверка | Результат | Вывод | +| --- | --- | --- | +| Public HTTPS and external trust | PASS | [public-checks.txt](public-checks.txt) | +| Final HTTPS smoke | PASS | [final-smoke.txt](final-smoke.txt) | +| TLS/identity/ACL matrix | PASS | [security-run.txt](security-run.txt) | +| Argon2/rehash/session expiry | PASS | [auth-run.txt](auth-run.txt) | +| Leaf rotation/revocation | PASS | [leaf-run.txt](leaf-run.txt) | +| Expired allowed-SAN leaf | PASS | [cert-expiry-run.txt](cert-expiry-run.txt) | +| Sealed OpenBao | PASS | [sealed-run.txt](sealed-run.txt) | +| Real telemetry and canaries | PASS | [telemetry-run.txt](telemetry-run.txt) | +| Grafana authentication | PASS | [admin-run.txt](admin-run.txt) | +| Image/ACL negatives | PASS | [images-run.txt](images-run.txt) | +| Staging/restore/retention | PASS | [storage-run.txt](storage-run.txt) | +| Broker volume durability | PASS | [broker-run.txt](broker-run.txt) | +| All stores restart/session persistence | PASS | [persistence-run.txt](persistence-run.txt) | +| Public failover 1 | PASS | [failover-first.txt](failover-first.txt) | +| Public failover 2 | PASS | [failover-second.txt](failover-second.txt) | +| Both physical replicas | PASS | [all-replicas-run.txt](all-replicas-run.txt) | +| Worker compromise/recovery | PASS | [worker-experiment-run.txt](worker-experiment-run.txt) | +| Independent source snapshot | PASS | [clean-clone-run.txt](clean-clone-run.txt) | +| Source vulnerability scan | 0 findings | [govulncheck.txt](govulncheck.txt) | +| Application image scan | 0 findings | [image-scan.json](image-scan.json) | + +## Worker experiment + +- Повтор 1: 5 accepted → 5 ready, lost=0, duplicate_jobs=0, recovery=40.323 s; старые S3/broker/Bao identities отвергнуты. +- Повтор 2: 5 accepted → 5 ready, lost=0, duplicate_jobs=0, recovery=39.502 s; старые S3/broker/Bao identities отвергнуты. +- Повтор 3: 5 accepted → 5 ready, lost=0, duplicate_jobs=0, recovery=43.285 s; старые S3/broker/Bao identities отвергнуты. + +Скриншоты не используются вместо команд. GitHub CI настроен, удалённый workflow не запускался агентом. Диагностические неудачные запуски не включены в PASS-таблицу. diff --git a/evidence/worker-experiment-run.txt b/evidence/worker-experiment-run.txt new file mode 100644 index 0000000..9298483 --- /dev/null +++ b/evidence/worker-experiment-run.txt @@ -0,0 +1,9 @@ +Local lab7 CA, identities and synthetic secrets prepared. No credentials printed. +Worker old cert/token/DB lease revoked; S3 key and leaf replaced; image unchanged. +PASS: worker compromise/recovery repeat 1 +Local lab7 CA, identities and synthetic secrets prepared. No credentials printed. +Worker old cert/token/DB lease revoked; S3 key and leaf replaced; image unchanged. +PASS: worker compromise/recovery repeat 2 +Local lab7 CA, identities and synthetic secrets prepared. No credentials printed. +Worker old cert/token/DB lease revoked; S3 key and leaf replaced; image unchanged. +PASS: worker compromise/recovery repeat 3 diff --git a/evidence/worker-experiment.json b/evidence/worker-experiment.json new file mode 100644 index 0000000..1deff30 --- /dev/null +++ b/evidence/worker-experiment.json @@ -0,0 +1,863 @@ +{ + "hypothesis": "Scoped worker identity limits DB/S3/broker access; revocation and replacement preserve accepted benign jobs.", + "timestamp": "2026-11-17T14:17:54Z", + "repeats": [ + { + "repeat": 1, + "accepted": 5, + "ready": 5, + "lost": 0, + "duplicate_jobs": 0, + "variant_records": 20, + "original_checksums_match": true, + "old_identity_rejected": true, + "recovery_seconds": 40.32301945798099, + "observation_errors": 115, + "upload_latency_ms": { + "p50": 15.432999935001135, + "max": 76.49250002577901 + }, + "restricted_probes": { + "db": { + "ddl_denied": true, + "jobs_read": true, + "messages_read_denied": true, + "mode": "db", + "password_read_denied": true, + "role": "worker", + "session_read_denied": true + }, + "s3": { + "forbidden/security-denied": true, + "mode": "s3", + "originals/security-denied": true, + "results_read": true, + "results_write": true, + "role": "worker" + }, + "amqp": { + "configure_profile": true, + "mode": "amqp", + "publish_profile": true, + "queue_read": true, + "role": "worker" + } + }, + "broad_db_read_and_ddl": true, + "broad_profiles": { + "s3": { + "forbidden/security-denied": true, + "mode": "s3", + "originals/security-denied": true, + "results_read": true, + "results_write": true, + "role": "worker" + }, + "amqp": { + "configure_profile": true, + "mode": "amqp", + "publish_profile": true, + "queue_read": true, + "role": "worker" + } + }, + "old_s3_key_rejected": true, + "old_broker_identity_rejected": true, + "resource_snapshot": [ + { + "BlockIO": "2.88MB / 0B", + "CPUPerc": "0.24%", + "Container": "56afe89db128bb4e04588e532e7aa9917c3f0d0bb82558a5632e2f9a80ecacab", + "ID": "56afe89db128", + "MemPerc": "18.87%", + "MemUsage": "24.16MiB / 128MiB", + "Name": "mtusi2026lab7-s3-tls-1", + "NetIO": "976kB / 1.29MB", + "PIDs": "9" + }, + { + "BlockIO": "6.73MB / 0B", + "CPUPerc": "0.26%", + "Container": "ad91f0eb9f57dad3124106fa2970a160e55206b2b5f7e5c1c42ae5e32136aca9", + "ID": "ad91f0eb9f57", + "MemPerc": "22.01%", + "MemUsage": "28.17MiB / 128MiB", + "Name": "mtusi2026lab7-otel-collector-tls-1", + "NetIO": "6.87MB / 3.43MB", + "PIDs": "9" + }, + { + "BlockIO": "10.8MB / 0B", + "CPUPerc": "1.12%", + "Container": "55c1a619dba179163f075c9a50e72eaf37bb9a87bb4a2b4ae968621c13108b3f", + "ID": "55c1a619dba1", + "MemPerc": "20.77%", + "MemUsage": "106.4MiB / 512MiB", + "Name": "mtusi2026lab7-api-2", + "NetIO": "1.1MB / 1.41MB", + "PIDs": "8" + }, + { + "BlockIO": "1.29MB / 0B", + "CPUPerc": "0.37%", + "Container": "4a795b20fcbdf066c7f656c0e55de0fbb7dcf24dceb073fe0f29496946d20a02", + "ID": "4a795b20fcbd", + "MemPerc": "1.45%", + "MemUsage": "14.86MiB / 1GiB", + "Name": "mtusi2026lab7-worker-1", + "NetIO": "669kB / 810kB", + "PIDs": "8" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "0.95%", + "Container": "ddc8ca7b147898e9d1ac4f9b7219ee2f8905bd2921df64181ffee02ba6cb97c0", + "ID": "ddc8ca7b1478", + "MemPerc": "2.66%", + "MemUsage": "13.62MiB / 512MiB", + "Name": "mtusi2026lab7-dispatcher-1", + "NetIO": "468kB / 841kB", + "PIDs": "8" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.33%", + "Container": "1292b6fbbd2a01262923490e3dd18bd64e253c326aa0834f9db53ae247fb18a9", + "ID": "1292b6fbbd2a", + "MemPerc": "19.01%", + "MemUsage": "97.35MiB / 512MiB", + "Name": "mtusi2026lab7-api-1", + "NetIO": "907kB / 1.25MB", + "PIDs": "8" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.10%", + "Container": "08cf52574facd688fb2497bdd32dcc3cb55c83b9814ea215e0f3a2cd50170b5a", + "ID": "08cf52574fac", + "MemPerc": "15.76%", + "MemUsage": "20.18MiB / 128MiB", + "Name": "mtusi2026lab7-tempo-tls-1", + "NetIO": "1.06MB / 134kB", + "PIDs": "9" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.09%", + "Container": "6ce0529bded56716bb6f3df214263b6cc6bf62bdf0defeba7e1f31d5b798e261", + "ID": "6ce0529bded5", + "MemPerc": "16.53%", + "MemUsage": "21.16MiB / 128MiB", + "Name": "mtusi2026lab7-prometheus-tls-1", + "NetIO": "1.09MB / 97.3kB", + "PIDs": "9" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.11%", + "Container": "9d289263bdf85dd8527aa2f2e5f712e288224d43cf263f933dc5b4b0ea7767c5", + "ID": "9d289263bdf8", + "MemPerc": "16.36%", + "MemUsage": "20.94MiB / 128MiB", + "Name": "mtusi2026lab7-loki-tls-1", + "NetIO": "471kB / 84.8kB", + "PIDs": "9" + }, + { + "BlockIO": "3.47MB / 328kB", + "CPUPerc": "43.64%", + "Container": "be47e533bc90852ec009a15c8f2e9d8bfaed3dfeaa73e80e8a03191e2e3370c1", + "ID": "be47e533bc90", + "MemPerc": "33.07%", + "MemUsage": "169.3MiB / 512MiB", + "Name": "mtusi2026lab7-broker-1", + "NetIO": "1.78MB / 1.74MB", + "PIDs": "96" + }, + { + "BlockIO": "30.9MB / 1.51MB", + "CPUPerc": "0.43%", + "Container": "26d14907a8d04095b62ae9b92073e9743526ad3d94e7fe58b89d2480b0a6baa8", + "ID": "26d14907a8d0", + "MemPerc": "23.71%", + "MemUsage": "91.05MiB / 384MiB", + "Name": "mtusi2026lab7-prometheus-1", + "NetIO": "1.09MB / 97.3kB", + "PIDs": "7" + }, + { + "BlockIO": "46.4MB / 2.37MB", + "CPUPerc": "0.34%", + "Container": "56dc536779db92bf8a1841c4b7e235a028c85a1a3b98f6227c28e2d8d847f6a7", + "ID": "56dc536779db", + "MemPerc": "20.84%", + "MemUsage": "53.35MiB / 256MiB", + "Name": "mtusi2026lab7-openbao-1", + "NetIO": "417kB / 590kB", + "PIDs": "19" + }, + { + "BlockIO": "546MB / 4.08MB", + "CPUPerc": "0.59%", + "Container": "adeb6ec904706c78545c39ed621c7484b74a1b277a02d6f88e989b5100a66262", + "ID": "adeb6ec90470", + "MemPerc": "71.52%", + "MemUsage": "274.6MiB / 384MiB", + "Name": "mtusi2026lab7-grafana-1", + "NetIO": "1.07MB / 39.9kB", + "PIDs": "111" + }, + { + "BlockIO": "3.07MB / 897kB", + "CPUPerc": "0.40%", + "Container": "e4e27b3ae65375241ad7f192aa0f82759e7929702cf9cff8082c3560d0366252", + "ID": "e4e27b3ae653", + "MemPerc": "18.64%", + "MemUsage": "71.57MiB / 384MiB", + "Name": "mtusi2026lab7-loki-1", + "NetIO": "471kB / 84.8kB", + "PIDs": "8" + }, + { + "BlockIO": "51.1MB / 63.7MB", + "CPUPerc": "0.27%", + "Container": "debd0f0fd4f810ed8edc27351a8d9c6926e8c2e036ed622e8eab21d7bdca0cfd", + "ID": "debd0f0fd4f8", + "MemPerc": "99.12%", + "MemUsage": "380.6MiB / 384MiB", + "Name": "mtusi2026lab7-tempo-1", + "NetIO": "1.06MB / 134kB", + "PIDs": "8" + }, + { + "BlockIO": "7.77MB / 0B", + "CPUPerc": "0.32%", + "Container": "29e8dc4c7eec171bcd572b91d52653fb10dd398f0693ec3903a14c02371cb88f", + "ID": "29e8dc4c7eec", + "MemPerc": "28.15%", + "MemUsage": "72.05MiB / 256MiB", + "Name": "mtusi2026lab7-otel-collector-1", + "NetIO": "6.87MB / 3.43MB", + "PIDs": "19" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.30%", + "Container": "bcdeea3ec62786e76b0226e58fd1184fe120855a02732090f0b05771a352a7d6", + "ID": "bcdeea3ec627", + "MemPerc": "17.45%", + "MemUsage": "22.34MiB / 128MiB", + "Name": "mtusi2026lab7-lb-1", + "NetIO": "7.77MB / 8.72MB", + "PIDs": "15" + }, + { + "BlockIO": "16.6MB / 6.99MB", + "CPUPerc": "1.69%", + "Container": "ef7ea1912bec58bd1f62c767c4321e5ef98180a479ef9d0d5bd7288517ddcc3a", + "ID": "ef7ea1912bec", + "MemPerc": "5.37%", + "MemUsage": "54.99MiB / 1GiB", + "Name": "mtusi2026lab7-postgres-1", + "NetIO": "3.68MB / 1.94MB", + "PIDs": "15" + }, + { + "BlockIO": "15MB / 53.2kB", + "CPUPerc": "0.26%", + "Container": "e5835ee95c53224aed3870c331c7af04d11b209944a12a8694524b20b72acca5", + "ID": "e5835ee95c53", + "MemPerc": "12.66%", + "MemUsage": "16.21MiB / 128MiB", + "Name": "mtusi2026lab7-redis-1", + "NetIO": "115kB / 153kB", + "PIDs": "6" + }, + { + "BlockIO": "24.2MB / 295kB", + "CPUPerc": "0.88%", + "Container": "79232cd23eb7cb0d401caffc771bdddf9d26925e069682c6fd89ec434b97b146", + "ID": "79232cd23eb7", + "MemPerc": "46.04%", + "MemUsage": "235.7MiB / 512MiB", + "Name": "mtusi2026lab7-s3-1", + "NetIO": "976kB / 1.29MB", + "PIDs": "20" + } + ] + }, + { + "repeat": 2, + "accepted": 5, + "ready": 5, + "lost": 0, + "duplicate_jobs": 0, + "variant_records": 20, + "original_checksums_match": true, + "old_identity_rejected": true, + "recovery_seconds": 39.502094082999974, + "observation_errors": 105, + "upload_latency_ms": { + "p50": 19.169374834746122, + "max": 25.899999774992466 + }, + "restricted_probes": { + "db": { + "ddl_denied": true, + "jobs_read": true, + "messages_read_denied": true, + "mode": "db", + "password_read_denied": true, + "role": "worker", + "session_read_denied": true + }, + "s3": { + "forbidden/security-denied": true, + "mode": "s3", + "originals/security-denied": true, + "results_read": true, + "results_write": true, + "role": "worker" + }, + "amqp": { + "configure_profile": true, + "mode": "amqp", + "publish_profile": true, + "queue_read": true, + "role": "worker" + } + }, + "broad_db_read_and_ddl": true, + "broad_profiles": { + "s3": { + "forbidden/security-denied": true, + "mode": "s3", + "originals/security-denied": true, + "results_read": true, + "results_write": true, + "role": "worker" + }, + "amqp": { + "configure_profile": true, + "mode": "amqp", + "publish_profile": true, + "queue_read": true, + "role": "worker" + } + }, + "old_s3_key_rejected": true, + "old_broker_identity_rejected": true, + "resource_snapshot": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.23%", + "Container": "d568da6c446d25681300facceef5e97da219f27ef17b60f0ace30bc625f5e493", + "ID": "d568da6c446d", + "MemPerc": "16.05%", + "MemUsage": "20.54MiB / 128MiB", + "Name": "mtusi2026lab7-s3-tls-1", + "NetIO": "963kB / 1.28MB", + "PIDs": "9" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.39%", + "Container": "6eea17d6edbcc586aa9caeee7f9bc7d73a82e8daeb40bbbd1b54066b6a91ae6f", + "ID": "6eea17d6edbc", + "MemPerc": "17.08%", + "MemUsage": "21.86MiB / 128MiB", + "Name": "mtusi2026lab7-otel-collector-tls-1", + "NetIO": "7.68MB / 3.83MB", + "PIDs": "9" + }, + { + "BlockIO": "10.8MB / 0B", + "CPUPerc": "0.53%", + "Container": "55c1a619dba179163f075c9a50e72eaf37bb9a87bb4a2b4ae968621c13108b3f", + "ID": "55c1a619dba1", + "MemPerc": "33.55%", + "MemUsage": "171.8MiB / 512MiB", + "Name": "mtusi2026lab7-api-2", + "NetIO": "2.17MB / 2.78MB", + "PIDs": "8" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.56%", + "Container": "4a795b20fcbdf066c7f656c0e55de0fbb7dcf24dceb073fe0f29496946d20a02", + "ID": "4a795b20fcbd", + "MemPerc": "1.16%", + "MemUsage": "11.89MiB / 1GiB", + "Name": "mtusi2026lab7-worker-1", + "NetIO": "664kB / 802kB", + "PIDs": "8" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "1.04%", + "Container": "ddc8ca7b147898e9d1ac4f9b7219ee2f8905bd2921df64181ffee02ba6cb97c0", + "ID": "ddc8ca7b1478", + "MemPerc": "2.72%", + "MemUsage": "13.91MiB / 512MiB", + "Name": "mtusi2026lab7-dispatcher-1", + "NetIO": "839kB / 1.51MB", + "PIDs": "8" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.28%", + "Container": "1292b6fbbd2a01262923490e3dd18bd64e253c326aa0834f9db53ae247fb18a9", + "ID": "1292b6fbbd2a", + "MemPerc": "31.25%", + "MemUsage": "160MiB / 512MiB", + "Name": "mtusi2026lab7-api-1", + "NetIO": "1.74MB / 2.4MB", + "PIDs": "8" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.20%", + "Container": "08cf52574facd688fb2497bdd32dcc3cb55c83b9814ea215e0f3a2cd50170b5a", + "ID": "08cf52574fac", + "MemPerc": "15.76%", + "MemUsage": "20.18MiB / 128MiB", + "Name": "mtusi2026lab7-tempo-tls-1", + "NetIO": "1.18MB / 148kB", + "PIDs": "9" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.11%", + "Container": "6ce0529bded56716bb6f3df214263b6cc6bf62bdf0defeba7e1f31d5b798e261", + "ID": "6ce0529bded5", + "MemPerc": "16.53%", + "MemUsage": "21.16MiB / 128MiB", + "Name": "mtusi2026lab7-prometheus-tls-1", + "NetIO": "1.22MB / 114kB", + "PIDs": "9" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.20%", + "Container": "9d289263bdf85dd8527aa2f2e5f712e288224d43cf263f933dc5b4b0ea7767c5", + "ID": "9d289263bdf8", + "MemPerc": "16.36%", + "MemUsage": "20.94MiB / 128MiB", + "Name": "mtusi2026lab7-loki-tls-1", + "NetIO": "529kB / 94.1kB", + "PIDs": "9" + }, + { + "BlockIO": "3.53MB / 446kB", + "CPUPerc": "0.65%", + "Container": "be47e533bc90852ec009a15c8f2e9d8bfaed3dfeaa73e80e8a03191e2e3370c1", + "ID": "be47e533bc90", + "MemPerc": "19.56%", + "MemUsage": "100.2MiB / 512MiB", + "Name": "mtusi2026lab7-broker-1", + "NetIO": "2.03MB / 1.98MB", + "PIDs": "50" + }, + { + "BlockIO": "31.4MB / 1.76MB", + "CPUPerc": "3.79%", + "Container": "26d14907a8d04095b62ae9b92073e9743526ad3d94e7fe58b89d2480b0a6baa8", + "ID": "26d14907a8d0", + "MemPerc": "23.97%", + "MemUsage": "92.03MiB / 384MiB", + "Name": "mtusi2026lab7-prometheus-1", + "NetIO": "1.22MB / 114kB", + "PIDs": "7" + }, + { + "BlockIO": "46.8MB / 2.94MB", + "CPUPerc": "0.43%", + "Container": "56dc536779db92bf8a1841c4b7e235a028c85a1a3b98f6227c28e2d8d847f6a7", + "ID": "56dc536779db", + "MemPerc": "21.02%", + "MemUsage": "53.8MiB / 256MiB", + "Name": "mtusi2026lab7-openbao-1", + "NetIO": "583kB / 806kB", + "PIDs": "19" + }, + { + "BlockIO": "546MB / 4.1MB", + "CPUPerc": "0.74%", + "Container": "adeb6ec904706c78545c39ed621c7484b74a1b277a02d6f88e989b5100a66262", + "ID": "adeb6ec90470", + "MemPerc": "71.52%", + "MemUsage": "274.6MiB / 384MiB", + "Name": "mtusi2026lab7-grafana-1", + "NetIO": "1.07MB / 39.9kB", + "PIDs": "111" + }, + { + "BlockIO": "3.07MB / 1.04MB", + "CPUPerc": "0.72%", + "Container": "e4e27b3ae65375241ad7f192aa0f82759e7929702cf9cff8082c3560d0366252", + "ID": "e4e27b3ae653", + "MemPerc": "18.65%", + "MemUsage": "71.61MiB / 384MiB", + "Name": "mtusi2026lab7-loki-1", + "NetIO": "529kB / 94.1kB", + "PIDs": "8" + }, + { + "BlockIO": "51.4MB / 64.2MB", + "CPUPerc": "1.19%", + "Container": "debd0f0fd4f810ed8edc27351a8d9c6926e8c2e036ed622e8eab21d7bdca0cfd", + "ID": "debd0f0fd4f8", + "MemPerc": "80.38%", + "MemUsage": "308.7MiB / 384MiB", + "Name": "mtusi2026lab7-tempo-1", + "NetIO": "1.18MB / 148kB", + "PIDs": "8" + }, + { + "BlockIO": "7.91MB / 0B", + "CPUPerc": "0.71%", + "Container": "29e8dc4c7eec171bcd572b91d52653fb10dd398f0693ec3903a14c02371cb88f", + "ID": "29e8dc4c7eec", + "MemPerc": "28.51%", + "MemUsage": "72.99MiB / 256MiB", + "Name": "mtusi2026lab7-otel-collector-1", + "NetIO": "7.68MB / 3.83MB", + "PIDs": "19" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.38%", + "Container": "bcdeea3ec62786e76b0226e58fd1184fe120855a02732090f0b05771a352a7d6", + "ID": "bcdeea3ec627", + "MemPerc": "17.77%", + "MemUsage": "22.75MiB / 128MiB", + "Name": "mtusi2026lab7-lb-1", + "NetIO": "9.82MB / 10.9MB", + "PIDs": "15" + }, + { + "BlockIO": "16.6MB / 7.8MB", + "CPUPerc": "2.15%", + "Container": "ef7ea1912bec58bd1f62c767c4321e5ef98180a479ef9d0d5bd7288517ddcc3a", + "ID": "ef7ea1912bec", + "MemPerc": "5.41%", + "MemUsage": "55.41MiB / 1GiB", + "Name": "mtusi2026lab7-postgres-1", + "NetIO": "4.1MB / 2.19MB", + "PIDs": "15" + }, + { + "BlockIO": "15MB / 77.8kB", + "CPUPerc": "0.35%", + "Container": "e5835ee95c53224aed3870c331c7af04d11b209944a12a8694524b20b72acca5", + "ID": "e5835ee95c53", + "MemPerc": "12.86%", + "MemUsage": "16.46MiB / 128MiB", + "Name": "mtusi2026lab7-redis-1", + "NetIO": "128kB / 164kB", + "PIDs": "6" + }, + { + "BlockIO": "1.7MB / 184kB", + "CPUPerc": "0.83%", + "Container": "79232cd23eb7cb0d401caffc771bdddf9d26925e069682c6fd89ec434b97b146", + "ID": "79232cd23eb7", + "MemPerc": "34.62%", + "MemUsage": "177.2MiB / 512MiB", + "Name": "mtusi2026lab7-s3-1", + "NetIO": "963kB / 1.28MB", + "PIDs": "9" + } + ] + }, + { + "repeat": 3, + "accepted": 5, + "ready": 5, + "lost": 0, + "duplicate_jobs": 0, + "variant_records": 20, + "original_checksums_match": true, + "old_identity_rejected": true, + "recovery_seconds": 43.28494662465528, + "observation_errors": 125, + "upload_latency_ms": { + "p50": 18.283792305737734, + "max": 30.815250240266323 + }, + "restricted_probes": { + "db": { + "ddl_denied": true, + "jobs_read": true, + "messages_read_denied": true, + "mode": "db", + "password_read_denied": true, + "role": "worker", + "session_read_denied": true + }, + "s3": { + "forbidden/security-denied": true, + "mode": "s3", + "originals/security-denied": true, + "results_read": true, + "results_write": true, + "role": "worker" + }, + "amqp": { + "configure_profile": true, + "mode": "amqp", + "publish_profile": true, + "queue_read": true, + "role": "worker" + } + }, + "broad_db_read_and_ddl": true, + "broad_profiles": { + "s3": { + "forbidden/security-denied": true, + "mode": "s3", + "originals/security-denied": true, + "results_read": true, + "results_write": true, + "role": "worker" + }, + "amqp": { + "configure_profile": true, + "mode": "amqp", + "publish_profile": true, + "queue_read": true, + "role": "worker" + } + }, + "old_s3_key_rejected": true, + "old_broker_identity_rejected": true, + "resource_snapshot": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.22%", + "Container": "e56f2477dfc58b1b705c5151a4a48dd371115de300a89a9db127900ba3334f3c", + "ID": "e56f2477dfc5", + "MemPerc": "15.81%", + "MemUsage": "20.23MiB / 128MiB", + "Name": "mtusi2026lab7-s3-tls-1", + "NetIO": "1.01MB / 1.3MB", + "PIDs": "9" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.28%", + "Container": "fd2bbddf84d99c077f5f525076c80d21053765c25405570f4c0057c1f1c9acf7", + "ID": "fd2bbddf84d9", + "MemPerc": "16.52%", + "MemUsage": "21.15MiB / 128MiB", + "Name": "mtusi2026lab7-otel-collector-tls-1", + "NetIO": "8.51MB / 4.26MB", + "PIDs": "9" + }, + { + "BlockIO": "10.8MB / 0B", + "CPUPerc": "1.20%", + "Container": "55c1a619dba179163f075c9a50e72eaf37bb9a87bb4a2b4ae968621c13108b3f", + "ID": "55c1a619dba1", + "MemPerc": "36.22%", + "MemUsage": "185.4MiB / 512MiB", + "Name": "mtusi2026lab7-api-2", + "NetIO": "3.24MB / 4.16MB", + "PIDs": "8" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.08%", + "Container": "4a795b20fcbdf066c7f656c0e55de0fbb7dcf24dceb073fe0f29496946d20a02", + "ID": "4a795b20fcbd", + "MemPerc": "1.09%", + "MemUsage": "11.18MiB / 1GiB", + "Name": "mtusi2026lab7-worker-1", + "NetIO": "668kB / 811kB", + "PIDs": "8" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "0.96%", + "Container": "ddc8ca7b147898e9d1ac4f9b7219ee2f8905bd2921df64181ffee02ba6cb97c0", + "ID": "ddc8ca7b1478", + "MemPerc": "2.78%", + "MemUsage": "14.25MiB / 512MiB", + "Name": "mtusi2026lab7-dispatcher-1", + "NetIO": "1.23MB / 2.22MB", + "PIDs": "8" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.40%", + "Container": "1292b6fbbd2a01262923490e3dd18bd64e253c326aa0834f9db53ae247fb18a9", + "ID": "1292b6fbbd2a", + "MemPerc": "34.93%", + "MemUsage": "178.8MiB / 512MiB", + "Name": "mtusi2026lab7-api-1", + "NetIO": "2.59MB / 3.59MB", + "PIDs": "8" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.12%", + "Container": "08cf52574facd688fb2497bdd32dcc3cb55c83b9814ea215e0f3a2cd50170b5a", + "ID": "08cf52574fac", + "MemPerc": "15.76%", + "MemUsage": "20.18MiB / 128MiB", + "Name": "mtusi2026lab7-tempo-tls-1", + "NetIO": "1.31MB / 164kB", + "PIDs": "9" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.09%", + "Container": "6ce0529bded56716bb6f3df214263b6cc6bf62bdf0defeba7e1f31d5b798e261", + "ID": "6ce0529bded5", + "MemPerc": "16.53%", + "MemUsage": "21.16MiB / 128MiB", + "Name": "mtusi2026lab7-prometheus-tls-1", + "NetIO": "1.36MB / 131kB", + "PIDs": "9" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.12%", + "Container": "9d289263bdf85dd8527aa2f2e5f712e288224d43cf263f933dc5b4b0ea7767c5", + "ID": "9d289263bdf8", + "MemPerc": "16.36%", + "MemUsage": "20.94MiB / 128MiB", + "Name": "mtusi2026lab7-loki-tls-1", + "NetIO": "589kB / 104kB", + "PIDs": "9" + }, + { + "BlockIO": "3.53MB / 594kB", + "CPUPerc": "49.70%", + "Container": "be47e533bc90852ec009a15c8f2e9d8bfaed3dfeaa73e80e8a03191e2e3370c1", + "ID": "be47e533bc90", + "MemPerc": "33.90%", + "MemUsage": "173.6MiB / 512MiB", + "Name": "mtusi2026lab7-broker-1", + "NetIO": "2.29MB / 2.23MB", + "PIDs": "96" + }, + { + "BlockIO": "31.5MB / 2.01MB", + "CPUPerc": "0.29%", + "Container": "26d14907a8d04095b62ae9b92073e9743526ad3d94e7fe58b89d2480b0a6baa8", + "ID": "26d14907a8d0", + "MemPerc": "24.05%", + "MemUsage": "92.35MiB / 384MiB", + "Name": "mtusi2026lab7-prometheus-1", + "NetIO": "1.36MB / 131kB", + "PIDs": "7" + }, + { + "BlockIO": "47.1MB / 3.51MB", + "CPUPerc": "0.35%", + "Container": "56dc536779db92bf8a1841c4b7e235a028c85a1a3b98f6227c28e2d8d847f6a7", + "ID": "56dc536779db", + "MemPerc": "21.39%", + "MemUsage": "54.77MiB / 256MiB", + "Name": "mtusi2026lab7-openbao-1", + "NetIO": "753kB / 1.03MB", + "PIDs": "19" + }, + { + "BlockIO": "546MB / 4.15MB", + "CPUPerc": "0.66%", + "Container": "adeb6ec904706c78545c39ed621c7484b74a1b277a02d6f88e989b5100a66262", + "ID": "adeb6ec90470", + "MemPerc": "71.57%", + "MemUsage": "274.8MiB / 384MiB", + "Name": "mtusi2026lab7-grafana-1", + "NetIO": "1.07MB / 39.9kB", + "PIDs": "111" + }, + { + "BlockIO": "3.07MB / 1.24MB", + "CPUPerc": "0.53%", + "Container": "e4e27b3ae65375241ad7f192aa0f82759e7929702cf9cff8082c3560d0366252", + "ID": "e4e27b3ae653", + "MemPerc": "18.71%", + "MemUsage": "71.84MiB / 384MiB", + "Name": "mtusi2026lab7-loki-1", + "NetIO": "589kB / 104kB", + "PIDs": "8" + }, + { + "BlockIO": "51.4MB / 64.7MB", + "CPUPerc": "0.33%", + "Container": "debd0f0fd4f810ed8edc27351a8d9c6926e8c2e036ed622e8eab21d7bdca0cfd", + "ID": "debd0f0fd4f8", + "MemPerc": "80.41%", + "MemUsage": "308.8MiB / 384MiB", + "Name": "mtusi2026lab7-tempo-1", + "NetIO": "1.31MB / 164kB", + "PIDs": "8" + }, + { + "BlockIO": "7.91MB / 0B", + "CPUPerc": "0.66%", + "Container": "29e8dc4c7eec171bcd572b91d52653fb10dd398f0693ec3903a14c02371cb88f", + "ID": "29e8dc4c7eec", + "MemPerc": "29.66%", + "MemUsage": "75.93MiB / 256MiB", + "Name": "mtusi2026lab7-otel-collector-1", + "NetIO": "8.51MB / 4.26MB", + "PIDs": "19" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.39%", + "Container": "bcdeea3ec62786e76b0226e58fd1184fe120855a02732090f0b05771a352a7d6", + "ID": "bcdeea3ec627", + "MemPerc": "17.62%", + "MemUsage": "22.55MiB / 128MiB", + "Name": "mtusi2026lab7-lb-1", + "NetIO": "11.9MB / 13.2MB", + "PIDs": "15" + }, + { + "BlockIO": "16.6MB / 8.81MB", + "CPUPerc": "1.49%", + "Container": "ef7ea1912bec58bd1f62c767c4321e5ef98180a479ef9d0d5bd7288517ddcc3a", + "ID": "ef7ea1912bec", + "MemPerc": "5.47%", + "MemUsage": "55.98MiB / 1GiB", + "Name": "mtusi2026lab7-postgres-1", + "NetIO": "4.54MB / 2.46MB", + "PIDs": "15" + }, + { + "BlockIO": "15.1MB / 94.2kB", + "CPUPerc": "0.27%", + "Container": "e5835ee95c53224aed3870c331c7af04d11b209944a12a8694524b20b72acca5", + "ID": "e5835ee95c53", + "MemPerc": "12.72%", + "MemUsage": "16.28MiB / 128MiB", + "Name": "mtusi2026lab7-redis-1", + "NetIO": "138kB / 174kB", + "PIDs": "6" + }, + { + "BlockIO": "1.2MB / 213kB", + "CPUPerc": "1.10%", + "Container": "79232cd23eb7cb0d401caffc771bdddf9d26925e069682c6fd89ec434b97b146", + "ID": "79232cd23eb7", + "MemPerc": "33.28%", + "MemUsage": "170.4MiB / 512MiB", + "Name": "mtusi2026lab7-s3-1", + "NetIO": "1.01MB / 1.3MB", + "PIDs": "8" + } + ] + } + ], + "limitations": [ + "Authorized worker can damage its permitted job metadata and object prefixes.", + "This experiment does not claim integrity after malicious modification of those authorized resources.", + "Single Docker host; no host/DB/LB high availability." + ] +} diff --git a/infra/README.md b/infra/README.md index 5a39e86..1bfdce5 100644 --- a/infra/README.md +++ b/infra/README.md @@ -5,3 +5,7 @@ В REPORT.md сопоставьте роль → service name → версия/digest → health/readiness → volume → сеть/порт → способ передачи credentials. Приведите bootstrap (schema, bucket, broker topology, а с lab6 provisioning) и безопасное повторение этих команд. Приложение строится из исходников. Не используйте заранее существующий образ преподавательского сервера вместо своей реализации. Restart hook перезапускает все соответствующие stores с volumes; failover hook останавливает только заданную API-реплику. + +## Реализация lab7 + +Топология находится в корневом `compose.yaml` (JSON является допустимым YAML). Начальная команда — `make up`; полная инструкция и версии — в [REPORT.md](../REPORT.md). TLS boundaries генерируются `infra/generate.py` и хранятся как открытые Envoy configurations. Runtime credentials, private keys и Grafana secure provisioning создаются только в игнорируемом `secrets/`. diff --git a/infra/envoy/loki.json b/infra/envoy/loki.json new file mode 100644 index 0000000..2b98c6e --- /dev/null +++ b/infra/envoy/loki.json @@ -0,0 +1,216 @@ +{ + "static_resources": { + "listeners": [ + { + "name": "loki", + "address": { + "socket_address": { + "address": "0.0.0.0", + "port_value": 9443 + } + }, + "filter_chains": [ + { + "transport_socket": { + "name": "envoy.transport_sockets.tls", + "typed_config": { + "@type": "type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.DownstreamTlsContext", + "require_client_certificate": true, + "common_tls_context": { + "tls_params": { + "tls_minimum_protocol_version": "TLSv1_2" + }, + "tls_certificates": [ + { + "certificate_chain": { + "filename": "/run/identity/tls.crt" + }, + "private_key": { + "filename": "/run/identity/tls.key" + } + } + ], + "validation_context": { + "trusted_ca": { + "filename": "/run/identity/ca.crt" + }, + "crl": { + "filename": "/run/identity/ca.crl" + }, + "match_typed_subject_alt_names": [ + { + "san_type": "DNS", + "matcher": { + "exact": "otel-collector" + } + }, + { + "san_type": "DNS", + "matcher": { + "exact": "grafana" + } + } + ] + } + } + } + }, + "filters": [ + { + "name": "envoy.filters.network.http_connection_manager", + "typed_config": { + "@type": "type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager", + "stat_prefix": "loki", + "codec_type": "AUTO", + "route_config": { + "name": "routes", + "virtual_hosts": [ + { + "name": "local", + "domains": [ + "*" + ], + "routes": [ + { + "match": { + "prefix": "/" + }, + "route": { + "cluster": "loki", + "timeout": "125s" + } + } + ] + } + ] + }, + "http_filters": [ + { + "name": "envoy.filters.http.rbac", + "typed_config": { + "@type": "type.googleapis.com/envoy.extensions.filters.http.rbac.v3.RBAC", + "rules": { + "action": "ALLOW", + "policies": { + "otel-collector": { + "principals": [ + { + "authenticated": { + "principal_name": { + "exact": "otel-collector" + } + } + } + ], + "permissions": [ + { + "and_rules": { + "rules": [ + { + "header": { + "name": ":method", + "string_match": { + "safe_regex": { + "regex": "^(POST)$" + } + } + } + }, + { + "url_path": { + "path": { + "safe_regex": { + "regex": "^/otlp/v1/logs$" + } + } + } + } + ] + } + } + ] + }, + "grafana": { + "principals": [ + { + "authenticated": { + "principal_name": { + "exact": "grafana" + } + } + } + ], + "permissions": [ + { + "and_rules": { + "rules": [ + { + "header": { + "name": ":method", + "string_match": { + "safe_regex": { + "regex": "^(GET)$" + } + } + } + }, + { + "url_path": { + "path": { + "safe_regex": { + "regex": "^/(loki/api/v1/.*|ready)$" + } + } + } + } + ] + } + } + ] + } + } + } + } + }, + { + "name": "envoy.filters.http.router", + "typed_config": { + "@type": "type.googleapis.com/envoy.extensions.filters.http.router.v3.Router" + } + } + ] + } + } + ] + } + ] + } + ], + "clusters": [ + { + "name": "loki", + "connect_timeout": "1s", + "type": "STATIC", + "load_assignment": { + "cluster_name": "loki", + "endpoints": [ + { + "lb_endpoints": [ + { + "endpoint": { + "address": { + "socket_address": { + "address": "127.0.0.1", + "port_value": 3100 + } + } + } + } + ] + } + ] + } + } + ] + } +} diff --git a/infra/envoy/otel-collector.json b/infra/envoy/otel-collector.json new file mode 100644 index 0000000..c317f8b --- /dev/null +++ b/infra/envoy/otel-collector.json @@ -0,0 +1,365 @@ +{ + "static_resources": { + "listeners": [ + { + "name": "otlp", + "address": { + "socket_address": { + "address": "0.0.0.0", + "port_value": 9443 + } + }, + "filter_chains": [ + { + "transport_socket": { + "name": "envoy.transport_sockets.tls", + "typed_config": { + "@type": "type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.DownstreamTlsContext", + "require_client_certificate": true, + "common_tls_context": { + "tls_params": { + "tls_minimum_protocol_version": "TLSv1_2" + }, + "tls_certificates": [ + { + "certificate_chain": { + "filename": "/run/identity/tls.crt" + }, + "private_key": { + "filename": "/run/identity/tls.key" + } + } + ], + "validation_context": { + "trusted_ca": { + "filename": "/run/identity/ca.crt" + }, + "crl": { + "filename": "/run/identity/ca.crl" + }, + "match_typed_subject_alt_names": [ + { + "san_type": "DNS", + "matcher": { + "exact": "api" + } + }, + { + "san_type": "DNS", + "matcher": { + "exact": "worker" + } + }, + { + "san_type": "DNS", + "matcher": { + "exact": "dispatcher" + } + } + ] + } + } + } + }, + "filters": [ + { + "name": "envoy.filters.network.http_connection_manager", + "typed_config": { + "@type": "type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager", + "stat_prefix": "otlp", + "codec_type": "AUTO", + "route_config": { + "name": "routes", + "virtual_hosts": [ + { + "name": "local", + "domains": [ + "*" + ], + "routes": [ + { + "match": { + "prefix": "/" + }, + "route": { + "cluster": "otlp", + "timeout": "125s" + } + } + ] + } + ] + }, + "http_filters": [ + { + "name": "envoy.filters.http.router", + "typed_config": { + "@type": "type.googleapis.com/envoy.extensions.filters.http.router.v3.Router" + } + } + ] + } + } + ] + } + ] + }, + { + "name": "metrics", + "address": { + "socket_address": { + "address": "0.0.0.0", + "port_value": 9444 + } + }, + "filter_chains": [ + { + "transport_socket": { + "name": "envoy.transport_sockets.tls", + "typed_config": { + "@type": "type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.DownstreamTlsContext", + "require_client_certificate": true, + "common_tls_context": { + "tls_params": { + "tls_minimum_protocol_version": "TLSv1_2" + }, + "tls_certificates": [ + { + "certificate_chain": { + "filename": "/run/identity/tls.crt" + }, + "private_key": { + "filename": "/run/identity/tls.key" + } + } + ], + "validation_context": { + "trusted_ca": { + "filename": "/run/identity/ca.crt" + }, + "crl": { + "filename": "/run/identity/ca.crl" + }, + "match_typed_subject_alt_names": [ + { + "san_type": "DNS", + "matcher": { + "exact": "prometheus" + } + } + ] + } + } + } + }, + "filters": [ + { + "name": "envoy.filters.network.http_connection_manager", + "typed_config": { + "@type": "type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager", + "stat_prefix": "metrics", + "codec_type": "AUTO", + "route_config": { + "name": "routes", + "virtual_hosts": [ + { + "name": "local", + "domains": [ + "*" + ], + "routes": [ + { + "match": { + "prefix": "/" + }, + "route": { + "cluster": "metrics", + "timeout": "125s" + } + } + ] + } + ] + }, + "http_filters": [ + { + "name": "envoy.filters.http.router", + "typed_config": { + "@type": "type.googleapis.com/envoy.extensions.filters.http.router.v3.Router" + } + } + ] + } + } + ] + } + ] + }, + { + "name": "collector-metrics", + "address": { + "socket_address": { + "address": "0.0.0.0", + "port_value": 9445 + } + }, + "filter_chains": [ + { + "transport_socket": { + "name": "envoy.transport_sockets.tls", + "typed_config": { + "@type": "type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.DownstreamTlsContext", + "require_client_certificate": true, + "common_tls_context": { + "tls_params": { + "tls_minimum_protocol_version": "TLSv1_2" + }, + "tls_certificates": [ + { + "certificate_chain": { + "filename": "/run/identity/tls.crt" + }, + "private_key": { + "filename": "/run/identity/tls.key" + } + } + ], + "validation_context": { + "trusted_ca": { + "filename": "/run/identity/ca.crt" + }, + "crl": { + "filename": "/run/identity/ca.crl" + }, + "match_typed_subject_alt_names": [ + { + "san_type": "DNS", + "matcher": { + "exact": "prometheus" + } + } + ] + } + } + } + }, + "filters": [ + { + "name": "envoy.filters.network.http_connection_manager", + "typed_config": { + "@type": "type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager", + "stat_prefix": "collector-metrics", + "codec_type": "AUTO", + "route_config": { + "name": "routes", + "virtual_hosts": [ + { + "name": "local", + "domains": [ + "*" + ], + "routes": [ + { + "match": { + "prefix": "/" + }, + "route": { + "cluster": "collector-metrics", + "timeout": "125s" + } + } + ] + } + ] + }, + "http_filters": [ + { + "name": "envoy.filters.http.router", + "typed_config": { + "@type": "type.googleapis.com/envoy.extensions.filters.http.router.v3.Router" + } + } + ] + } + } + ] + } + ] + } + ], + "clusters": [ + { + "name": "otlp", + "connect_timeout": "1s", + "type": "STATIC", + "load_assignment": { + "cluster_name": "otlp", + "endpoints": [ + { + "lb_endpoints": [ + { + "endpoint": { + "address": { + "socket_address": { + "address": "127.0.0.1", + "port_value": 4318 + } + } + } + } + ] + } + ] + } + }, + { + "name": "metrics", + "connect_timeout": "1s", + "type": "STATIC", + "load_assignment": { + "cluster_name": "metrics", + "endpoints": [ + { + "lb_endpoints": [ + { + "endpoint": { + "address": { + "socket_address": { + "address": "127.0.0.1", + "port_value": 8889 + } + } + } + } + ] + } + ] + } + }, + { + "name": "collector-metrics", + "connect_timeout": "1s", + "type": "STATIC", + "load_assignment": { + "cluster_name": "collector-metrics", + "endpoints": [ + { + "lb_endpoints": [ + { + "endpoint": { + "address": { + "socket_address": { + "address": "127.0.0.1", + "port_value": 8888 + } + } + } + } + ] + } + ] + } + } + ] + } +} diff --git a/infra/envoy/prometheus.json b/infra/envoy/prometheus.json new file mode 100644 index 0000000..d2a91b6 --- /dev/null +++ b/infra/envoy/prometheus.json @@ -0,0 +1,172 @@ +{ + "static_resources": { + "listeners": [ + { + "name": "query", + "address": { + "socket_address": { + "address": "0.0.0.0", + "port_value": 9443 + } + }, + "filter_chains": [ + { + "transport_socket": { + "name": "envoy.transport_sockets.tls", + "typed_config": { + "@type": "type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.DownstreamTlsContext", + "require_client_certificate": true, + "common_tls_context": { + "tls_params": { + "tls_minimum_protocol_version": "TLSv1_2" + }, + "tls_certificates": [ + { + "certificate_chain": { + "filename": "/run/identity/tls.crt" + }, + "private_key": { + "filename": "/run/identity/tls.key" + } + } + ], + "validation_context": { + "trusted_ca": { + "filename": "/run/identity/ca.crt" + }, + "crl": { + "filename": "/run/identity/ca.crl" + }, + "match_typed_subject_alt_names": [ + { + "san_type": "DNS", + "matcher": { + "exact": "grafana" + } + } + ] + } + } + } + }, + "filters": [ + { + "name": "envoy.filters.network.http_connection_manager", + "typed_config": { + "@type": "type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager", + "stat_prefix": "query", + "codec_type": "AUTO", + "route_config": { + "name": "routes", + "virtual_hosts": [ + { + "name": "local", + "domains": [ + "*" + ], + "routes": [ + { + "match": { + "prefix": "/" + }, + "route": { + "cluster": "query", + "timeout": "125s" + } + } + ] + } + ] + }, + "http_filters": [ + { + "name": "envoy.filters.http.rbac", + "typed_config": { + "@type": "type.googleapis.com/envoy.extensions.filters.http.rbac.v3.RBAC", + "rules": { + "action": "ALLOW", + "policies": { + "grafana": { + "principals": [ + { + "authenticated": { + "principal_name": { + "exact": "grafana" + } + } + } + ], + "permissions": [ + { + "and_rules": { + "rules": [ + { + "header": { + "name": ":method", + "string_match": { + "safe_regex": { + "regex": "^(GET|POST)$" + } + } + } + }, + { + "url_path": { + "path": { + "safe_regex": { + "regex": "^/(api/v1/(query|query_range|labels|label/[^/]+/values|series|metadata|status/buildinfo)|-/ready).*" + } + } + } + } + ] + } + } + ] + } + } + } + } + }, + { + "name": "envoy.filters.http.router", + "typed_config": { + "@type": "type.googleapis.com/envoy.extensions.filters.http.router.v3.Router" + } + } + ] + } + } + ] + } + ] + } + ], + "clusters": [ + { + "name": "query", + "connect_timeout": "1s", + "type": "STATIC", + "load_assignment": { + "cluster_name": "query", + "endpoints": [ + { + "lb_endpoints": [ + { + "endpoint": { + "address": { + "socket_address": { + "address": "127.0.0.1", + "port_value": 9090 + } + } + } + } + ] + } + ] + } + } + ] + } +} diff --git a/infra/envoy/s3.json b/infra/envoy/s3.json new file mode 100644 index 0000000..b90cf5b --- /dev/null +++ b/infra/envoy/s3.json @@ -0,0 +1,135 @@ +{ + "static_resources": { + "listeners": [ + { + "name": "s3", + "address": { + "socket_address": { + "address": "0.0.0.0", + "port_value": 9443 + } + }, + "filter_chains": [ + { + "transport_socket": { + "name": "envoy.transport_sockets.tls", + "typed_config": { + "@type": "type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.DownstreamTlsContext", + "require_client_certificate": true, + "common_tls_context": { + "tls_params": { + "tls_minimum_protocol_version": "TLSv1_2" + }, + "tls_certificates": [ + { + "certificate_chain": { + "filename": "/run/identity/tls.crt" + }, + "private_key": { + "filename": "/run/identity/tls.key" + } + } + ], + "validation_context": { + "trusted_ca": { + "filename": "/run/identity/ca.crt" + }, + "crl": { + "filename": "/run/identity/ca.crl" + }, + "match_typed_subject_alt_names": [ + { + "san_type": "DNS", + "matcher": { + "exact": "api" + } + }, + { + "san_type": "DNS", + "matcher": { + "exact": "worker" + } + }, + { + "san_type": "DNS", + "matcher": { + "exact": "operator-client" + } + } + ] + } + } + } + }, + "filters": [ + { + "name": "envoy.filters.network.http_connection_manager", + "typed_config": { + "@type": "type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager", + "stat_prefix": "s3", + "codec_type": "AUTO", + "route_config": { + "name": "routes", + "virtual_hosts": [ + { + "name": "local", + "domains": [ + "*" + ], + "routes": [ + { + "match": { + "prefix": "/" + }, + "route": { + "cluster": "s3", + "timeout": "125s" + } + } + ] + } + ] + }, + "http_filters": [ + { + "name": "envoy.filters.http.router", + "typed_config": { + "@type": "type.googleapis.com/envoy.extensions.filters.http.router.v3.Router" + } + } + ] + } + } + ] + } + ] + } + ], + "clusters": [ + { + "name": "s3", + "connect_timeout": "1s", + "type": "STATIC", + "load_assignment": { + "cluster_name": "s3", + "endpoints": [ + { + "lb_endpoints": [ + { + "endpoint": { + "address": { + "socket_address": { + "address": "127.0.0.1", + "port_value": 9000 + } + } + } + } + ] + } + ] + } + } + ] + } +} diff --git a/infra/envoy/tempo.json b/infra/envoy/tempo.json new file mode 100644 index 0000000..7e5aa62 --- /dev/null +++ b/infra/envoy/tempo.json @@ -0,0 +1,238 @@ +{ + "static_resources": { + "listeners": [ + { + "name": "query", + "address": { + "socket_address": { + "address": "0.0.0.0", + "port_value": 9443 + } + }, + "filter_chains": [ + { + "transport_socket": { + "name": "envoy.transport_sockets.tls", + "typed_config": { + "@type": "type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.DownstreamTlsContext", + "require_client_certificate": true, + "common_tls_context": { + "tls_params": { + "tls_minimum_protocol_version": "TLSv1_2" + }, + "tls_certificates": [ + { + "certificate_chain": { + "filename": "/run/identity/tls.crt" + }, + "private_key": { + "filename": "/run/identity/tls.key" + } + } + ], + "validation_context": { + "trusted_ca": { + "filename": "/run/identity/ca.crt" + }, + "crl": { + "filename": "/run/identity/ca.crl" + }, + "match_typed_subject_alt_names": [ + { + "san_type": "DNS", + "matcher": { + "exact": "grafana" + } + } + ] + } + } + } + }, + "filters": [ + { + "name": "envoy.filters.network.http_connection_manager", + "typed_config": { + "@type": "type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager", + "stat_prefix": "query", + "codec_type": "AUTO", + "route_config": { + "name": "routes", + "virtual_hosts": [ + { + "name": "local", + "domains": [ + "*" + ], + "routes": [ + { + "match": { + "prefix": "/" + }, + "route": { + "cluster": "query", + "timeout": "125s" + } + } + ] + } + ] + }, + "http_filters": [ + { + "name": "envoy.filters.http.router", + "typed_config": { + "@type": "type.googleapis.com/envoy.extensions.filters.http.router.v3.Router" + } + } + ] + } + } + ] + } + ] + }, + { + "name": "ingest", + "address": { + "socket_address": { + "address": "0.0.0.0", + "port_value": 9444 + } + }, + "filter_chains": [ + { + "transport_socket": { + "name": "envoy.transport_sockets.tls", + "typed_config": { + "@type": "type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.DownstreamTlsContext", + "require_client_certificate": true, + "common_tls_context": { + "tls_params": { + "tls_minimum_protocol_version": "TLSv1_2" + }, + "tls_certificates": [ + { + "certificate_chain": { + "filename": "/run/identity/tls.crt" + }, + "private_key": { + "filename": "/run/identity/tls.key" + } + } + ], + "validation_context": { + "trusted_ca": { + "filename": "/run/identity/ca.crt" + }, + "crl": { + "filename": "/run/identity/ca.crl" + }, + "match_typed_subject_alt_names": [ + { + "san_type": "DNS", + "matcher": { + "exact": "otel-collector" + } + } + ] + } + } + } + }, + "filters": [ + { + "name": "envoy.filters.network.http_connection_manager", + "typed_config": { + "@type": "type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager", + "stat_prefix": "ingest", + "codec_type": "AUTO", + "route_config": { + "name": "routes", + "virtual_hosts": [ + { + "name": "local", + "domains": [ + "*" + ], + "routes": [ + { + "match": { + "prefix": "/" + }, + "route": { + "cluster": "ingest", + "timeout": "125s" + } + } + ] + } + ] + }, + "http_filters": [ + { + "name": "envoy.filters.http.router", + "typed_config": { + "@type": "type.googleapis.com/envoy.extensions.filters.http.router.v3.Router" + } + } + ] + } + } + ] + } + ] + } + ], + "clusters": [ + { + "name": "query", + "connect_timeout": "1s", + "type": "STATIC", + "load_assignment": { + "cluster_name": "query", + "endpoints": [ + { + "lb_endpoints": [ + { + "endpoint": { + "address": { + "socket_address": { + "address": "127.0.0.1", + "port_value": 3200 + } + } + } + } + ] + } + ] + } + }, + { + "name": "ingest", + "connect_timeout": "1s", + "type": "STATIC", + "load_assignment": { + "cluster_name": "ingest", + "endpoints": [ + { + "lb_endpoints": [ + { + "endpoint": { + "address": { + "socket_address": { + "address": "127.0.0.1", + "port_value": 4318 + } + } + } + } + ] + } + ] + } + } + ] + } +} diff --git a/infra/generate.py b/infra/generate.py new file mode 100644 index 0000000..136d3e1 --- /dev/null +++ b/infra/generate.py @@ -0,0 +1,31 @@ +#!/usr/bin/env python3 +"""Generate deterministic public Envoy TLS boundaries (JSON is valid YAML).""" +import json +from pathlib import Path +ROOT=Path(__file__).resolve().parent + +def generate(name,listeners): + result={'static_resources':{'listeners':[],'clusters':[]}} + for label,port,upstream,identities in listeners: + tls={'@type':'type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.DownstreamTlsContext','require_client_certificate':True,'common_tls_context':{'tls_params':{'tls_minimum_protocol_version':'TLSv1_2'},'tls_certificates':[{'certificate_chain':{'filename':'/run/identity/tls.crt'},'private_key':{'filename':'/run/identity/tls.key'}}], 'validation_context':{'trusted_ca':{'filename':'/run/identity/ca.crt'},'crl':{'filename':'/run/identity/ca.crl'},'match_typed_subject_alt_names':[{'san_type':'DNS','matcher':{'exact':i}} for i in identities]}}} + route={'name':'routes','virtual_hosts':[{'name':'local','domains':['*'],'routes':[{'match':{'prefix':'/'},'route':{'cluster':label,'timeout':'125s'}}]}]} + filters=[] + if name in ['loki','prometheus']: + rules={} + for identity in identities: + methods=['GET','POST'] if identity=='grafana' and name=='prometheus' else ['GET'] if identity=='grafana' else ['POST'] + pattern='^/(api/v1/(query|query_range|labels|label/[^/]+/values|series|metadata|status/buildinfo)|-/ready).*' if name=='prometheus' else '^/(loki/api/v1/.*|ready)$' if identity=='grafana' else '^/otlp/v1/logs$' + rules[identity]={'principals':[{'authenticated':{'principal_name':{'exact':identity}}}],'permissions':[{'and_rules':{'rules':[{'header':{'name':':method','string_match':{'safe_regex':{'regex':'^('+'|'.join(methods)+')$'}}}},{'url_path':{'path':{'safe_regex':{'regex':pattern}}}}]}}]} + filters.append({'name':'envoy.filters.http.rbac','typed_config':{'@type':'type.googleapis.com/envoy.extensions.filters.http.rbac.v3.RBAC','rules':{'action':'ALLOW','policies':rules}}}) + filters.append({'name':'envoy.filters.http.router','typed_config':{'@type':'type.googleapis.com/envoy.extensions.filters.http.router.v3.Router'}}) + hcm={'@type':'type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager','stat_prefix':label,'codec_type':'AUTO','route_config':route,'http_filters':filters} + result['static_resources']['listeners'].append({'name':label,'address':{'socket_address':{'address':'0.0.0.0','port_value':port}},'filter_chains':[{'transport_socket':{'name':'envoy.transport_sockets.tls','typed_config':tls},'filters':[{'name':'envoy.filters.network.http_connection_manager','typed_config':hcm}]}]}) + result['static_resources']['clusters'].append({'name':label,'connect_timeout':'1s','type':'STATIC','load_assignment':{'cluster_name':label,'endpoints':[{'lb_endpoints':[{'endpoint':{'address':{'socket_address':{'address':'127.0.0.1','port_value':upstream}}}}]}]}}) + (ROOT/'envoy'/f'{name}.json').write_text(json.dumps(result,indent=2)+'\n') +for name,ls in { + 's3':[('s3',9443,9000,['api','worker','operator-client'])], + 'otel-collector':[('otlp',9443,4318,['api','worker','dispatcher']),('metrics',9444,8889,['prometheus']),('collector-metrics',9445,8888,['prometheus'])], + 'loki':[('loki',9443,3100,['otel-collector','grafana'])], + 'tempo':[('query',9443,3200,['grafana']),('ingest',9444,4318,['otel-collector'])], + 'prometheus':[('query',9443,9090,['grafana'])], +}.items():generate(name,ls) diff --git a/infra/haproxy.cfg b/infra/haproxy.cfg new file mode 100644 index 0000000..d890537 --- /dev/null +++ b/infra/haproxy.cfg @@ -0,0 +1,31 @@ +global + maxconn 1024 + stats socket /tmp/haproxy.sock mode 600 level admin + ssl-default-bind-options ssl-min-ver TLSv1.2 + ssl-default-server-options ssl-min-ver TLSv1.2 + +defaults + mode http + timeout connect 1s + timeout client 125s + timeout server 125s + timeout check 1s + retries 0 + +resolvers docker + nameserver dns 127.0.0.11:53 + hold valid 1s + hold obsolete 1s + +frontend public + bind :8443 ssl crt /run/identity/bundle.pem + http-request deny deny_status 404 if { path_beg /internal } + http-request set-header X-Forwarded-For %[src] + default_backend api + +backend api + balance roundrobin + option httpchk + http-check send meth GET uri /health/ready ver HTTP/1.1 hdr Host api + http-check expect status 200 + server-template replica 1-4 api:8080 ssl verify required ca-file /run/identity/ca.crt crt /run/identity/bundle.pem verifyhost api sni str(api) check inter 1s fall 2 rise 1 resolvers docker init-addr none diff --git a/infra/imagemagick-policy.xml b/infra/imagemagick-policy.xml new file mode 100644 index 0000000..decc717 --- /dev/null +++ b/infra/imagemagick-policy.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + diff --git a/infra/openbao/README.md b/infra/openbao/README.md index 0850e23..bbfb8fc 100644 --- a/infra/openbao/README.md +++ b/infra/openbao/README.md @@ -5,3 +5,7 @@ Решение должно показать реальную сервисную аутентификацию и получение нужного secret по ограниченной policy; просто работающий контейнер OpenBao не засчитывается. На 3 статический KV secret допустим. На 4 покажите expiry/renewal/rotation и allow/deny. Путь KV v2 data/metadata и ACL должен соответствовать фактическому mount; не копируйте широкое `path "*"` для приложения. Документируйте шаги оператора init/unseal и восстановление после restart. Скрипт не должен печатать tokens/shares или хранить их в Git. См. contracts/trust-matrix.md и REPORT.md. + +## Реализованный workflow + +`server.hcl` задаёт persistent file storage, TLS и declarative audit. Отдельные API/worker/dispatcher policies находятся рядом. `scripts/bao.py` делает operator init/unseal и provisioning; начальный root token отзывается после успеха. Приложение использует cert auth, scoped KV и dynamic database credentials. CA key и unseal shares не монтируются сервисам. После restart оператор выполняет `python3 scripts/bao.py unseal`; recovery и отказ sealed проверяются отдельным сценарием. Подробности и ограничения — в [REPORT.md](../../REPORT.md). diff --git a/infra/openbao/api.hcl b/infra/openbao/api.hcl new file mode 100644 index 0000000..cace6d7 --- /dev/null +++ b/infra/openbao/api.hcl @@ -0,0 +1,5 @@ +path "secret/data/api" { capabilities = ["read"] } +path "database/creds/api" { capabilities = ["read"] } +path "auth/token/renew-self" { capabilities = ["update"] } +path "auth/token/revoke-self" { capabilities = ["update"] } +path "sys/leases/renew" { capabilities = ["update"] } diff --git a/infra/openbao/dispatcher.hcl b/infra/openbao/dispatcher.hcl new file mode 100644 index 0000000..eddca3b --- /dev/null +++ b/infra/openbao/dispatcher.hcl @@ -0,0 +1,5 @@ +path "secret/data/dispatcher" { capabilities = ["read"] } +path "database/creds/dispatcher" { capabilities = ["read"] } +path "auth/token/renew-self" { capabilities = ["update"] } +path "auth/token/revoke-self" { capabilities = ["update"] } +path "sys/leases/renew" { capabilities = ["update"] } diff --git a/infra/openbao/server.hcl b/infra/openbao/server.hcl new file mode 100644 index 0000000..f4f0aea --- /dev/null +++ b/infra/openbao/server.hcl @@ -0,0 +1,14 @@ +ui = false +api_addr = "https://openbao:8200" +storage "file" { path = "/openbao/data" } +listener "tcp" { + address = "0.0.0.0:8200" + tls_cert_file = "/run/identity/tls.crt" + tls_key_file = "/run/identity/tls.key" + tls_client_ca_file = "/run/identity/ca.crt" + tls_min_version = "tls12" +} + +audit "file" "local" { + options { file_path = "/openbao/data/audit.json" } +} diff --git a/infra/openbao/worker.hcl b/infra/openbao/worker.hcl new file mode 100644 index 0000000..1c4e528 --- /dev/null +++ b/infra/openbao/worker.hcl @@ -0,0 +1,5 @@ +path "secret/data/worker" { capabilities = ["read"] } +path "database/creds/worker" { capabilities = ["read"] } +path "auth/token/renew-self" { capabilities = ["update"] } +path "auth/token/revoke-self" { capabilities = ["update"] } +path "sys/leases/renew" { capabilities = ["update"] } diff --git a/infra/otel/README.md b/infra/otel/README.md index c46fe89..cde5a62 100644 --- a/infra/otel/README.md +++ b/infra/otel/README.md @@ -5,3 +5,7 @@ Добавьте конфигурации storage, Grafana provisioning/datasources, provisioning/dashboards и импортируемые dashboard JSON. На 4 — alert rules и runbooks. Пустой dashboard или debug exporter, который только печатает данные в stdout, не заменяют хранилища/доказательства из `contracts/observability.md`. Не публикуйте OTLP/metrics/административные ports в общую сеть без необходимости. В lab7 реализуйте TLS/mTLS по trust matrix; экспорт Authorization и message bodies запрещён. + +## Реализованный workflow + +Collector принимает OTLP через mTLS Envoy boundary; plaintext receiver доступен только по loopback. Prometheus/Loki/Tempo и Grafana используют проверяемый TLS с разными identities и ingest/query restrictions. Dashboard/alerts — в этом каталоге; secure datasource provisioning создаёт bootstrap вне Git. Проверка реальных сигналов — `scripts/prove-telemetry.py`, диагностика — [RUNBOOK.md](RUNBOOK.md). diff --git a/infra/otel/RUNBOOK.md b/infra/otel/RUNBOOK.md new file mode 100644 index 0000000..5ca5174 --- /dev/null +++ b/infra/otel/RUNBOOK.md @@ -0,0 +1,13 @@ +# Диагностика lab7 + +Grafana доступна по HTTPS на localhost:13443. CA — локальный lab7 CA; anonymous выключен. Datasource client identity и trust material генерируются в закрытом secrets-каталоге, исходные dashboard и alert rules находятся в Git. + +Сначала проверить свежесть observation metrics и `up`, затем HTTP 5xx/p95, backlog/oldest age, DB pool и worker busy/capacity. Trace ID связывает upload/publish/consume/S3/DB с логами; request ID связывает HTTP-ошибку. Пропуск telemetry не означает нулевой backlog. + +При остановке worker API продолжает durable acceptance, растут backlog/age и QueueLag; после запуска backlog снижается. При отказе DB возрастают 5xx и DB spans/errors; при исчерпании pool readiness уходит в 503. При отказе Tempo/Loki SDK и Collector используют ограниченные queues/retry; API не делает эти backend критичными для readiness. Grafana также не влияет на readiness. + +Если TLS proxy не отвечает после рестарта backend, пересоздать соответствующий `*-tls` service: он должен войти в новый network namespace родителя. Не открывать backend plaintext listener на bridge. Проверить CA/SAN, срок leaf и актуальность CRL; не использовать insecure mode. + +При sealed OpenBao действующие grants используются до ограниченного deadline; затем процесс завершает операции и restart требует нового cert auth. Оператор выполняет `python3 scripts/bao.py unseal`, проверяет readiness и новый lease. При компрометации worker использовать `scripts/rotate-worker.py`, затем проверить jobs/checksums и старые credentials. + +Пороговые условия и окна alerts перенесены из lab6. CPU/RAM и error ratios интерпретировать в пределах synthetic profile и фактических limits из Compose. diff --git a/infra/otel/alerts.yaml b/infra/otel/alerts.yaml new file mode 100644 index 0000000..686b412 --- /dev/null +++ b/infra/otel/alerts.yaml @@ -0,0 +1,126 @@ +{ + "groups": [ + { + "name": "messenger-slo", + "interval": "5s", + "rules": [ + { + "record": "messenger:api_error_ratio:1m", + "expr": "(sum(rate(messenger_http_requests_total{route=~\"/api/v1/.*\",status_class=\"5xx\"}[1m])) or vector(0)) / sum(rate(messenger_http_requests_total{route=~\"/api/v1/.*\"}[1m]))" + }, + { + "record": "messenger:api_error_ratio:2m", + "expr": "(sum(rate(messenger_http_requests_total{route=~\"/api/v1/.*\",status_class=\"5xx\"}[2m])) or vector(0)) / sum(rate(messenger_http_requests_total{route=~\"/api/v1/.*\"}[2m]))" + }, + { + "record": "messenger:api_error_ratio:5m", + "expr": "(sum(rate(messenger_http_requests_total{route=~\"/api/v1/.*\",status_class=\"5xx\"}[5m])) or vector(0)) / sum(rate(messenger_http_requests_total{route=~\"/api/v1/.*\"}[5m]))" + }, + { + "record": "messenger:api_error_ratio:10m", + "expr": "(sum(rate(messenger_http_requests_total{route=~\"/api/v1/.*\",status_class=\"5xx\"}[10m])) or vector(0)) / sum(rate(messenger_http_requests_total{route=~\"/api/v1/.*\"}[10m]))" + }, + { + "alert": "ApiBurnRateFast", + "expr": "(messenger:api_error_ratio:1m > 0.06) and (messenger:api_error_ratio:5m > 0.06) and (sum(increase(messenger_http_requests_total{route=~\"/api/v1/.*\"}[1m])) >= 20)", + "for": "20s", + "labels": { + "severity": "warning", + "slo": "api" + }, + "annotations": { + "summary": "api SLO budget burning above 6x", + "runbook": "infra/otel/RUNBOOK.md" + } + }, + { + "alert": "ApiBurnRateSlow", + "expr": "(messenger:api_error_ratio:2m > 0.03) and (messenger:api_error_ratio:10m > 0.03) and (sum(increase(messenger_http_requests_total{route=~\"/api/v1/.*\"}[2m])) >= 20)", + "for": "30s", + "labels": { + "severity": "warning", + "slo": "api" + }, + "annotations": { + "summary": "api SLO budget burning above 3x", + "runbook": "infra/otel/RUNBOOK.md" + } + }, + { + "record": "messenger:ready_error_ratio:1m", + "expr": "sum(rate(messenger_ready_slo_total{outcome=\"bad\"}[1m])) / sum(rate(messenger_ready_slo_total[1m]))" + }, + { + "record": "messenger:ready_error_ratio:2m", + "expr": "sum(rate(messenger_ready_slo_total{outcome=\"bad\"}[2m])) / sum(rate(messenger_ready_slo_total[2m]))" + }, + { + "record": "messenger:ready_error_ratio:5m", + "expr": "sum(rate(messenger_ready_slo_total{outcome=\"bad\"}[5m])) / sum(rate(messenger_ready_slo_total[5m]))" + }, + { + "record": "messenger:ready_error_ratio:10m", + "expr": "sum(rate(messenger_ready_slo_total{outcome=\"bad\"}[10m])) / sum(rate(messenger_ready_slo_total[10m]))" + }, + { + "alert": "ReadyBurnRateFast", + "expr": "(messenger:ready_error_ratio:1m > 0.06) and (messenger:ready_error_ratio:5m > 0.06) and (sum(increase(messenger_ready_slo_total[1m])) >= 5)", + "for": "20s", + "labels": { + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "summary": "ready SLO budget burning above 6x", + "runbook": "infra/otel/RUNBOOK.md" + } + }, + { + "alert": "ReadyBurnRateSlow", + "expr": "(messenger:ready_error_ratio:2m > 0.03) and (messenger:ready_error_ratio:10m > 0.03) and (sum(increase(messenger_ready_slo_total[2m])) >= 5)", + "for": "30s", + "labels": { + "severity": "warning", + "slo": "ready" + }, + "annotations": { + "summary": "ready SLO budget burning above 3x", + "runbook": "infra/otel/RUNBOOK.md" + } + }, + { + "alert": "QueueLag", + "expr": "max(messenger_queue_oldest_age_seconds) > 30", + "for": "20s", + "labels": { + "severity": "warning" + }, + "annotations": { + "summary": "Unfinished attachment age exceeds 30s", + "runbook": "infra/otel/RUNBOOK.md" + } + }, + { + "alert": "ObservationsStale", + "expr": "time() - max(messenger_observation_timestamp) > 20 or absent(messenger_observation_timestamp)", + "for": "20s", + "labels": { + "severity": "warning" + }, + "annotations": { + "summary": "No fresh dispatcher observations", + "runbook": "infra/otel/RUNBOOK.md" + } + }, + { + "record": "messenger:api_error_ratio:1h", + "expr": "(sum(rate(messenger_http_requests_total{route=~\"/api/v1/.*\",status_class=\"5xx\"}[1h])) or vector(0)) / sum(rate(messenger_http_requests_total{route=~\"/api/v1/.*\"}[1h]))" + }, + { + "record": "messenger:ready_error_ratio:1h", + "expr": "sum(rate(messenger_ready_slo_total{outcome=\"bad\"}[1h])) / sum(rate(messenger_ready_slo_total[1h]))" + } + ] + } + ] +} diff --git a/infra/otel/collector.yaml b/infra/otel/collector.yaml new file mode 100644 index 0000000..0c266db --- /dev/null +++ b/infra/otel/collector.yaml @@ -0,0 +1,73 @@ +receivers: + otlp: + protocols: + http: + endpoint: 127.0.0.1:4318 +processors: + memory_limiter: + check_interval: 1s + limit_mib: 180 + spike_limit_mib: 40 + batch: + timeout: 1s + send_batch_size: 128 + send_batch_max_size: 256 +exporters: + prometheus: + endpoint: 127.0.0.1:8889 + resource_to_telemetry_conversion: + enabled: true + otlphttp/tempo: + tls: + ca_file: /run/identity/ca.crt + cert_file: /run/identity/tls.crt + key_file: /run/identity/tls.key + endpoint: https://tempo:9444 + timeout: 2s + retry_on_failure: + initial_interval: 1s + max_interval: 5s + max_elapsed_time: 30s + sending_queue: + queue_size: 32 + num_consumers: 2 + otlphttp/loki: + tls: + ca_file: /run/identity/ca.crt + cert_file: /run/identity/tls.crt + key_file: /run/identity/tls.key + endpoint: https://loki:9443/otlp + timeout: 2s + retry_on_failure: + initial_interval: 1s + max_interval: 5s + max_elapsed_time: 30s + sending_queue: + queue_size: 32 + num_consumers: 2 +extensions: + health_check: + endpoint: 127.0.0.1:13133 +service: + telemetry: + metrics: + readers: + - pull: + exporter: + prometheus: + host: 127.0.0.1 + port: 8888 + extensions: [health_check] + pipelines: + metrics: + receivers: [otlp] + processors: [memory_limiter, batch] + exporters: [prometheus] + traces: + receivers: [otlp] + processors: [memory_limiter, batch] + exporters: [otlphttp/tempo] + logs: + receivers: [otlp] + processors: [memory_limiter, batch] + exporters: [otlphttp/loki] diff --git a/infra/otel/dashboards/messenger.json b/infra/otel/dashboards/messenger.json new file mode 100644 index 0000000..0aea20d --- /dev/null +++ b/infra/otel/dashboards/messenger.json @@ -0,0 +1,770 @@ +{ + "uid": "messenger-lab6", + "title": "Messenger \u2014 lab6", + "schemaVersion": 39, + "version": 2, + "refresh": "5s", + "time": { + "from": "now-30m", + "to": "now" + }, + "panels": [ + { + "id": 1, + "title": "HTTP RPS", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 0, + "y": 0, + "w": 12, + "h": 8 + }, + "targets": [ + { + "refId": "A", + "expr": "sum by (route) (rate(messenger_http_requests_total[1m]))", + "legendFormat": "{{service_name}} {{route}} {{outcome}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "reqps" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + } + } + }, + { + "id": 2, + "title": "HTTP 5xx ratio", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 12, + "y": 0, + "w": 12, + "h": 8 + }, + "targets": [ + { + "refId": "A", + "expr": "messenger:api_error_ratio:1m", + "legendFormat": "{{service_name}} {{route}} {{outcome}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "percentunit" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + } + } + }, + { + "id": 3, + "title": "HTTP p50/p95/p99", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 0, + "y": 8, + "w": 12, + "h": 8 + }, + "targets": [ + { + "refId": "0.5", + "expr": "histogram_quantile(0.5,sum by (le)(rate(messenger_http_duration_seconds_bucket{route=~\"/api/v1/.*\"}[1m])))", + "legendFormat": "0.5" + }, + { + "refId": "0.95", + "expr": "histogram_quantile(0.95,sum by (le)(rate(messenger_http_duration_seconds_bucket{route=~\"/api/v1/.*\"}[1m])))", + "legendFormat": "0.95" + }, + { + "refId": "0.99", + "expr": "histogram_quantile(0.99,sum by (le)(rate(messenger_http_duration_seconds_bucket{route=~\"/api/v1/.*\"}[1m])))", + "legendFormat": "0.99" + } + ], + "fieldConfig": { + "defaults": { + "unit": "s" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + } + }, + "description": "Business routes only; histogram quantiles across all API replicas. Bucket interpolation is approximate." + }, + { + "id": 4, + "title": "Job outcomes", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 12, + "y": 8, + "w": 12, + "h": 8 + }, + "targets": [ + { + "refId": "A", + "expr": "sum by (outcome) (rate(messenger_jobs_total[1m]))", + "legendFormat": "{{service_name}} {{route}} {{outcome}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "ops" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + } + } + }, + { + "id": 5, + "title": "Processing p95", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 0, + "y": 16, + "w": 12, + "h": 8 + }, + "targets": [ + { + "refId": "A", + "expr": "histogram_quantile(0.95,sum by(le)(rate(messenger_job_duration_seconds_bucket[1m])))", + "legendFormat": "{{service_name}} {{route}} {{outcome}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "s" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + } + } + }, + { + "id": 6, + "title": "Time to ready p95", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 12, + "y": 16, + "w": 12, + "h": 8 + }, + "targets": [ + { + "refId": "A", + "expr": "histogram_quantile(0.95,sum by(le)(rate(messenger_job_ready_duration_seconds_bucket[1m])))", + "legendFormat": "{{service_name}} {{route}} {{outcome}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "s" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + } + } + }, + { + "id": 7, + "title": "RabbitMQ ready messages", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 0, + "y": 24, + "w": 12, + "h": 8 + }, + "targets": [ + { + "refId": "A", + "expr": "messenger_queue_depth", + "legendFormat": "{{service_name}} {{route}} {{outcome}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + } + } + }, + { + "id": 8, + "title": "Logical pending jobs / outbox", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 12, + "y": 24, + "w": 12, + "h": 8 + }, + "targets": [ + { + "refId": "A", + "expr": "messenger_jobs_pending", + "legendFormat": "jobs" + }, + { + "refId": "B", + "expr": "messenger_outbox_pending", + "legendFormat": "outbox" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + } + } + }, + { + "id": 9, + "title": "Oldest unfinished job", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 0, + "y": 32, + "w": 12, + "h": 8 + }, + "targets": [ + { + "refId": "A", + "expr": "messenger_queue_oldest_age_seconds", + "legendFormat": "{{service_name}} {{route}} {{outcome}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "s" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + } + } + }, + { + "id": 10, + "title": "DB connections used/capacity/waiting", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 12, + "y": 32, + "w": 12, + "h": 8 + }, + "targets": [ + { + "refId": "A", + "expr": "sum by (service_name)(messenger_db_pool_used)", + "legendFormat": "used {{service_name}}" + }, + { + "refId": "B", + "expr": "sum by (service_name)(messenger_db_pool_capacity)", + "legendFormat": "capacity {{service_name}}" + }, + { + "refId": "C", + "expr": "sum by (service_name)(messenger_db_pool_waiting)", + "legendFormat": "waiting {{service_name}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + } + } + }, + { + "id": 11, + "title": "Worker busy/capacity", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 0, + "y": 40, + "w": 12, + "h": 8 + }, + "targets": [ + { + "refId": "A", + "expr": "messenger_worker_busy{service_name=\"messenger-worker\"}", + "legendFormat": "busy" + }, + { + "refId": "B", + "expr": "messenger_worker_capacity{service_name=\"messenger-worker\"}", + "legendFormat": "capacity" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + } + } + }, + { + "id": 12, + "title": "Process memory RSS", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 12, + "y": 40, + "w": 12, + "h": 8 + }, + "targets": [ + { + "refId": "A", + "expr": "messenger_process_memory", + "legendFormat": "{{service_name}} {{route}} {{outcome}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "bytes" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + } + } + }, + { + "id": 13, + "title": "CPU by service (% of one core)", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 0, + "y": 48, + "w": 12, + "h": 8 + }, + "targets": [ + { + "refId": "A", + "expr": "100 * sum by(service_name)(rate(messenger_process_cpu[1m]))", + "legendFormat": "{{service_name}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "percent" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + } + } + }, + { + "id": 14, + "title": "API burn rate", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 12, + "y": 48, + "w": 12, + "h": 8 + }, + "targets": [ + { + "refId": "A", + "expr": "messenger:api_error_ratio:1m / 0.01", + "legendFormat": "{{service_name}} {{route}} {{outcome}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + } + } + }, + { + "id": 15, + "title": "Ready burn rate", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 0, + "y": 56, + "w": 12, + "h": 8 + }, + "targets": [ + { + "refId": "A", + "expr": "messenger:ready_error_ratio:1m / 0.01", + "legendFormat": "{{service_name}} {{route}} {{outcome}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + } + } + }, + { + "id": 16, + "title": "Alerts", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 12, + "y": 56, + "w": 12, + "h": 8 + }, + "targets": [ + { + "refId": "A", + "expr": "ALERTS{alertstate=\"firing\"}", + "legendFormat": "{{service_name}} {{route}} {{outcome}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + } + } + }, + { + "id": 17, + "title": "Collector export failures", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 0, + "y": 64, + "w": 12, + "h": 8 + }, + "targets": [ + { + "refId": "A", + "expr": "rate(otelcol_exporter_send_failed_spans_total[1m])", + "legendFormat": "{{service_name}} {{route}} {{outcome}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "ops" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + } + } + }, + { + "id": 18, + "title": "Observation freshness", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 12, + "y": 64, + "w": 12, + "h": 8 + }, + "targets": [ + { + "refId": "A", + "expr": "time() - messenger_observation_timestamp", + "legendFormat": "{{service_name}} {{route}} {{outcome}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "s" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + } + } + }, + { + "id": 19, + "title": "API SLI \u2014 1 hour", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 0, + "y": 72, + "w": 12, + "h": 8 + }, + "targets": [ + { + "refId": "A", + "expr": "1 - messenger:api_error_ratio:1h" + } + ], + "fieldConfig": { + "defaults": { + "unit": "percentunit" + }, + "overrides": [] + } + }, + { + "id": 20, + "title": "Ready SLI \u2014 1 hour", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 12, + "y": 72, + "w": 12, + "h": 8 + }, + "targets": [ + { + "refId": "A", + "expr": "1 - messenger:ready_error_ratio:1h" + } + ], + "fieldConfig": { + "defaults": { + "unit": "percentunit" + }, + "overrides": [] + } + }, + { + "id": 21, + "title": "Exporter bounded queue", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 0, + "y": 80, + "w": 12, + "h": 8 + }, + "targets": [ + { + "refId": "A", + "expr": "otelcol_exporter_queue_size", + "legendFormat": "size {{exporter}}" + }, + { + "refId": "B", + "expr": "otelcol_exporter_queue_capacity", + "legendFormat": "capacity {{exporter}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + } + }, + { + "id": 22, + "title": "DB lock waiters", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 12, + "y": 80, + "w": 12, + "h": 8 + }, + "targets": [ + { + "refId": "A", + "expr": "messenger_db_locks_waiting" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + } + } + ] +} diff --git a/infra/otel/grafana/dashboards/default.yaml b/infra/otel/grafana/dashboards/default.yaml new file mode 100644 index 0000000..0f96809 --- /dev/null +++ b/infra/otel/grafana/dashboards/default.yaml @@ -0,0 +1,7 @@ +apiVersion: 1 +providers: + - name: Messenger + folder: Lab6 + type: file + options: + path: /var/lib/grafana/dashboards diff --git a/infra/otel/loki.yaml b/infra/otel/loki.yaml new file mode 100644 index 0000000..4b2f95a --- /dev/null +++ b/infra/otel/loki.yaml @@ -0,0 +1,40 @@ +auth_enabled: false +server: + grpc_listen_address: 127.0.0.1 + http_listen_address: 127.0.0.1 + http_listen_port: 3100 +common: + instance_addr: 127.0.0.1 + path_prefix: /loki + replication_factor: 1 + ring: + kvstore: + store: inmemory + storage: + filesystem: + chunks_directory: /loki/chunks + rules_directory: /loki/rules +schema_config: + configs: + - from: 2024-01-01 + store: tsdb + object_store: filesystem + schema: v13 + index: + prefix: index_ + period: 24h +limits_config: + allow_structured_metadata: true + retention_period: 168h + otlp_config: + resource_attributes: + ignore_defaults: true + attributes_config: + - action: index_label + attributes: [service.name, deployment.environment.name] + - action: structured_metadata + attributes: [service.instance.id, service.version] +compactor: + working_directory: /loki/compactor + retention_enabled: true + delete_request_store: filesystem diff --git a/infra/otel/prometheus.yaml b/infra/otel/prometheus.yaml new file mode 100644 index 0000000..f647aea --- /dev/null +++ b/infra/otel/prometheus.yaml @@ -0,0 +1,22 @@ +global: + scrape_interval: 5s + evaluation_interval: 5s +rule_files: + - /etc/prometheus/alerts.yaml +scrape_configs: + - job_name: messenger + scheme: https + tls_config: &tls + ca_file: /run/identity/ca.crt + cert_file: /run/identity/tls.crt + key_file: /run/identity/tls.key + static_configs: + - targets: ['otel-collector:9444'] + - job_name: collector + scheme: https + tls_config: *tls + static_configs: + - targets: ['otel-collector:9445'] + - job_name: prometheus + static_configs: + - targets: ['localhost:9090'] diff --git a/infra/otel/tempo.yaml b/infra/otel/tempo.yaml new file mode 100644 index 0000000..f21e9ef --- /dev/null +++ b/infra/otel/tempo.yaml @@ -0,0 +1,26 @@ +server: + http_listen_address: 127.0.0.1 + grpc_listen_address: 127.0.0.1 + http_listen_port: 3200 +distributor: + receivers: + otlp: + protocols: + http: + endpoint: 127.0.0.1:4318 +ingester: + lifecycler: + address: 127.0.0.1 + max_block_duration: 5m +compactor: + compaction: + block_retention: 168h +storage: + trace: + backend: local + wal: + path: /var/tempo/wal + local: + path: /var/tempo/blocks +usage_report: + reporting_enabled: false diff --git a/infra/postgres/grants.sql b/infra/postgres/grants.sql new file mode 100644 index 0000000..9b0fe47 --- /dev/null +++ b/infra/postgres/grants.sql @@ -0,0 +1,10 @@ +REVOKE CREATE ON SCHEMA public FROM PUBLIC; +GRANT USAGE ON SCHEMA public TO api_role,worker_role,dispatcher_role; +GRANT SELECT,INSERT,UPDATE ON users,refresh_tokens,chats,chat_members,messages,message_idempotency,message_attachments,attachments,attachment_jobs,job_slo,outbox TO api_role; +GRANT SELECT ON attachment_variants TO api_role; +GRANT SELECT,UPDATE ON attachments,attachment_jobs,job_slo TO worker_role; +GRANT INSERT,SELECT ON attachment_variants,job_audit TO worker_role; +GRANT USAGE ON SEQUENCE job_audit_id_seq TO worker_role; +GRANT INSERT,SELECT ON outbox TO worker_role; +GRANT SELECT,UPDATE ON outbox,job_slo,attachments,attachment_jobs TO dispatcher_role; +GRANT SELECT ON attachment_variants TO dispatcher_role; diff --git a/infra/postgres/pg_hba.conf b/infra/postgres/pg_hba.conf new file mode 100644 index 0000000..cffea8c --- /dev/null +++ b/infra/postgres/pg_hba.conf @@ -0,0 +1,5 @@ +local all all trust +hostssl messenger all 0.0.0.0/0 scram-sha-256 +hostssl messenger all ::/0 scram-sha-256 +hostnossl all all 0.0.0.0/0 reject +hostnossl all all ::/0 reject diff --git a/infra/rabbitmq/definitions.json b/infra/rabbitmq/definitions.json new file mode 100644 index 0000000..9184878 --- /dev/null +++ b/infra/rabbitmq/definitions.json @@ -0,0 +1,87 @@ +{ + "users": [ + { + "name": "worker", + "password_hash": "", + "tags": [] + }, + { + "name": "dispatcher", + "password_hash": "", + "tags": [] + }, + { + "name": "operator-client", + "password_hash": "", + "tags": [] + } + ], + "vhosts": [ + { + "name": "attachments" + } + ], + "permissions": [ + { + "user": "worker", + "vhost": "attachments", + "configure": "^$", + "write": "^$", + "read": "^attachments\\.v1$" + }, + { + "user": "dispatcher", + "vhost": "attachments", + "configure": "^$", + "write": "^(attachments\\.publish|amq\\.default)$", + "read": "^attachments\\.v1$" + }, + { + "user": "operator-client", + "vhost": "attachments", + "configure": "^attachments.*", + "write": "^(attachments.*|amq\\.default)$", + "read": "^attachments.*" + } + ], + "queues": [ + { + "name": "attachments.v1", + "vhost": "attachments", + "durable": true, + "auto_delete": false, + "arguments": { + "x-dead-letter-exchange": "attachments.invalid", + "x-dead-letter-routing-key": "invalid" + } + }, + { + "name": "attachments.v1.invalid", + "vhost": "attachments", + "durable": true, + "auto_delete": false, + "arguments": {} + } + ], + "exchanges": [ + { + "name": "attachments.invalid", + "vhost": "attachments", + "type": "direct", + "durable": true, + "auto_delete": false, + "internal": false, + "arguments": {} + } + ], + "bindings": [ + { + "source": "attachments.invalid", + "vhost": "attachments", + "destination": "attachments.v1.invalid", + "destination_type": "queue", + "routing_key": "invalid", + "arguments": {} + } + ] +} diff --git a/infra/rabbitmq/enabled_plugins b/infra/rabbitmq/enabled_plugins new file mode 100644 index 0000000..70d609f --- /dev/null +++ b/infra/rabbitmq/enabled_plugins @@ -0,0 +1 @@ +[rabbitmq_auth_mechanism_ssl]. diff --git a/infra/rabbitmq/rabbitmq.conf b/infra/rabbitmq/rabbitmq.conf new file mode 100644 index 0000000..10a3a3e --- /dev/null +++ b/infra/rabbitmq/rabbitmq.conf @@ -0,0 +1,11 @@ +listeners.tcp = none +listeners.ssl.default = 5671 +ssl_options.cacertfile = /run/identity/ca.crt +ssl_options.certfile = /run/identity/tls.crt +ssl_options.keyfile = /run/identity/tls.key +ssl_options.verify = verify_peer +ssl_options.fail_if_no_peer_cert = true +auth_mechanisms.1 = EXTERNAL +ssl_cert_login_from = common_name +loopback_users = none +load_definitions = /run/identity/definitions.json diff --git a/scripts/audit-secrets.py b/scripts/audit-secrets.py new file mode 100644 index 0000000..ca8393f --- /dev/null +++ b/scripts/audit-secrets.py @@ -0,0 +1,23 @@ +#!/usr/bin/env python3 +"""Check versionable files against locally generated private material without printing it.""" +import json,subprocess +from pathlib import Path +ROOT=Path(__file__).resolve().parents[1];S=ROOT/'secrets/lab7' +needles=[] +for p in S.rglob('*'): + if p.is_file() and (p.name.endswith('_secret') or p.name.endswith('_password') or p.name in ['old_secret','old_access','tls.key','ca.key','jwt.key','password']): + value=p.read_bytes().strip() + if len(value)>=16:needles.append(value) +init=json.loads((S/'operator/init.json').read_text()) +for key in ['keys','keys_base64']: + needles.extend(x.encode() for x in init.get(key,[])) +assert not init.get('root_token'),'initial root token still stored' +files=subprocess.check_output(['git','ls-files','--cached','--others','--exclude-standard','-z'],cwd=ROOT).decode().split('\0');count=0 +for name in files: + p=ROOT/name + if name and p.is_file(): + count+=1;body=p.read_bytes();assert not any(n in body for n in needles),'generated secret detected in '+name +for name in ['secrets/lab7/operator/ca.key','secrets/lab7/operator/init.json','secrets/lab7/api/jwt.key']: + assert subprocess.run(['git','check-ignore','-q',name],cwd=ROOT).returncode==0 +ignore=(ROOT/'.dockerignore').read_text().splitlines();assert 'secrets' in ignore and 'certs' in ignore +(ROOT/'evidence/secret-audit.json').write_text(json.dumps({'versionable_files_checked':count,'generated_secrets_checked':len(needles),'matches':0,'private_material_ignored':True,'build_context_excludes_secrets':True,'initial_root_token_removed':True},indent=2)+'\n');print('PASS: generated credentials/private keys/unseal material absent from versionable files and excluded from build context') diff --git a/scripts/bao.py b/scripts/bao.py new file mode 100644 index 0000000..e283a30 --- /dev/null +++ b/scripts/bao.py @@ -0,0 +1,59 @@ +#!/usr/bin/env python3 +"""Operator-only OpenBao client. Does not log request/response bodies.""" +import os, json, ssl, urllib.request, urllib.error, time +from pathlib import Path +ROOT=Path(__file__).resolve().parents[1];S=ROOT/'secrets/lab7';O=S/'operator' +def call(path,body=None,token='',method=None,identity='operator-client'): + ctx=ssl.create_default_context(cafile=str(S/identity/'ca.crt'));ctx.load_cert_chain(S/identity/'tls.crt',S/identity/'tls.key') + req=urllib.request.Request(os.environ.get('LAB7_BAO_OPERATOR_ADDR','https://localhost:18200')+'/v1/'+path,data=json.dumps(body).encode() if body is not None else None,method=method or ('POST' if body is not None else 'GET'),headers={'Content-Type':'application/json','X-Vault-Token':token}) + with urllib.request.urlopen(req,context=ctx,timeout=5) as r: + b=r.read();return json.loads(b) if b else {} +def operator_token():return call('auth/cert/login',{'name':'operator'})['auth']['client_token'] +def unseal(): + for _ in range(60): + try:call('sys/init');break + except (OSError,urllib.error.URLError):time.sleep(1) + else:raise RuntimeError('OpenBao listener did not start') + data=json.loads((O/'init.json').read_text()) + for key in data['keys_base64'][:2]:call('sys/unseal',{'key':key}) +def bootstrap(): + O.mkdir(exist_ok=True) + for _ in range(60): + try: state=call('sys/init');break + except (OSError,urllib.error.URLError):time.sleep(1) + else:raise RuntimeError('OpenBao did not start') + if not state['initialized']: + data=call('sys/init',{'secret_shares':3,'secret_threshold':2});p=O/'init.json';p.write_text(json.dumps(data));p.chmod(0o600) + unseal();saved=json.loads((O/'init.json').read_text());token=saved.get('root_token') or operator_token() + if saved.get('root_token'): + for path,body in [('sys/auth/cert',{'type':'cert'}),('sys/mounts/secret',{'type':'kv','options':{'version':'2'}}),('sys/mounts/database',{'type':'database'})]: + try:call(path,body,token) + except urllib.error.HTTPError as e: + if e.code!=400:raise + policy='''path "sys/seal" { capabilities = ["update","sudo"] } +path "secret/*" { capabilities = ["create","read","update","delete","list"] } +path "database/*" { capabilities = ["create","read","update","delete","list"] } +path "auth/cert/*" { capabilities = ["create","read","update","delete","list"] } +path "auth/token/*" { capabilities = ["create","read","update","delete","list","sudo"] } +path "sys/leases/*" { capabilities = ["create","read","update","delete","list","sudo"] } +path "sys/policies/acl/*" { capabilities = ["create","read","update","delete","list"] } +path "sys/audit/*" { capabilities = ["create","read","update","delete","list","sudo"] } +''' + call('sys/policies/acl/operator',{'policy':policy},token) + call('auth/cert/certs/operator',{'certificate':(S/'operator-client/tls.crt').read_text(),'token_policies':['operator'],'token_ttl':300,'token_max_ttl':1800},token) + for role in ['api','worker','dispatcher']: + call('sys/policies/acl/'+role,{'policy':(ROOT/'infra/openbao'/f'{role}.hcl').read_text()},token) + call('auth/cert/certs/'+role,{'certificate':(S/role/'tls.crt').read_text(),'token_policies':[role],'token_ttl':300,'token_max_ttl':1800,'token_no_default_policy':True},token) + creds={'s3_access':(O/('s3_'+('api' if role=='dispatcher' else role)+'_access')).read_text(),'s3_secret':(O/('s3_'+('api' if role=='dispatcher' else role)+'_secret')).read_text()} + if role=='dispatcher':creds={'s3_access':'unused','s3_secret':'unused'} + if role=='api':creds['redis_password']=(O/'redis_password').read_text() + call('secret/data/'+role,{'data':creds},token) + call('database/roles/'+role,{'db_name':'messenger','creation_statements':['CREATE ROLE "{{name}}" WITH LOGIN PASSWORD \'{{password}}\' VALID UNTIL \'{{expiration}}\' IN ROLE '+role+'_role;'],'revocation_statements':['SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE usename=\'{{name}}\';','DROP ROLE IF EXISTS "{{name}}";'],'default_ttl':'600s','max_ttl':'1800s'},token) + call('database/config/messenger',{'plugin_name':'postgresql-database-plugin','allowed_roles':['api','worker','dispatcher'],'connection_url':'postgresql://{{username}}:{{password}}@postgres:5432/messenger?sslmode=verify-full&sslrootcert=/run/identity/ca.crt','username':'bao_manager','password':(O/'bao_db_password').read_text()},token) + if saved.get('root_token'): + call('auth/token/revoke-self',{},token);saved.pop('root_token',None);(O/'init.json').write_text(json.dumps(saved));(O/'init.json').chmod(0o600) + print('OpenBao unsealed and scoped policies/leases configured; initial root token revoked.') +if __name__=='__main__': + import sys + if len(sys.argv)>1 and sys.argv[1]=='unseal':unseal();print('OpenBao operator unseal completed.') + else:bootstrap() diff --git a/scripts/bootstrap.py b/scripts/bootstrap.py new file mode 100644 index 0000000..e202011 --- /dev/null +++ b/scripts/bootstrap.py @@ -0,0 +1,97 @@ +#!/usr/bin/env python3 +"""Idempotent local lab7 bootstrap. Never prints secrets or modifies lab6.""" +import json, os, secrets, shutil, subprocess +from pathlib import Path +ROOT=Path(__file__).resolve().parents[1]; S=ROOT/'secrets/lab7'; O=S/'operator' +def run(*args): subprocess.run(args,check=True,stdout=subprocess.DEVNULL,stderr=subprocess.PIPE) +def secret(name): + p=O/name + if not p.exists(): p.write_text(secrets.token_hex(24)) + p.chmod(0o600);return p.read_text().strip() +def issue(role,days=1,cn=None): + d=S/role;d.mkdir(exist_ok=True) + if not (d/'tls.crt').exists() or (d/'tls.crt').stat().st_size==0: + run('openssl','req','-new','-newkey','rsa:2048','-nodes','-keyout',str(d/'tls.key'),'-out',str(O/'request.csr'),'-subj','/CN='+(cn or role)) + ext=O/'leaf.ext';ext.write_text('[leaf]\nbasicConstraints=critical,CA:FALSE\nkeyUsage=critical,digitalSignature,keyEncipherment\nextendedKeyUsage=serverAuth,clientAuth\nsubjectAltName=DNS:'+role+(',DNS:localhost,IP:127.0.0.1' if role in ('lb','openbao','grafana') else '')+'\n') + run('openssl','ca','-batch','-config',str(O/'ca.cnf'),'-extensions','leaf','-extfile',str(ext),'-days',str(days),'-in',str(O/'request.csr'),'-out',str(d/'tls.crt'),'-notext') + for f in ['ca.crt','ca.crl']: + temporary=d/(f+'.tmp');shutil.copyfile(O/f,temporary);temporary.replace(d/f) + # The directory is the security boundary; only this identity directory is mounted in its container. + d.chmod(0o755) + + return d + +def main(): + os.umask(0o077);S.mkdir(parents=True,exist_ok=True);O.mkdir(exist_ok=True);O.chmod(0o700) + for p in S.rglob('*'): + if p.is_file():p.chmod(0o600) + for f,value in [('index.txt',''),('serial','1000\n'),('crlnumber','1000\n')]: + if not (O/f).exists():(O/f).write_text(value) + (O/'newcerts').mkdir(exist_ok=True) + (O/'ca.cnf').write_text(f'''[ca] +default_ca=local +[local] +dir={O} +database=$dir/index.txt +new_certs_dir=$dir/newcerts +certificate=$dir/ca.crt +private_key=$dir/ca.key +serial=$dir/serial +crlnumber=$dir/crlnumber +default_md=sha256 +default_days=1 +default_crl_days=1 +unique_subject=no +policy=policy +[policy] +commonName=supplied +[leaf] +''') + if not (O/'ca.key').exists():run('openssl','req','-x509','-newkey','rsa:3072','-nodes','-days','365','-keyout',str(O/'ca.key'),'-out',str(O/'ca.crt'),'-subj','/CN=lab7-local-ca','-addext','basicConstraints=critical,CA:TRUE','-addext','keyUsage=critical,keyCertSign,cRLSign') + run('openssl','ca','-config',str(O/'ca.cnf'),'-gencrl','-out',str(O/'ca.crl')) + roles=['lb','api','worker','dispatcher','postgres','redis','broker','s3','otel-collector','prometheus','loki','tempo','grafana','openbao','operator-client','stranger'] + for role in roles:issue(role) + if not (S/'api/jwt.key').exists(): + run('openssl','genrsa','-out',str(S/'api/jwt.key'),'2048');run('openssl','rsa','-in',str(S/'api/jwt.key'),'-pubout','-out',str(S/'api/jwt.pub')) + for p in (S/'api').glob('jwt.*'):p.chmod(0o444) + for name in ['db_password','bao_db_password','redis_password','grafana_password','s3_admin_access','s3_admin_secret','s3_api_access','s3_api_secret','s3_worker_access','s3_worker_secret','s3_baseline_access','s3_baseline_secret']:secret(name) + bucket='messenger-attachments' + identities=[] + for role,actions in [('baseline',[f'Read:{bucket}',f'Write:{bucket}',f'List:{bucket}']),('admin',['Admin']),('api',[f'Read:{bucket}/results/*',f'Read:{bucket}/originals/*',f'Write:{bucket}/staging/*',f'Read:{bucket}']),('worker',[f'Read:{bucket}/staging/*',f'Write:{bucket}/staging/*',f'Read:{bucket}/results/*',f'Write:{bucket}/results/*',f'Read:{bucket}'])]: + # Bucket-scoped Read also permits arbitrary object reads; do not grant it. HeadBucket is handled by health prefix probes instead. + if role not in ['admin','baseline']:actions=[a for a in actions if a!=f'Read:{bucket}'] + identities.append({'name':role,'credentials':[{'accessKey':secret('s3_'+role+'_access'),'secretKey':secret('s3_'+role+'_secret')}],'actions':actions}) + (S/'s3/s3.json').write_text(json.dumps({'identities':identities}));(S/'s3/s3.json').chmod(0o444) + (S/'postgres/password').write_text(secret('db_password'));(S/'postgres/password').chmod(0o444) + (S/'postgres/init.sql').write_text("CREATE ROLE bao_manager WITH LOGIN CREATEROLE PASSWORD '"+secret('bao_db_password')+"';\nCREATE ROLE api_role NOLOGIN; CREATE ROLE worker_role NOLOGIN; CREATE ROLE dispatcher_role NOLOGIN;\nGRANT api_role,worker_role,dispatcher_role TO bao_manager WITH ADMIN OPTION;\nGRANT pg_signal_backend TO bao_manager;\n") + (S/'postgres/init.sql').chmod(0o444) + (S/'redis/users.acl').write_text('user default off\nuser api on >'+secret('redis_password')+' ~ratelimit:* ~chat:* ~chats:* +@read +@write +ping +evalsha +eval +script|load +script|exists -flushall -flushdb -config -acl\n');(S/'redis/users.acl').chmod(0o444) + (S/'grafana/password').write_text(secret('grafana_password'));(S/'grafana/password').chmod(0o444) + for role in ('api','worker','dispatcher','operator-client'): + (S/role/'broker_url').write_text('amqps://broker:5671/attachments');(S/role/'broker_url').chmod(0o444) + (S/'lb/bundle.pem').write_text((S/'lb/tls.crt').read_text()+(S/'lb/tls.key').read_text());(S/'lb/bundle.pem').chmod(0o444) + for name in ['s3_admin_access','s3_admin_secret','s3_baseline_access','s3_baseline_secret']: + (S/'operator-client'/name).write_text(secret(name));(S/'operator-client'/name).chmod(0o444) + # Provisioning contains the Grafana client private key, so it is generated outside Git. + sources=[] + for name in ['prometheus','loki','tempo']: + sources.append({'name':name.title(),'uid':name,'type':name,'url':'https://'+name+':9443','access':'proxy','isDefault':name=='prometheus','jsonData':{'tlsAuth':True,'tlsAuthWithCACert':True,'serverName':name,'timeInterval':'5s'},'secureJsonData':{'tlsCACert':(O/'ca.crt').read_text(),'tlsClientCert':(S/'grafana/tls.crt').read_text(),'tlsClientKey':(S/'grafana/tls.key').read_text()}}) + (S/'grafana/datasources.yaml').write_text(json.dumps({'apiVersion':1,'datasources':sources},indent=2)) + for role in roles: + for p in (S/role).iterdir():p.chmod(0o444) + definitions=json.loads((ROOT/'infra/rabbitmq/definitions.json').read_text()) + cn=subprocess.check_output(['openssl','x509','-in',str(S/'worker/tls.crt'),'-noout','-subject','-nameopt','RFC2253'],text=True).strip().split('CN=')[-1] + for user in definitions['users']: + if user['name']=='worker':user['name']=cn + for permission in definitions['permissions']: + if permission['user']=='worker':permission['user']=cn + (S/'broker/definitions.json').chmod(0o600) if (S/'broker/definitions.json').exists() else None + (S/'broker/definitions.json').write_text(json.dumps(definitions));(S/'broker/definitions.json').chmod(0o444) + foreign=S/'foreign';foreign.mkdir(exist_ok=True);foreign.chmod(0o755) + if not (foreign/'tls.crt').exists(): + run('openssl','req','-x509','-newkey','rsa:2048','-nodes','-days','1','-keyout',str(foreign/'tls.key'),'-out',str(foreign/'tls.crt'),'-subj','/CN=foreign','-addext','subjectAltName=DNS:foreign') + (foreign/'ca.crt').chmod(0o600) if (foreign/'ca.crt').exists() else None + shutil.copyfile(O/'ca.crt',foreign/'ca.crt') + for p in foreign.iterdir():p.chmod(0o444) + print('Local lab7 CA, identities and synthetic secrets prepared. No credentials printed.') +if __name__=='__main__':main() diff --git a/scripts/build-image.py b/scripts/build-image.py new file mode 100644 index 0000000..ce5dd0b --- /dev/null +++ b/scripts/build-image.py @@ -0,0 +1,10 @@ +#!/usr/bin/env python3 +"""Build public images with a temporary Docker CLI config, avoiding platform keychain helpers.""" +import json,os,subprocess,tempfile +from pathlib import Path +ROOT=Path(__file__).resolve().parents[1] +endpoint=subprocess.check_output(['docker','context','inspect','--format','{{.Endpoints.docker.Host}}'],text=True).strip() +with tempfile.TemporaryDirectory(prefix='lab7-docker-public-') as directory: + Path(directory,'config.json').write_text(json.dumps({'auths':{},'cliPluginsExtraDirs':[str(Path.home()/'.docker/cli-plugins')]})) + env=dict(os.environ,DOCKER_CONFIG=directory,DOCKER_HOST=endpoint);env.pop('DOCKER_CONTEXT',None) + subprocess.run([str(ROOT/'scripts/compose.sh'),'build','api'],cwd=ROOT,env=env,check=True) diff --git a/scripts/capture-environment.py b/scripts/capture-environment.py new file mode 100644 index 0000000..ad3e11a --- /dev/null +++ b/scripts/capture-environment.py @@ -0,0 +1,11 @@ +#!/usr/bin/env python3 +import datetime,json,platform,subprocess +from pathlib import Path +root=Path(__file__).resolve().parents[1] +def cmd(*a):return subprocess.check_output(a,text=True,cwd=root).strip() +containers=[] +for cid in cmd(str(root/'scripts/compose.sh'),'ps','-q').split(): + c=json.loads(cmd('docker','inspect',cid))[0];h=c['HostConfig'] + containers.append({'name':c['Name'],'image_id':c['Image'],'started_at':c['State']['StartedAt'],'user':c['Config']['User'],'memory_limit':h['Memory'],'nano_cpus':h['NanoCpus'],'read_only':h['ReadonlyRootfs'],'cap_drop':h['CapDrop'],'security_opt':h['SecurityOpt'],'port_bindings':h['PortBindings']}) +data={'timestamp':datetime.datetime.now(datetime.timezone.utc).isoformat(),'base_commit':cmd('git','rev-parse','HEAD'),'working_tree_changes':True,'system':platform.platform(),'architecture':platform.machine(),'docker':cmd('docker','version','--format','{{.Server.Version}}'),'compose':cmd('docker','compose','version','--short'),'containers':containers,'docker_cpus':int(cmd('docker','info','--format','{{.NCPU}}')),'docker_memory_bytes':int(cmd('docker','info','--format','{{.MemTotal}}'))} +(root/'evidence/environment.json').write_text(json.dumps(data,indent=2)+'\n');print('Sanitized environment recorded.') diff --git a/scripts/compose.sh b/scripts/compose.sh new file mode 100755 index 0000000..9d81876 --- /dev/null +++ b/scripts/compose.sh @@ -0,0 +1,4 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +exec docker compose -p "${LAB7_PROJECT:-mtusi2026lab7}" -f compose.yaml "$@" diff --git a/scripts/experiment-worker.py b/scripts/experiment-worker.py new file mode 100644 index 0000000..450575a --- /dev/null +++ b/scripts/experiment-worker.py @@ -0,0 +1,59 @@ +#!/usr/bin/env python3 +"""Repeatable synthetic worker compromise/recovery experiment. No credentials in results.""" +import hashlib,importlib.util,json,statistics,subprocess,sys,time,urllib.error,uuid +from pathlib import Path +ROOT=Path(__file__).resolve().parents[1];sys.path.insert(0,str(ROOT/'scripts')) +import security,bao +spec=importlib.util.spec_from_file_location('rotation',ROOT/'scripts/rotate-worker.py');rotation=importlib.util.module_from_spec(spec);spec.loader.exec_module(rotation) + +def db(sql):return security.compose('exec','-T','postgres','psql','-U','messenger','-d','messenger','-At','-v','ON_ERROR_STOP=1','-c',sql).stdout.strip() +def one(index): + name,pw,token,user=security.actor();auth={'Authorization':'Bearer '+token} + status,_,b=security.request('/api/v1/chats','POST',json.dumps({'title':'Synthetic recovery','member_ids':[user]}).encode(),dict(auth,**{'Content-Type':'application/json'}));assert status==201 + chat=json.loads(b)['id'];security.compose('stop','worker') + fixture=(ROOT/'tests/fixtures/sample.png').read_bytes();ids=[];latencies=[] + # Work is accepted durably while the consumer is absent. + for i in range(5): + start=time.monotonic();status,_,b=security.request('/api/v1/chats/'+chat+'/attachments?filename=synthetic.png','POST',fixture,dict(auth,**{'Content-Type':'image/png'}));latencies.append(time.monotonic()-start);assert status==202;ids.append(json.loads(b)['id']) + idsql=','.join("'"+i+"'" for i in ids) + before=db('SELECT count(*) FROM attachment_jobs WHERE attachment_id IN ('+idsql+')');assert before=='5' + restricted={m:security.probe(m) for m in ['db','s3','amqp']} + # The broad DB profile from lab6 is measured without reading secrets or retaining changes. + broad=db("BEGIN; SELECT count(*)=0 FROM (SELECT password_hash FROM users LIMIT 0) x; CREATE TABLE baseline_probe(id int); ROLLBACK;") + broad_profiles={} + for mode in ['s3','amqp']: + args=['run','--rm','-T','--no-deps','--entrypoint','securityprobe','-e','BAO_ADDR=','-e','PROBE_BASELINE=1','-e','PROBE_MODE='+mode,'-e','S3_ACCESS_KEY_ID_FILE=/run/identity/s3_baseline_access','-e','S3_SECRET_ACCESS_KEY_FILE=/run/identity/s3_baseline_secret','-v',str(security.S/'operator-client')+':/run/identity:ro','worker'] + broad_profiles[mode]=json.loads(security.compose(*args).stdout) + start=time.monotonic();rotation.rotate(start=False) + # Old worker auth is explicitly rejected even though its CA is still trusted. + try:bao.call('auth/cert/login',{'name':'worker'},identity='worker-old') + except urllib.error.HTTPError as e:assert e.code in [400,403] + else:raise AssertionError('old worker can authenticate') + security.probe('tls',target='https://otel-collector:9443/v1/traces',expect='deny',material='worker-old') + security.probe('amqp-tls',expect='deny',material='worker-old') + old_result=security.compose('run','--rm','-T','--no-deps','--entrypoint','securityprobe','-e','BAO_ADDR=','-e','PROBE_MODE=s3','-e','PROBE_OLD_SECRET=1','-e','S3_ACCESS_KEY_ID_FILE=/run/old_access','-e','S3_SECRET_ACCESS_KEY_FILE=/run/old_secret','-v',str(security.S/'worker-old/old_access')+':/run/old_access:ro','-v',str(security.S/'worker-old/old_secret')+':/run/old_secret:ro','worker').stdout + assert json.loads(old_result)['old_s3_secret_rejected'] + security.compose('up','-d','worker') + errors=0 + for _ in range(90): + statuses=[] + for aid in ids: + status,_,b=security.request('/api/v1/attachments/'+aid,headers=auth) + if status!=200:errors+=1;continue + statuses.append(json.loads(b)['status']) + if len(statuses)==5 and set(statuses)=={'ready'}:break + time.sleep(1) + else:raise AssertionError('accepted jobs did not recover') + recovery=time.monotonic()-start + sha=hashlib.sha256(fixture).hexdigest() + for aid in ids: + status,_,b=security.request('/api/v1/attachments/'+aid+'/content',headers=auth);assert status==200 and hashlib.sha256(b).hexdigest()==sha + total,unique=db('SELECT count(*),count(DISTINCT attachment_id) FROM attachment_jobs WHERE attachment_id IN ('+idsql+')').split('|');assert total==unique=='5' + effects=db('SELECT count(*) FROM attachment_variants WHERE attachment_id IN ('+idsql+')');assert effects=='20',effects + samples=security.compose('stats','--no-stream','--format','json').stdout + return {'repeat':index,'accepted':5,'ready':5,'lost':0,'duplicate_jobs':0,'variant_records':int(effects),'original_checksums_match':True,'old_identity_rejected':True,'recovery_seconds':recovery,'observation_errors':errors,'upload_latency_ms':{'p50':statistics.median(latencies)*1000,'max':max(latencies)*1000},'restricted_probes':restricted,'broad_db_read_and_ddl':True,'broad_profiles':broad_profiles,'old_s3_key_rejected':True,'old_broker_identity_rejected':True,'resource_snapshot':[json.loads(x) for x in samples.splitlines() if x.startswith('{')]} +def main(): + results=[] + for i in range(1,4): + results.append(one(i));(ROOT/'evidence/worker-experiment.json').write_text(json.dumps({'hypothesis':'Scoped worker identity limits DB/S3/broker access; revocation and replacement preserve accepted benign jobs.','timestamp':time.strftime('%Y-%m-%dT%H:%M:%SZ',time.gmtime()),'repeats':results,'limitations':['Authorized worker can damage its permitted job metadata and object prefixes.','This experiment does not claim integrity after malicious modification of those authorized resources.','Single Docker host; no host/DB/LB high availability.']},indent=2)+'\n');print('PASS: worker compromise/recovery repeat',i) +if __name__=='__main__':main() diff --git a/scripts/final-checks.sh b/scripts/final-checks.sh new file mode 100755 index 0000000..0221942 --- /dev/null +++ b/scripts/final-checks.sh @@ -0,0 +1,33 @@ +#!/bin/sh +# State-changing scenarios are deliberately sequential, on synthetic lab7 data. +set -eu +cd "$(dirname "$0")/.." +make check validate PYTHON=.venv/bin/python > evidence/contracts.txt +make build vet test-go > evidence/go-checks.txt +make test test-security > evidence/public-checks.txt +python3 scripts/security.py > evidence/security-run.txt +python3 scripts/prove-auth.py > evidence/auth-run.txt +python3 scripts/prove-leaf.py > evidence/leaf-run.txt +python3 scripts/prove-sealed.py > evidence/sealed-run.txt +python3 scripts/prove-telemetry.py > evidence/telemetry-run.txt +python3 scripts/prove-admin.py > evidence/admin-run.txt +python3 scripts/prove-cert-expiry.py > evidence/cert-expiry-run.txt +python3 student-tests/images.py > evidence/images-run.txt +python3 student-tests/storage.py > evidence/storage-run.txt +python3 student-tests/broker_persistence.py > evidence/broker-run.txt +make test-persistence ACTION_SCRIPT=scripts/restart.sh > evidence/persistence-run.txt +make test-failover ACTION_SCRIPT=scripts/stop-one.sh > evidence/failover-first.txt +./scripts/compose.sh up -d api +python3 scripts/wait-ready.py +make test-failover ACTION_SCRIPT=scripts/stop-one.sh > evidence/failover-second.txt +./scripts/compose.sh up -d api +python3 scripts/wait-ready.py +python3 scripts/prove-all-replicas.py > evidence/all-replicas-run.txt +python3 scripts/experiment-worker.py > evidence/worker-experiment-run.txt +make test > evidence/final-smoke.txt +python3 scripts/capture-environment.py +python3 scripts/prove-clean-clone.py > evidence/clean-clone-run.txt +python3 scripts/scan-images.py +make scan-dependencies +python3 scripts/audit-secrets.py +python3 scripts/summarize-evidence.py diff --git a/scripts/generate-types.py b/scripts/generate-types.py new file mode 100644 index 0000000..b823e11 --- /dev/null +++ b/scripts/generate-types.py @@ -0,0 +1,17 @@ +#!/usr/bin/env python3 +"""Convert a temporary generation view to the oapi-codegen 3.0 subset. +Runtime handlers enforce anyOf and required fields; normative contract is untouched. +""" +import json +from pathlib import Path +s=json.loads(Path('contracts/openapi.json').read_text()) +for schema in s['components']['schemas'].values(): + schema.pop('anyOf',None) + for prop in schema.get('properties',{}).values(): + if prop.get('type')==['string','null']: + prop.update(type='string',nullable=True) +# Preserve pointers so handlers can distinguish missing fields from empty values. +for name,field in [('CreateUser','password'),('CreateMessage','text')]: + s['components']['schemas'][name]['properties'][field]['x-go-type-skip-optional-pointer']=False + s['components']['schemas'][name]['required'].remove(field) +print(json.dumps(s)) diff --git a/scripts/prove-admin.py b/scripts/prove-admin.py new file mode 100644 index 0000000..8dc15f9 --- /dev/null +++ b/scripts/prove-admin.py @@ -0,0 +1,12 @@ +#!/usr/bin/env python3 +import base64,json,ssl,sys,urllib.error,urllib.request +from pathlib import Path +root=Path(__file__).resolve().parents[1];s=root/'secrets/lab7';ctx=ssl.create_default_context(cafile=str(s/'grafana/ca.crt')) +url='https://localhost:13443/api/user' +try:urllib.request.urlopen(url,context=ctx) +except urllib.error.HTTPError as e:assert e.code==401 +else:raise AssertionError('Grafana user API allowed anonymous access') +password=(s/'operator/grafana_password').read_text().strip();req=urllib.request.Request(url,headers={'Authorization':'Basic '+base64.b64encode(('admin:'+password).encode()).decode()}) +with urllib.request.urlopen(req,context=ctx) as response: + data=json.load(response);assert data['login']=='admin' +(root/'evidence/admin.json').write_text(json.dumps({'grafana_verified_https':True,'anonymous_user_api_denied':True,'operator_authenticated':True},indent=2)+'\n');print('PASS: Grafana HTTPS operator authentication and anonymous denial') diff --git a/scripts/prove-all-replicas.py b/scripts/prove-all-replicas.py new file mode 100644 index 0000000..f23e578 --- /dev/null +++ b/scripts/prove-all-replicas.py @@ -0,0 +1,26 @@ +#!/usr/bin/env python3 +import json,subprocess,sys,time,uuid +from pathlib import Path +ROOT=Path(__file__).resolve().parents[1];sys.path.insert(0,str(ROOT/'scripts'));import security +name,password,token,user=security.actor();auth={'Authorization':'Bearer '+token} +status,_,b=security.request('/api/v1/chats','POST',json.dumps({'title':'Sequential failures','member_ids':[user]}).encode(),dict(auth,**{'Content-Type':'application/json'}));assert status==201;chat=json.loads(b)['id'];results=[] +for cid in security.compose('ps','-q','api').stdout.split(): + started=time.monotonic();subprocess.run(['docker','stop',cid],check=True,stdout=subprocess.DEVNULL) + try: + key=uuid.uuid4().hex;body=json.dumps({'text':'synthetic persistence during replica failure'}).encode();h=dict(auth,**{'Content-Type':'application/json','Idempotency-Key':key}) + for _ in range(30): + status,headers,b=security.request('/api/v1/chats/'+chat+'/messages','POST',body,h) + if status==201:break + time.sleep(.2) + else:raise AssertionError('surviving replica did not accept write') + accepted=json.loads(b);elapsed=time.monotonic()-started;assert elapsed<10 + for _ in range(30): + status,_,b=security.request('/api/v1/chats/'+chat+'/messages','POST',body,h) + if status==201:break + assert status==503,status;time.sleep(.2) + assert status==201 and json.loads(b)['id']==accepted['id'] + status,_,b=security.request('/api/v1/chats/'+chat+'/messages',headers=auth);assert status==200;assert sum(x['id']==accepted['id'] for x in json.loads(b)['items'])==1 + results.append({'stopped_container':cid,'recovery_seconds':elapsed,'original_session_valid':True,'no_duplicate':True}) + finally: + subprocess.run(['docker','start',cid],check=True,stdout=subprocess.DEVNULL);time.sleep(3) +(ROOT/'evidence/all-replicas.json').write_text(json.dumps(results,indent=2)+'\n');print('PASS: each physical API container stopped sequentially; session/write/idempotency preserved') diff --git a/scripts/prove-auth.py b/scripts/prove-auth.py new file mode 100644 index 0000000..5773cea --- /dev/null +++ b/scripts/prove-auth.py @@ -0,0 +1,33 @@ +#!/usr/bin/env python3 +import base64,json,subprocess,sys,time,concurrent.futures,statistics,uuid +from pathlib import Path +ROOT=Path(__file__).resolve().parents[1];sys.path.insert(0,str(ROOT/'scripts'));import security + +def main(): + name,password,token,user=security.actor() + args=['run','--rm','-T','--no-deps','--entrypoint','securityprobe','-e','BAO_ADDR=','-e','PROBE_MODE=hash','-e','ARGON2_PARALLELISM=4','-e','PROBE_PASSWORD='+password,'worker'] + old=security.compose(*args).stdout.strip();assert old.startswith('$argon2id$v=19$m=65536,t=3,p=4$') + sql="UPDATE users SET password_hash='"+old+"' WHERE id='"+user+"';" + subprocess.run([str(ROOT/'scripts/compose.sh'),'exec','-T','postgres','psql','-U','messenger','-d','messenger','-v','ON_ERROR_STOP=1'],input=sql,text=True,check=True,stdout=subprocess.DEVNULL,stderr=subprocess.PIPE) + status,_,_=security.request('/api/v1/auth/sessions','POST',headers={'Authorization':'Basic '+base64.b64encode((name+':'+password).encode()).decode()});assert status==201 + query="SELECT password_hash LIKE '$argon2id$v=19$m=65536,t=3,p=1$%' FROM users WHERE id='"+user+"';" + assert security.compose('exec','-T','postgres','psql','-U','messenger','-d','messenger','-Atc',query).stdout.strip()=='t' + kdf=security.compose('run','--rm','-T','--no-deps','--entrypoint','securityprobe','-e','BAO_ADDR=','-e','PROBE_MODE=kdf','worker').stdout + result={'rehash_lab6_profile_on_login':True,'kdf':json.loads(kdf)} + def login_attempt(i): + credentials=base64.b64encode(('absent_'+uuid.uuid4().hex[:12]+':synthetic-wrong-password').encode()).decode();start=time.monotonic() + status,_,_=security.request('/api/v1/auth/sessions','POST',headers={'Authorization':'Basic '+credentials});return status,(time.monotonic()-start)*1000 + started=time.monotonic() + with concurrent.futures.ThreadPoolExecutor(max_workers=8) as executor:attempts=list(executor.map(login_attempt,range(24))) + result['login_dos_budget']={'requests':24,'concurrency':8,'duration_seconds':time.monotonic()-started,'statuses':{str(s):sum(x[0]==s for x in attempts) for s in set(x[0] for x in attempts)},'p95_all_ms':sorted(x[1] for x in attempts)[22],'api_memory_limit_mib':512,'max_kdf_memory_mib_per_process':128,'max_kdf_parallel_per_process':2} + + override=ROOT/'secrets/lab7/operator/expiry.json';override.write_text(json.dumps({'services':{'api':{'environment':{'SESSION_TTL_SECONDS':'2'}}}})) + base=[str(ROOT/'scripts/compose.sh')] + try: + subprocess.run(['docker','compose','-p','mtusi2026lab7','-f',str(ROOT/'compose.yaml'),'-f',str(override),'up','-d','api'],check=True,stdout=subprocess.DEVNULL,stderr=subprocess.PIPE) + subprocess.run([sys.executable,str(ROOT/'scripts/wait-ready.py')],check=True,stdout=subprocess.DEVNULL) + _,_,short,_=security.actor();time.sleep(3);assert security.request('/api/v1/users/me',headers={'Authorization':'Bearer '+short})[0]==401;result['session_ttl_2_seconds_expires']=True + finally: + security.compose('up','-d','api');subprocess.run([sys.executable,str(ROOT/'scripts/wait-ready.py')],check=True,stdout=subprocess.DEVNULL) + (ROOT/'evidence/auth.json').write_text(json.dumps(result,indent=2)+'\n');print('PASS: legacy rehash, KDF measurements and session expiry') +if __name__=='__main__':main() diff --git a/scripts/prove-cert-expiry.py b/scripts/prove-cert-expiry.py new file mode 100644 index 0000000..441b478 --- /dev/null +++ b/scripts/prove-cert-expiry.py @@ -0,0 +1,14 @@ +#!/usr/bin/env python3 +import datetime,json,subprocess,sys +from pathlib import Path +ROOT=Path(__file__).resolve().parents[1];sys.path.insert(0,str(ROOT/'scripts'));import bootstrap,security +p=security.S/'expired';p.mkdir(exist_ok=True);p.chmod(0o755) +for f in p.iterdir():f.chmod(0o600) +bootstrap.run('openssl','req','-new','-newkey','rsa:2048','-nodes','-keyout',str(p/'tls.key'),'-out',str(p/'request.csr'),'-subj','/CN=expired-lb') +(p/'extensions.cnf').write_text('[leaf]\nbasicConstraints=critical,CA:FALSE\nkeyUsage=critical,digitalSignature,keyEncipherment\nextendedKeyUsage=clientAuth\nsubjectAltName=DNS:lb\n') +now=datetime.datetime.now(datetime.timezone.utc);date=lambda d:(now-datetime.timedelta(days=d)).strftime('%y%m%d%H%M%SZ') +bootstrap.run('openssl','ca','-batch','-config',str(bootstrap.O/'ca.cnf'),'-extensions','leaf','-extfile',str(p/'extensions.cnf'),'-startdate',date(2),'-enddate',date(1),'-in',str(p/'request.csr'),'-out',str(p/'tls.crt'),'-notext') +(p/'ca.crt').write_text((bootstrap.O/'ca.crt').read_text()) +for f in p.iterdir():f.chmod(0o444) +security.probe('tls',target='https://api:8080/health/ready',expect='deny',material='expired') +(ROOT/'evidence/cert-expiry.json').write_text(json.dumps({'trusted_ca':True,'allowed_lb_san':True,'expired_leaf_rejected':True},indent=2)+'\n');print('PASS: expired leaf with trusted CA and allowed LB SAN rejected') diff --git a/scripts/prove-clean-clone.py b/scripts/prove-clean-clone.py new file mode 100644 index 0000000..47242e5 --- /dev/null +++ b/scripts/prove-clean-clone.py @@ -0,0 +1,31 @@ +#!/usr/bin/env python3 +"""Exercise a source-only snapshot in a fresh project, with new CA/secrets/volumes and distinct ports.""" +import json,os,shutil,subprocess,tempfile,time +from pathlib import Path +ROOT=Path(__file__).resolve().parents[1] +files=subprocess.check_output(['git','ls-files','--cached','--others','--exclude-standard','-z'],cwd=ROOT).decode().split('\0') +with tempfile.TemporaryDirectory(prefix='lab7-clean-') as work: + target=Path(work) + for name in files: + p=ROOT/name + if not name or not p.is_file() or name.startswith('evidence/'):continue + out=target/name;out.parent.mkdir(parents=True,exist_ok=True);shutil.copy2(p,out) + (target/'evidence').mkdir(exist_ok=True) + image='mtusi-messenger:lab7-clean-'+str(os.getpid()) + cfg=json.loads((target/'compose.yaml').read_text()) + for service in cfg['services'].values(): + if service.get('image')=='mtusi-messenger:lab7':service['image']=image + for role,port in [('lb',18443),('openbao',18201),('grafana',13444)]: + current=cfg['services'][role]['ports'][0].split(':');cfg['services'][role]['ports']=['127.0.0.1:'+str(port)+':'+current[-1]] + (target/'compose.yaml').write_text(json.dumps(cfg)) + env=dict(os.environ,LAB7_PROJECT='mtusi2026lab7-clean-'+str(os.getpid()),LAB7_BASE_URL='https://localhost:18443',LAB7_BAO_OPERATOR_ADDR='https://localhost:18201') + started=time.monotonic() + try: + subprocess.run(['./scripts/up.sh'],cwd=target,env=env,check=True,stdout=subprocess.PIPE,stderr=subprocess.PIPE) + smoke=subprocess.run(['python3','tests/smoke.py','--base-url',env['LAB7_BASE_URL'],'--ca-file',str(target/'secrets/lab7/lb/ca.crt')],cwd=target,env=env,check=True,text=True,stdout=subprocess.PIPE,stderr=subprocess.PIPE) + subprocess.run(['python3','tests/security.py','--base-url',env['LAB7_BASE_URL'],'--ca-file',str(target/'secrets/lab7/lb/ca.crt')],cwd=target,env=env,check=True,stdout=subprocess.PIPE,stderr=subprocess.PIPE) + result={'fresh_source_snapshot':True,'new_ca_and_secrets':True,'new_project_and_volumes':True,'separate_application_image_tag':True,'no_lab6_access':True,'verified_https_smoke':smoke.stdout.strip(),'duration_seconds':time.monotonic()-started} + (ROOT/'evidence/clean-clone.json').write_text(json.dumps(result,indent=2)+'\n');print('PASS: fresh source-only snapshot with new secrets/volumes passes HTTPS smoke') + finally: + subprocess.run(['./scripts/compose.sh','down','-v'],cwd=target,env=env,stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL) + subprocess.run(['docker','image','rm',image],stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL) diff --git a/scripts/prove-leaf.py b/scripts/prove-leaf.py new file mode 100644 index 0000000..69b4e35 --- /dev/null +++ b/scripts/prove-leaf.py @@ -0,0 +1,11 @@ +#!/usr/bin/env python3 +import json,subprocess,sys +from pathlib import Path +ROOT=Path(__file__).resolve().parents[1];sys.path.insert(0,str(ROOT/'scripts'));import security +before=json.loads(subprocess.check_output(['docker','inspect',security.compose('ps','-q','lb').stdout.strip()],text=True))[0]['Image'] +subprocess.run([sys.executable,str(ROOT/'scripts/renew-leaf.py'),'lb'],check=True) +subprocess.run([sys.executable,str(ROOT/'scripts/wait-ready.py')],check=True) +security.probe('tls',target='https://api:8080/health/ready',material='lb-old',expect='deny') +security.probe('tls',target='https://api:8080/health/ready',material='lb') +after=json.loads(subprocess.check_output(['docker','inspect',security.compose('ps','-q','lb').stdout.strip()],text=True))[0]['Image'];assert before==after +(ROOT/'evidence/leaf-rotation.json').write_text(json.dumps({'image_unchanged':True,'old_leaf_rejected':True,'new_leaf_accepted':True,'verified_https_ready':True},indent=2)+'\n');print('PASS: leaf rotation without image rebuild and old-serial revocation') diff --git a/scripts/prove-sealed.py b/scripts/prove-sealed.py new file mode 100644 index 0000000..dfe52fc --- /dev/null +++ b/scripts/prove-sealed.py @@ -0,0 +1,23 @@ +#!/usr/bin/env python3 +"""Bounded short-token test: sealed Bao cannot silently refresh application credentials.""" +import json,subprocess,sys,time,urllib.error +from pathlib import Path +ROOT=Path(__file__).resolve().parents[1];sys.path.insert(0,str(ROOT/'scripts'));import bao,security + +def main(): + token=bao.operator_token();normal=bao.call('auth/cert/certs/api',token=token)['data'];short=dict(normal,token_ttl=30,token_max_ttl=30) + bao.call('auth/cert/certs/api',short,token) + security.compose('restart','api');subprocess.run([sys.executable,str(ROOT/'scripts/wait-ready.py')],check=True,stdout=subprocess.DEVNULL) + started=time.monotonic() + try: + bao.call('sys/seal',{},token) + for _ in range(55): + status,_,_=security.request('/health/ready') + if status==503:break + time.sleep(1) + else:raise AssertionError('API served beyond short grant lifetime while Bao sealed') + elapsed=time.monotonic()-started;assert elapsed<=45 + finally: + bao.unseal();token=bao.operator_token();bao.call('auth/cert/certs/api',normal,token);security.compose('restart','api');subprocess.run([sys.executable,str(ROOT/'scripts/wait-ready.py')],check=True,stdout=subprocess.DEVNULL) + (ROOT/'evidence/sealed.json').write_text(json.dumps({'short_token_ttl_seconds':30,'fail_closed_seconds':elapsed,'recovered_after_operator_unseal':True,'plaintext_fallback':False},indent=2)+'\n');print('PASS: sealed OpenBao produces bounded fail-closed and recovers after operator unseal') +if __name__=='__main__':main() diff --git a/scripts/prove-telemetry.py b/scripts/prove-telemetry.py new file mode 100644 index 0000000..8410704 --- /dev/null +++ b/scripts/prove-telemetry.py @@ -0,0 +1,28 @@ +#!/usr/bin/env python3 +"""Verify real signal storage through TLS and scan captured telemetry for synthetic secret canaries.""" +import json,re,sys,time,uuid +from pathlib import Path +ROOT=Path(__file__).resolve().parents[1];sys.path.insert(0,str(ROOT/'scripts'));import security + +def main(): + name,password,token,user=security.actor();auth={'Authorization':'Bearer '+token};canary='body-canary-'+uuid.uuid4().hex + status,_,b=security.request('/api/v1/chats','POST',json.dumps({'title':'Synthetic','member_ids':[user]}).encode(),dict(auth,**{'Content-Type':'application/json'}));assert status==201;chat=json.loads(b)['id'] + status,_,_=security.request('/api/v1/chats/'+chat+'/messages','POST',json.dumps({'text':canary}).encode(),dict(auth,**{'Content-Type':'application/json','Idempotency-Key':uuid.uuid4().hex}));assert status==201 + # A fresh upload produces a real worker trace after any backend restart. + fixture=(ROOT/'tests/fixtures/sample.png').read_bytes();status,_,b=security.request('/api/v1/chats/'+chat+'/attachments?filename=signals.png','POST',fixture,dict(auth,**{'Content-Type':'image/png'}));assert status==202;aid=json.loads(b)['id'] + for _ in range(60): + if json.loads(security.request('/api/v1/attachments/'+aid,headers=auth)[2]).get('status')=='ready':break + time.sleep(1) + else:raise AssertionError('telemetry upload did not finish') + logs=security.compose('logs','--no-color','--tail=80','worker').stdout + trace=re.findall(r'"trace_id":"([a-f0-9]{32})"',logs)[-1] + time.sleep(8) + data=security.compose('run','--rm','-T','--no-deps','--entrypoint','securityprobe','-e','BAO_ADDR=','-e','PROBE_MODE=telemetry','-e','PROBE_TRACE_ID='+trace,'-v',str(security.S/'grafana')+':/run/identity:ro','worker').stdout + signals=json.loads(data);assert signals['metrics']['data']['result'];assert signals['logs']['data']['result'];assert signals['traces'].get('batches') or signals['traces'].get('resourceSpans') + captured=security.compose('logs','--no-color').stdout+data + needles=[password,token,canary,'BEGIN PRIVATE KEY','BEGIN RSA PRIVATE KEY'] + for p in (security.S/'operator').iterdir(): + if p.is_file() and (p.name.endswith('_secret') or p.name.endswith('_password')):needles.append(p.read_text().strip()) + assert not any(v and v in captured for v in needles),'secret or body found in telemetry' + (ROOT/'evidence/telemetry.json').write_text(json.dumps({'signals':signals,'secret_canaries_absent':True,'checked_container_logs':True,'checked_backend_sample':True,'limitations':'Checks cover captured logs and sampled backend signals, not a mathematical proof for every historical span.'},indent=2)+'\n');print('PASS: metrics, worker logs and real trace queried over TLS; secret/body canaries absent') +if __name__=='__main__':main() diff --git a/scripts/renew-leaf.py b/scripts/renew-leaf.py new file mode 100644 index 0000000..6d7a8a6 --- /dev/null +++ b/scripts/renew-leaf.py @@ -0,0 +1,18 @@ +#!/usr/bin/env python3 +"""Operator leaf rotation without image rebuild; root CA remains offline from services.""" +import shutil,sys +from pathlib import Path +sys.path.insert(0,str(Path(__file__).resolve().parent));import bootstrap,bao,security +role=sys.argv[1] if len(sys.argv)>1 else 'lb' +if role not in ['lb','api','dispatcher']:raise SystemExit('Use lb/api/dispatcher; worker incident response uses rotate-worker.py') +s=bao.S;o=bao.O;old=s/(role+'-old') +if old.exists():shutil.rmtree(old) +shutil.copytree(s/role,old);old.chmod(0o755) +token=bao.operator_token() +bootstrap.run('openssl','ca','-config',str(o/'ca.cnf'),'-revoke',str(old/'tls.crt')) +for f in ['tls.crt','tls.key']:(s/role/f).unlink() +bootstrap.issue(role);bootstrap.main() +for p in old.iterdir():p.chmod(0o444) +if role!='lb':bao.call('auth/cert/certs/'+role,{'certificate':(s/role/'tls.crt').read_text(),'token_policies':[role],'token_ttl':300,'token_max_ttl':1800,'token_no_default_policy':True},token) +security.compose('restart',role) +print('Leaf replaced and old serial revoked without rebuilding the image:',role) diff --git a/scripts/restart-evidence.py b/scripts/restart-evidence.py new file mode 100644 index 0000000..529dbcd --- /dev/null +++ b/scripts/restart-evidence.py @@ -0,0 +1,14 @@ +#!/usr/bin/env python3 +import json,subprocess,sys +from pathlib import Path +root=Path(__file__).resolve().parents[1];path=root/'evidence/restart-evidence.json' +phase=sys.argv[1];out={} +for cid in subprocess.check_output([str(root/'scripts/compose.sh'),'ps','-q'],text=True).split(): + c=json.loads(subprocess.check_output(['docker','inspect',cid],text=True))[0] + out[c['Name']]={'id':c['Id'],'started_at':c['State']['StartedAt'],'volumes':[{'name':m.get('Name'),'destination':m['Destination']} for m in c['Mounts'] if m['Type']=='volume']} +data=json.loads(path.read_text()) if phase=='after' and path.exists() else {};data[phase]=out +if phase=='after': + required=['postgres','redis','broker','s3','openbao','prometheus','loki','tempo','grafana'] + data['all_stores_restarted']=all(any('-'+role+'-' in name and value['started_at']!=data['before'].get(name,{}).get('started_at') for name,value in out.items()) for role in required) + assert data['all_stores_restarted'] +path.write_text(json.dumps(data,indent=2)+'\n') diff --git a/scripts/restart.sh b/scripts/restart.sh new file mode 100755 index 0000000..36fcbd4 --- /dev/null +++ b/scripts/restart.sh @@ -0,0 +1,9 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +python3 scripts/restart-evidence.py before +./scripts/compose.sh restart postgres redis broker s3 otel-collector prometheus loki tempo grafana openbao +python3 scripts/bao.py unseal +./scripts/compose.sh up -d --force-recreate --no-deps s3-tls otel-collector-tls prometheus-tls loki-tls tempo-tls +./scripts/compose.sh restart api worker dispatcher lb +python3 scripts/restart-evidence.py after diff --git a/scripts/rotate-worker.py b/scripts/rotate-worker.py new file mode 100644 index 0000000..417e99e --- /dev/null +++ b/scripts/rotate-worker.py @@ -0,0 +1,41 @@ +#!/usr/bin/env python3 +"""Operator incident response: revoke old worker identity, rotate its S3 key and leaf, restore.""" +import importlib.util,json,os,secrets,shutil,subprocess,time,sys +from pathlib import Path +sys.path.insert(0,str(Path(__file__).resolve().parent));import bao,bootstrap +S=bao.S;O=bao.O;ROOT=bao.ROOT + +def compose(*args):subprocess.run([str(ROOT/'scripts/compose.sh'),*args],check=True,stdout=subprocess.DEVNULL,stderr=subprocess.PIPE) +def rotate(start=True): + token=bao.operator_token();old=S/'worker-old' + if old.exists():shutil.rmtree(old) + shutil.copytree(S/'worker',old);old.chmod(0o755) + for source,target in [('s3_worker_access','old_access'),('s3_worker_secret','old_secret')]: + (old/target).write_text((O/source).read_text());(old/target).chmod(0o444) + oldcn=subprocess.check_output(['openssl','x509','-in',str(old/'tls.crt'),'-noout','-subject','-nameopt','RFC2253'],text=True).strip().split('CN=')[-1] + # Revoke service auth, already-issued tokens and dynamic DB leases, including established DB connections. + bao.call('auth/cert/certs/worker',token=token,method='DELETE') + accessors=bao.call('auth/token/accessors',token=token,method='LIST').get('data',{}).get('keys',[]) + for accessor in accessors: + info=bao.call('auth/token/lookup-accessor',{'accessor':accessor},token)['data'] + if 'worker' in info.get('policies',[]):bao.call('auth/token/revoke-accessor',{'accessor':accessor},token) + bao.call('sys/leases/revoke-prefix/database/creds/worker',{},token) + compose('exec','-T','broker','rabbitmqctl','delete_user',oldcn) + compose('stop','worker') + bootstrap.run('openssl','ca','-config',str(O/'ca.cnf'),'-revoke',str(old/'tls.crt')) + for name in ['s3_worker_access','s3_worker_secret']: + p=O/name;p.chmod(0o600);p.write_text(secrets.token_hex(24)) + for f in ['tls.key','tls.crt']:(S/'worker'/f).unlink() + bootstrap.issue('worker',cn='worker-'+str(time.time_ns())) + # Reuses existing CA and storage, publishes new CRL and broker user mapping. + bootstrap.main() + for p in old.iterdir():p.chmod(0o444) + compose('restart','s3') + compose('up','-d','--force-recreate','--no-deps','s3-tls','otel-collector-tls') + compose('exec','-T','broker','rabbitmqctl','import_definitions','/run/identity/definitions.json') + bao.call('auth/cert/certs/worker',{'certificate':(S/'worker/tls.crt').read_text(),'token_policies':['worker'],'token_ttl':300,'token_max_ttl':1800,'token_no_default_policy':True},token) + bao.call('secret/data/worker',{'data':{'s3_access':(O/'s3_worker_access').read_text(),'s3_secret':(O/'s3_worker_secret').read_text()}},token) + if start:compose('up','-d','worker') + print('Worker old cert/token/DB lease revoked; S3 key and leaf replaced; image unchanged.') + return oldcn +if __name__=='__main__':rotate() diff --git a/scripts/scan-images.py b/scripts/scan-images.py new file mode 100644 index 0000000..a25d687 --- /dev/null +++ b/scripts/scan-images.py @@ -0,0 +1,15 @@ +#!/usr/bin/env python3 +"""Scan final runtime image without giving any container a Docker socket.""" +import collections,json,subprocess,tempfile +from pathlib import Path +ROOT=Path(__file__).resolve().parents[1] +with tempfile.TemporaryDirectory(prefix='lab7-scan-') as work: + image_id=subprocess.check_output(['docker','image','inspect','mtusi-messenger:lab7','--format','{{.Id}}'],text=True).strip() + archive=Path(work)/'image.tar';cache=ROOT/'.cache/trivy';cache.mkdir(parents=True,exist_ok=True) + subprocess.run(['docker','image','save','-o',str(archive),image_id],check=True) + r=subprocess.run(['docker','run','--rm','-v',str(cache)+':/root/.cache/trivy','-v',str(archive)+':/scan/image.tar:ro','aquasec/trivy:0.67.2','image','--quiet','--scanners','vuln','--format','json','--input','/scan/image.tar'],check=True,stdout=subprocess.PIPE,text=True) + data=json.loads(r.stdout);summary=[] + for target in data.get('Results',[]): + vulns=target.get('Vulnerabilities',[]);summary.append({'target':target['Target'],'counts':dict(collections.Counter(v['Severity'] for v in vulns)),'findings':[{'id':v['VulnerabilityID'],'package':v['PkgName'],'installed':v['InstalledVersion'],'fixed':v.get('FixedVersion'),'severity':v['Severity'],'url':v.get('PrimaryURL')} for v in vulns]}) + (ROOT/'evidence/image-scan.json').write_text(json.dumps({'scanner':'aquasec/trivy:0.67.2','image_id':image_id,'results':summary},indent=2)+'\n') + print('Runtime image scan recorded; applicability is discussed in security/SCAN_TRIAGE.md.') diff --git a/scripts/security.py b/scripts/security.py new file mode 100644 index 0000000..39e0ef5 --- /dev/null +++ b/scripts/security.py @@ -0,0 +1,76 @@ +#!/usr/bin/env python3 +"""Real negative TLS, identity, authorization and public security checks.""" +import base64,json,os,ssl,subprocess,sys,time,urllib.request,urllib.error,uuid +from pathlib import Path +ROOT=Path(__file__).resolve().parents[1];sys.path.insert(0,str(ROOT/'scripts'));import bao +S=ROOT/'secrets/lab7';results=[] +def compose(*args,check=True):return subprocess.run([str(ROOT/'scripts/compose.sh'),*args],cwd=ROOT,check=check,text=True,stdout=subprocess.PIPE,stderr=subprocess.PIPE) +def record(name,details=True):results.append({'scenario':name,'passed':True,'details':details});print('PASS:',name) +def probe(mode,role='worker',target='',expect='allow',no_cert=False,server_name='',material=None,method='',untrusted=False): + args=['run','--rm','-T','--no-deps','--entrypoint','securityprobe','-e','PROBE_MODE='+mode,'-e','PROBE_TARGET='+target,'-e','PROBE_EXPECT='+expect,'-e','PROBE_NO_CERT='+('1'if no_cert else'0'),'-e','PROBE_SERVER_NAME='+server_name,'-e','PROBE_METHOD='+method,'-e','PROBE_UNTRUST_SERVER='+('1' if untrusted else '0')] + if mode in ['tls','amqp-tls','db-tls']:args+=['-e','BAO_ADDR='] + if material:args+=['-v',str(S/material)+':/run/identity:ro'] + r=compose(*args,role);lines=[x for x in r.stdout.splitlines() if x.startswith('{')];return json.loads(lines[-1]) +def request(path,method='GET',body=None,headers=None): + ctx=ssl.create_default_context(cafile=str(S/'lb/ca.crt'));req=urllib.request.Request(os.environ.get('LAB7_BASE_URL','https://localhost:8443')+path,data=body,method=method,headers=headers or {}) + try: + with urllib.request.urlopen(req,context=ctx,timeout=10) as r:return r.status,dict(r.headers),r.read() + except urllib.error.HTTPError as e:return e.code,dict(e.headers),e.read() +def actor(): + name='sec_'+uuid.uuid4().hex[:16];password='Synthetic-'+uuid.uuid4().hex + status,_,b=request('/api/v1/users','POST',json.dumps({'username':name,'display_name':'Synthetic','password':password}).encode(),{'Content-Type':'application/json'});assert status==201,status + auth='Basic '+base64.b64encode((name+':'+password).encode()).decode();status,_,b=request('/api/v1/auth/sessions','POST',headers={'Authorization':auth});assert status==201,status + a=json.loads(b);return name,password,a['token'],a['user']['id'] +def main(): + # Inspect only container addresses; never export environment/secrets. + for index,cid in enumerate(compose('ps','-q','api').stdout.split(),1): + inspection=json.loads(subprocess.check_output(['docker','inspect',cid],text=True))[0] + address=next(v['IPAddress'] for v in inspection['NetworkSettings']['Networks'].values() if v['IPAddress']) + target='https://'+address+':8080/health/ready' + record(f'API replica {index} accepts LB',probe('tls',target=target,material='lb',server_name='api')) + for label,options in [('no certificate',{'no_cert':True,'material':'lb'}),('same CA wrong identity',{'material':'stranger'}),('foreign CA',{'material':'foreign'})]:record(f'API replica {index} rejects '+label,probe('tls',target=target,expect='deny',server_name='api',**options)) + record('API server hostname checked',probe('tls',target='https://api:8080/health/ready',expect='deny',server_name='wrong-host',material='lb')) + record('DB TLS verify-full',probe('db-tls')) + record('DB rejects wrong hostname',probe('db-tls',expect='deny',server_name='wrong-host')) + record('DB rejects untrusted server CA',probe('db-tls',expect='deny',untrusted=True)) + for role in ['api','worker','dispatcher']: + record('Collector accepts '+role,probe('tls',role,'https://otel-collector:9443/v1/traces')) + for material,none in [('stranger',False),('foreign',False),('worker',True)]:record('Collector denies '+material+(' no-cert'if none else''),probe('tls',target='https://otel-collector:9443/v1/traces',expect='deny',material=material,no_cert=none)) + for material,none in [('stranger',False),('foreign',False),('worker',True)]:record('Broker denies '+material+(' no-cert'if none else''),probe('amqp-tls',expect='deny',material=material,no_cert=none)) + for material,none in [('stranger',False),('foreign',False),('worker',True)]:record('S3 denies unauthenticated identity '+material,probe('tls',target='https://s3:9443/messenger-attachments/',expect='deny',material=material,no_cert=none)) + for mode in ['db','s3','amqp']:record('Worker least privilege '+mode,probe(mode)) + record('Redis scoped ACL',probe('redis',role='api')) + for target,material,method in [ + ('https://otel-collector:9444/metrics','prometheus','GET'), + ('https://otel-collector:9445/metrics','prometheus','GET'), + ('https://prometheus:9443/api/v1/query?query=up','grafana','GET'), + ('https://loki:9443/ready','grafana','GET'), + ('https://tempo:9443/ready','grafana','GET'), + ('https://loki:9443/otlp/v1/logs','otel-collector','POST'), + ('https://tempo:9444/v1/traces','otel-collector','POST'), + ('https://redis:6380/','api','GET')]: + if 'redis:' in target:continue # Redis uses a binary protocol; scoped ACL probe verifies its TLS client. + record('Matrix allowed '+target,probe('tls',target=target,material=material,method=method)) + record('Matrix rejects unknown identity '+target,probe('tls',target=target,expect='deny',material='stranger',method=method)) + record('Grafana cannot ingest Loki logs',probe('tls',target='https://loki:9443/otlp/v1/logs',material='grafana',method='POST',expect='deny')) + record('Grafana cannot use Tempo ingest',probe('tls',target='https://tempo:9444/v1/traces',material='grafana',method='POST',expect='deny')) + record('Collector cannot query Loki',probe('tls',target='https://loki:9443/loki/api/v1/labels',material='otel-collector',expect='deny')) + token=bao.call('auth/cert/login',{'name':'worker'},identity='worker')['auth']['client_token'] + for path,body in [('secret/data/api',None),('secret/data/operator',None),('sys/policies/acl/escape',{'policy':'path "*" { capabilities = ["read"] }'})]: + try:bao.call(path,body,token,identity='worker') + except urllib.error.HTTPError as e:assert e.code==403 + else:raise AssertionError('OpenBao privilege escalation') + record('OpenBao worker policy allow/deny') + name,password,token,user=actor();header={'Authorization':'Bearer '+token} + for _ in range(11):status,h,b=request('/api/v1/auth/sessions','POST',headers={'Authorization':'Basic '+base64.b64encode((name+':wrong-password-value').encode()).decode()}) + assert status==429 and int({k.lower():v for k,v in h.items()}['retry-after'])>0 + record('Shared login rate limit returns 429 Retry-After') + status,_,_=request('/api/v1/users','POST',b'{"padding":"'+b'x'*(1<<20)+b'"}',{'Content-Type':'application/json'});assert status==413,status;record('JSON body limit') + status,_,_=request('/api/v1/auth/sessions/current','DELETE',headers=header);assert status==204 + assert request('/api/v1/users/me',headers=header)[0]==401;record('Logout revocation') + # Validate that direct backend listeners are loopback-only, not reachable over the bridge. + for host,port in [('s3',9000),('otel-collector',4318),('loki',3100),('tempo',3200),('prometheus',9090)]: + r=compose('run','--rm','-T','--no-deps','--entrypoint','wget','worker','-T','2','-q','-O','/dev/null',f'http://{host}:{port}/',check=False);assert r.returncode!=0 + record('Plaintext backend bypasses unreachable') + (ROOT/'evidence/security.json').write_text(json.dumps({'timestamp':time.strftime('%Y-%m-%dT%H:%M:%SZ',time.gmtime()),'results':results},indent=2)+'\n') +if __name__=='__main__':main() diff --git a/scripts/stop-one.py b/scripts/stop-one.py new file mode 100644 index 0000000..3037bf9 --- /dev/null +++ b/scripts/stop-one.py @@ -0,0 +1,14 @@ +#!/usr/bin/env python3 +import json,os,subprocess +from pathlib import Path +c=str(Path(__file__).resolve().parent/'compose.sh') +target=os.environ['TARGET_INSTANCE_ID'] +ids=subprocess.check_output([c,'ps','-q','api'],text=True).split() +for cid in ids: + logs=subprocess.check_output(['docker','logs',cid],stderr=subprocess.STDOUT,text=True) + if any('instance_started'in line and target in line for line in logs.splitlines()): + subprocess.run(['docker','stop',cid],check=True,stdout=subprocess.DEVNULL) + evidence=Path(__file__).resolve().parents[1]/'evidence/failover-targets.json' + records=json.loads(evidence.read_text()) if evidence.exists() else [] + records.append({'instance_alias':target,'container_id':cid});evidence.write_text(json.dumps(records,indent=2)+'\n');break +else:raise SystemExit('instance alias not found') diff --git a/scripts/stop-one.sh b/scripts/stop-one.sh new file mode 100755 index 0000000..b222513 --- /dev/null +++ b/scripts/stop-one.sh @@ -0,0 +1,4 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +python3 scripts/stop-one.py diff --git a/scripts/summarize-evidence.py b/scripts/summarize-evidence.py new file mode 100644 index 0000000..0a2dc33 --- /dev/null +++ b/scripts/summarize-evidence.py @@ -0,0 +1,23 @@ +#!/usr/bin/env python3 +"""Require concrete outputs before publishing the local verification summary.""" +import datetime,hashlib,json,subprocess +from pathlib import Path +ROOT=Path(__file__).resolve().parents[1];E=ROOT/'evidence' +checks=[('Public HTTPS and external trust','public-checks.txt'),('Final HTTPS smoke','final-smoke.txt'),('TLS/identity/ACL matrix','security-run.txt'),('Argon2/rehash/session expiry','auth-run.txt'),('Leaf rotation/revocation','leaf-run.txt'),('Expired allowed-SAN leaf','cert-expiry-run.txt'),('Sealed OpenBao','sealed-run.txt'),('Real telemetry and canaries','telemetry-run.txt'),('Grafana authentication','admin-run.txt'),('Image/ACL negatives','images-run.txt'),('Staging/restore/retention','storage-run.txt'),('Broker volume durability','broker-run.txt'),('All stores restart/session persistence','persistence-run.txt'),('Public failover 1','failover-first.txt'),('Public failover 2','failover-second.txt'),('Both physical replicas','all-replicas-run.txt'),('Worker compromise/recovery','worker-experiment-run.txt'),('Independent source snapshot','clean-clone-run.txt')] +for _,filename in checks: + text=(E/filename).read_text();assert 'PASS:' in text and 'Traceback' not in text and 'FAIL:' not in text,filename +worker=json.loads((E/'worker-experiment.json').read_text());assert len(worker['repeats'])==3 and all(r['lost']==0 and r['old_s3_key_rejected'] and r['old_broker_identity_rejected'] for r in worker['repeats']) +assert json.loads((E/'secret-audit.json').read_text())['matches']==0 +scan=json.loads((E/'image-scan.json').read_text());assert all(not r['findings'] for r in scan['results']) +assert 'No vulnerabilities found.' in (E/'govulncheck.txt').read_text() +manifest={} +for name in subprocess.check_output(['git','ls-files','--cached','--others','--exclude-standard','-z'],cwd=ROOT).decode().split('\0'): + p=ROOT/name + if name and p.is_file() and not name.startswith('evidence/'):manifest[name]=hashlib.sha256(p.read_bytes()).hexdigest() +(E/'source-manifest.json').write_text(json.dumps(manifest,sort_keys=True,indent=2)+'\n') +lines=['# Фактическая итоговая проверка','',datetime.datetime.now(datetime.timezone.utc).isoformat(),'','Результаты относятся к synthetic lab7 stand. Source snapshot: `source-manifest.json`; environment/image IDs: `environment.json` и `image-scan.json`.','','| Проверка | Результат | Вывод |','| --- | --- | --- |'] +for label,filename in checks:lines.append(f'| {label} | PASS | [{filename}]({filename}) |') +lines+=['| Source vulnerability scan | 0 findings | [govulncheck.txt](govulncheck.txt) |','| Application image scan | 0 findings | [image-scan.json](image-scan.json) |','','## Worker experiment',''] +for r in worker['repeats']:lines.append(f"- Повтор {r['repeat']}: {r['accepted']} accepted → {r['ready']} ready, lost={r['lost']}, duplicate_jobs={r['duplicate_jobs']}, recovery={r['recovery_seconds']:.3f} s; старые S3/broker/Bao identities отвергнуты.") +lines+=['','Скриншоты не используются вместо команд. GitHub CI настроен, удалённый workflow не запускался агентом. Диагностические неудачные запуски не включены в PASS-таблицу.'] +(E/'verification-summary.md').write_text('\n'.join(lines)+'\n');print('All required evidence verified; summary and source hashes written.') diff --git a/scripts/test-storage.sh b/scripts/test-storage.sh new file mode 100755 index 0000000..9cb8307 --- /dev/null +++ b/scripts/test-storage.sh @@ -0,0 +1,11 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +mkdir -p bin +(cd service && GOOS=linux GOARCH=$(docker info --format '{{.Architecture}}' | sed 's/aarch64/arm64/;s/x86_64/amd64/') CGO_ENABLED=0 go test -c -o ../bin/storage.test ./internal/adapter/repository/storage) +database="lab5_atomic_$(date +%s)_$$" +./scripts/compose.sh exec -T postgres psql -U messenger -d messenger -v ON_ERROR_STOP=1 -c "CREATE DATABASE $database" >/dev/null +trap './scripts/compose.sh exec -T postgres psql -U messenger -d messenger -c "DROP DATABASE IF EXISTS $database WITH (FORCE)" >/dev/null' EXIT +id=$(./scripts/compose.sh ps -q api | head -n 1) +docker cp bin/storage.test "$id:/tmp/storage.test" >/dev/null +docker exec -e "TEST_DATABASE_NAME=$database" "$id" /tmp/storage.test -test.v diff --git a/scripts/up.sh b/scripts/up.sh new file mode 100755 index 0000000..bf2b38b --- /dev/null +++ b/scripts/up.sh @@ -0,0 +1,14 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +python3 scripts/bootstrap.py +python3 scripts/build-image.py +./scripts/compose.sh run --rm -T bao-permissions +./scripts/compose.sh up -d --wait --wait-timeout 120 postgres +./scripts/compose.sh up -d postgres redis broker s3 s3-tls openbao otel-collector otel-collector-tls prometheus prometheus-tls loki loki-tls tempo tempo-tls grafana +python3 scripts/bao.py +./scripts/compose.sh run --rm -T migrate +./scripts/compose.sh exec -T postgres psql -U messenger -d messenger -v ON_ERROR_STOP=1 < infra/postgres/grants.sql +./scripts/compose.sh run --rm -T operator storage bootstrap +./scripts/compose.sh up -d api worker dispatcher lb +python3 scripts/wait-ready.py diff --git a/scripts/wait-ready.py b/scripts/wait-ready.py new file mode 100644 index 0000000..e2215e1 --- /dev/null +++ b/scripts/wait-ready.py @@ -0,0 +1,12 @@ +#!/usr/bin/env python3 +import ssl,time,urllib.request,os +from pathlib import Path +root=Path(__file__).resolve().parents[1] +ctx=ssl.create_default_context(cafile=str(root/'secrets/lab7/lb/ca.crt')) +for _ in range(90): + try: + with urllib.request.urlopen(os.environ.get('LAB7_BASE_URL','https://localhost:8443')+'/health/ready',context=ctx,timeout=2) as r: + if r.status==200:print('PASS: verified HTTPS readiness');break + except OSError:pass + time.sleep(1) +else:raise SystemExit('HTTPS readiness timeout') diff --git a/security/SCAN_TRIAGE.md b/security/SCAN_TRIAGE.md new file mode 100644 index 0000000..d833b62 --- /dev/null +++ b/security/SCAN_TRIAGE.md @@ -0,0 +1,20 @@ +# Dependency и image scan + +Использованы `govulncheck v1.1.4` и `aquasec/trivy:0.67.2`. Source scan выполняется по всем Go packages; image scan проверяет OS packages и оба Go binaries конечного application image. Scanner не получает Docker socket: образ экспортируется оператором, scanner читает archive read-only. Image ID фиксируется до экспорта. + +До исправлений source scan находил 23 достижимых findings стандартной библиотеки Go 1.27.0 и amqp091-go 1.10.0. Исходный вывод сохранён в `evidence/govulncheck-before.txt`. Эти компоненты участвуют в HTTP/TLS/AMQP paths и не были отклонены как неприменимые. Runtime image scan также обнаружил исправляемые musl/zlib findings. + +Исправления: + +| Компонент | Итоговая версия | Применимость и действие | +| --- | --- | --- | +| Go | 1.27.4 | HTTP/TLS/X.509 достижимы, toolchain обновлён, insecure fallback не используется | +| amqp091-go | 1.14.0 | Broker client используется API pipeline, обновлён вместо подавления findings | +| x/crypto | 0.59.0 | Используется Argon2id; обновлён также модуль целиком, даже при недостижимости других функций | +| x/net | 0.62.0 | Transitive HTTP dependencies, обновлён вместо исключения package-level findings | +| musl / musl-utils | 1.2.5-r24 | Runtime packages обновлены и закреплены явно | +| zlib | 1.4-r0 | ImageMagick обрабатывает недоверенные изображения, обновление применено | + +Итоговый source output — `evidence/govulncheck.txt`, image findings и точный image ID — `evidence/image-scan.json`. Отсутствие findings относится только к проверенной базе scanner на дату запуска и его scope. Оно не доказывает полную безопасность приложения. Остальные infrastructure images закреплены версиями/digests, но этот отчёт не выдаёт scan application image за отдельный анализ каждого стороннего infrastructure image. + +Остаточные риски: разрешённые worker операции могут повредить его prefixes/metadata; компрометация operator host вне модели; single-host storage и ручной PKI renewal не являются production HA/PKI. Эти риски не CVE и не исчезают при нулевом результате scanner. diff --git a/security/THREAT_MODEL.md b/security/THREAT_MODEL.md index 31da287..f0a8c59 100644 --- a/security/THREAT_MODEL.md +++ b/security/THREAT_MODEL.md @@ -1,19 +1,32 @@ -# Модель угроз — заполнить +# Модель угроз — lab7 -## Активы и акторы +Автор: Поликарпов Даниил Александрович, БПИ2303. -TODO: пользовательские credentials/сессии, сообщения/вложения, service identities, CA/secrets, telemetry. Акторы: участник чата, посторонний пользователь, внешний клиент, скомпрометированный сервис, оператор. Опишите доступ атакующего и границы учебного стенда. +Активы: пароли и session fingerprints, сообщения, приватные оригиналы/варианты, durable jobs/outbox, service credentials, PKI и telemetry. Акторы: внешний клиент, участник чата, посторонний пользователь, скомпрометированный worker и оператор. Под компрометацией worker понимается возможность читать его identity/credentials и обращаться к зависимостям с его правами. Контроль Docker host, CA и оператора находится вне модели. -## Границы доверия +Границы и реальные endpoints перечислены в [trust matrix](../contracts/trust-matrix.md). Внешний REST-клиент проходит TLS → LB → mTLS → API. API и background processes обращаются к dependencies через TLS; credentials получают из OpenBao. PostgreSQL — источник истины, S3 — байты, RabbitMQ — at-least-once delivery. Авторизация пользователей и переходы pipeline находятся в service; repository сохраняет результаты. -TODO: схема и ссылка на заполненную contracts/trust-matrix.md; публичные/внутренние интерфейсы, процессы, секреты и их выдача. - -| Угроза и конкретный путь | Нарушаемый инвариант | Контроль | Отрицательный тест | Остаточное ограничение | +| Угроза и путь | Инвариант | Контроль | Отрицательный сценарий | Остаточное ограничение | | --- | --- | --- | --- | --- | -| TODO | TODO | TODO | TODO | TODO | - -На 4 минимум 6 конкретных угроз. Не заменяйте сценарии словами «SQL injection», «безопасный Docker», «mTLS включён» без пути и проверки. +| Посторонний подбирает chat/attachment UUID | Данные доступны только участникам | Bearer session и service ACL, чужое = 404 | Public smoke, images.py | Компрометация аккаунта даёт его законные права | +| Клиент подставляет X-User-Id или sender_id | Нельзя сменить identity | Проверенная сессия, строгий JSON | Public smoke | Кража действительного Bearer требует logout/expiry | +| Контейнер с сертификатом того же CA выдаёт себя за LB | CA не означает разрешение | DNS SAN allowlist lb и CRL | security.py: stranger, foreign, no cert | Компрометация LB входит в другую границу доверия | +| Подбор паролей нагружает KDF на двух репликах | Расход CPU/RAM ограничен | Общий Redis limit, 2 KDF slots/process, bounded hash parameters | security.py, prove-auth.py | Общий NAT/IP может ограничить легитимных пользователей | +| Большой/повреждённый upload истощает worker | Декодирование ограничено | Bytes/pixels/time limits, ImageMagick policy, CPU/RAM/tmpfs | images.py; resource configuration | Допустимое сложное изображение всё ещё расходует выделенный бюджет | +| Worker читает password/session hashes или меняет схему | Credentials пользователей вне его прав | Отдельная DB role без users/refresh_tokens/DDL | securityprobe db | Worker может менять разрешённые job/attachment поля | +| Worker пишет чужой S3 prefix или управляет broker topology | Blast radius ограничен | Prefix-scoped SigV4, producer/consumer ACL | securityprobe s3/amqp | Разрешённые results/staging доступны worker; он способен повредить их | +| Worker читает API secret или расширяет Bao policy | Сервис получает только свои secrets | Leaf-bound cert auth, scoped policy, короткий TTL | security.py allow/deny | Украденные текущие права сохраняются до отзыва/expiry | +| Отозванная identity использует кеш/старое соединение | Отзыв действует на фактический доступ | Token/lease revoke, DB session termination, удаление broker user, S3 key rotation, CRL, restart worker | experiment-worker.py ×3 | Одного изменения CA/сертификата недостаточно без закрытия соединений | +| Поддельный telemetry client пишет/читает чужие сигналы | Ingest/query разделены | SAN allowlists, loopback boundary, Envoy RBAC | security.py, prove-telemetry.py | Telemetry оператора содержит технические ID синтетических данных | +| Пароль/Bearer/body попадает в logs/traces | Credentials не экспортируются | Безопасные events и DB trace metadata, фильтрация attributes | prove-telemetry.py canaries | Выборочная проверка не доказывает отсутствие утечек во всех будущих изменениях | +| OpenBao sealed или grant истёк | Нет бесконечного использования stale credentials | Bounded renewals, процесс останавливается к expiry, restart требует auth | prove-sealed.py | Во время допустимого TTL приложение продолжает использовать действующие grants | ## Lifecycle и инциденты -TODO: bootstrap, expiry/renewal, revoke/rotation, компрометация identity, sealed OpenBao и восстановление. Указать владельца CA/unseal и способ защиты bootstrap credentials. Сопоставить scan findings с реальной применимостью и исправлением/остаточным риском. +Первоначальный bootstrap выполняет оператор. Secrets создаются с umask 077 в игнорируемом каталоге; каталог оператора 0700. Каждый контейнер монтирует только собственную identity. Внутри read-only identity mount файлы доступны его runtime UID; внешний родительский каталог остаётся закрытым для других пользователей host. + +Root token нужен только для начальной настройки и отзывается после неё. CA key и 2/3 unseal shares не попадают в приложение или образ. Хранение shares в одном защищённом каталоге — учебное доверие к операторскому компьютеру, не независимое распределение custodians. Для другой машины оператор переносит shares отдельным защищённым способом. + +При компрометации worker: отозвать cert auth и tokens, DB leases/соединения; удалить старого broker user; остановить worker; отозвать serial; ротировать S3 key и leaf; обновить CRL/proxies; создать новый broker CN user; восстановить worker и сверить jobs/checksums. Методика и границы находятся в REPORT.md. Подтверждена сохранность задач при этих отказах, но не после умышленной порчи разрешённых worker данных. + +Scan findings и исправления: [SCAN_TRIAGE.md](SCAN_TRIAGE.md). diff --git a/service/cmd/app/main.go b/service/cmd/app/main.go new file mode 100644 index 0000000..c7e1d32 --- /dev/null +++ b/service/cmd/app/main.go @@ -0,0 +1,194 @@ +package main + +import ( + "context" + "fmt" + "gitea.privcloud.tech/mtusi2026/internal/pkg/openbao" + "gitea.privcloud.tech/mtusi2026/internal/pkg/ratelimit" + "gitea.privcloud.tech/mtusi2026/internal/pkg/transportsecurity" + "log/slog" + "os" + "os/signal" + "syscall" + "time" + + "encoding/json" + "gitea.privcloud.tech/mtusi2026/internal/adapter/controller/http" + "gitea.privcloud.tech/mtusi2026/internal/adapter/repository/cache/outside" + "gitea.privcloud.tech/mtusi2026/internal/adapter/repository/storage" + "gitea.privcloud.tech/mtusi2026/internal/config" + "gitea.privcloud.tech/mtusi2026/internal/pkg/auth/jwtprovider" + "gitea.privcloud.tech/mtusi2026/internal/pkg/cache" + "gitea.privcloud.tech/mtusi2026/internal/pkg/encrypting" + "gitea.privcloud.tech/mtusi2026/internal/pkg/migrator" + "gitea.privcloud.tech/mtusi2026/internal/pkg/objectstore" + "gitea.privcloud.tech/mtusi2026/internal/pkg/postgres" + "gitea.privcloud.tech/mtusi2026/internal/pkg/postgres/txmanager" + "gitea.privcloud.tech/mtusi2026/internal/pkg/rest/middleware" + "gitea.privcloud.tech/mtusi2026/internal/pkg/rest/router" + "gitea.privcloud.tech/mtusi2026/internal/pkg/telemetry" + "gitea.privcloud.tech/mtusi2026/internal/pkg/validator" + "gitea.privcloud.tech/mtusi2026/internal/service/attachment" + "gitea.privcloud.tech/mtusi2026/internal/service/auth" + "gitea.privcloud.tech/mtusi2026/internal/service/chat" + "gitea.privcloud.tech/mtusi2026/internal/service/outbox" + "gitea.privcloud.tech/mtusi2026/internal/service/user" + stdhttp "net/http" +) + +func main() { + time.Local = time.UTC + if err := run(); err != nil { + slog.Error("application stopped", "error", err) + os.Exit(1) + } +} + +func run() error { + if os.Getenv("APPLICATION_ENVIRONMENT") == "prod" && !transportsecurity.Enabled() { + return fmt.Errorf("production TLS is required") + } + processCtx, stopProcess := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) + defer stopProcess() + ctx, stopSecrets, err := openbao.Connect(processCtx) + if err != nil { + return err + } + defer stopSecrets() + if err = encrypting.ValidateProfile(); err != nil { + return err + } + cfg, err := config.New() + if err != nil { + return fmt.Errorf("load server config: %w", err) + } + + if len(os.Args) > 1 && os.Args[1] == "migrate" { + cfg.Postgres.RunMigrations = true + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + defer cancel() + return migrator.New(cfg).Start(ctx) + } + name := "messenger-api" + if len(os.Args) > 1 { + name = "messenger-" + os.Args[1] + } + shutdown, err := telemetry.Init(context.Background(), name) + if err != nil { + return err + } + defer shutdown() + attachmentConfig, err := config.LoadAttachment() + if err != nil { + return err + } + if len(os.Args) > 1 && (os.Args[1] == "worker" || os.Args[1] == "dispatcher" || os.Args[1] == "jobs" || os.Args[1] == "outbox") { + return runPipeline(ctx, cfg, attachmentConfig, os.Args[1:]) + } + if len(os.Args) > 1 && os.Args[1] == "storage" { + return runStorage(cfg, attachmentConfig, os.Args[2:]) + } + + // API processes never run migrations; the one-shot migrator owns schema changes. + cfg.Postgres.RunMigrations = false + var level slog.Level + if err := level.UnmarshalText([]byte(cfg.Application.LogLevel)); err != nil { + return err + } + // Logging is initialized for every process by telemetry.Init. + instanceID := telemetry.InstanceID + slog.Info("instance_started", "instance_id", instanceID) + pgx := postgres.New(cfg.Postgres) + rx := cache.New(cfg) + postgresMigrator := migrator.New(cfg) + defer pgx.Close() + telemetry.ObservePool(pgx, 0) + defer rx.Close() + tm := txmanager.New(pgx) + provider := jwtprovider.New(cfg) + encryptor := encrypting.New() + + chatsRepo := storage.NewChatsRepo(pgx) + messageRepo := storage.NewMessagesRepo(pgx) + refreshTokensRepo := storage.NewRefreshTokensRepo(pgx) + userRepo := storage.NewUsersRepo(pgx) + + chatCache := outside.NewChatCache(rx) + + authSvc := auth.New(provider, tm, encryptor, userRepo, refreshTokensRepo) + authSvc.SetLimiter(&ratelimit.Limiter{Client: rx.Client}) + userSvc := user.New(encryptor, userRepo) + chatSvc := chat.New(tm, chatsRepo, messageRepo, chatCache) + objectStore := objectstore.New(attachmentConfig) + attachmentSvc := attachment.New(&storage.AttachmentsRepo{Pool: pgx}, objectStore, attachmentConfig, chatSvc) + pipelineConfig, err := config.LoadPipeline() + if err != nil { + return err + } + outboxSvc := outbox.New(&storage.OutboxRepo{Pool: pgx}, tm, nil) + attachmentSvc.Configure(tm, outboxSvc, pipelineConfig) + + httpRouter := router.New( + middleware.NewAuthMiddleware(authSvc), + middleware.Instance(instanceID), + middleware.RequestIDMiddleware(), + telemetry.HTTP, + middleware.RequestTimeout(cfg.Postgres.QueryTimeout, attachmentConfig.Timeout), + ) + httpRouter.Get("/internal/metrics", func(w stdhttp.ResponseWriter, r *stdhttp.Request) { + stat := pgx.Stat() + _ = json.NewEncoder(w).Encode(map[string]any{"instance_id": instanceID, "acquire_count": stat.AcquireCount(), "acquire_duration_ns": stat.AcquireDuration().Nanoseconds(), "empty_acquire_count": stat.EmptyAcquireCount(), "acquired_conns": stat.AcquiredConns(), "total_conns": stat.TotalConns(), "max_conns": stat.MaxConns()}) + }) + server := http.New(cfg, httpRouter) + requestValidator := validator.NewValidatorImpl() + registerRoutes( + http.NewAuthHandler(httpRouter, authSvc), + http.NewHealthHandler(httpRouter, server, pgx, objectStore), + http.NewChatHandler(httpRouter, provider, requestValidator, chatSvc), + http.NewUserHandler(httpRouter, provider, requestValidator, userSvc), + http.NewAttachmentHandler(httpRouter, provider, attachmentSvc), + ) + + // The only plaintext listener is loopback and exposes exactly the two health routes. + localHealth := &stdhttp.Server{Addr: "127.0.0.1:8081", ReadHeaderTimeout: time.Second, Handler: stdhttp.HandlerFunc(func(w stdhttp.ResponseWriter, r *stdhttp.Request) { + if r.URL.Path != "/health/live" && r.URL.Path != "/health/ready" { + stdhttp.NotFound(w, r) + return + } + httpRouter.ServeHTTP(w, r) + })} + go func() { _ = localHealth.ListenAndServe() }() + defer localHealth.Close() + + if err = postgresMigrator.Start(ctx); err != nil { + return fmt.Errorf("start postgres migrator: %w", err) + } + if err = server.Start(ctx); err != nil { + return fmt.Errorf("start HTTP server: %w", err) + } + if err = rx.Start(ctx); err != nil { + slog.Warn("redis unavailable at startup; continuing without cache", "error", err) + } + slog.Info("HTTP server started", "address", cfg.Server.BindAddress()) + + <-ctx.Done() + shutdownCtx, cancel := context.WithTimeout(context.Background(), cfg.Server.ShutdownTimeout) + defer cancel() + if err = server.Stop(shutdownCtx); err != nil { + return fmt.Errorf("stop HTTP server: %w", err) + } + if err = postgresMigrator.Close(shutdownCtx); err != nil { + return fmt.Errorf("close postgres migrator: %w", err) + } + if err = rx.Close(); err != nil { + return fmt.Errorf("close redis: %w", err) + } + pgx.Close() + return nil +} + +func registerRoutes(handlers ...http.Handler) { + for _, handler := range handlers { + handler.RegisterRoutes() + } +} diff --git a/service/cmd/app/pipeline.go b/service/cmd/app/pipeline.go new file mode 100644 index 0000000..28bf97b --- /dev/null +++ b/service/cmd/app/pipeline.go @@ -0,0 +1,156 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "flag" + "gitea.privcloud.tech/mtusi2026/internal/adapter/repository/storage" + "gitea.privcloud.tech/mtusi2026/internal/config" + "gitea.privcloud.tech/mtusi2026/internal/pkg/broker" + "gitea.privcloud.tech/mtusi2026/internal/pkg/objectstore" + "gitea.privcloud.tech/mtusi2026/internal/pkg/postgres" + "gitea.privcloud.tech/mtusi2026/internal/pkg/postgres/txmanager" + "gitea.privcloud.tech/mtusi2026/internal/pkg/telemetry" + "gitea.privcloud.tech/mtusi2026/internal/service/attachment" + "gitea.privcloud.tech/mtusi2026/internal/service/observation" + "gitea.privcloud.tech/mtusi2026/internal/service/outbox" + "github.com/google/uuid" + "github.com/jackc/pgx/v5" + "log/slog" + "os" + "os/signal" + "syscall" + "time" +) + +func runPipeline(lifetime context.Context, cfg *config.Config, a *config.Attachment, args []string) error { + c, e := config.LoadPipeline() + if e != nil { + return e + } + ctx, stop := signal.NotifyContext(lifetime, os.Interrupt, syscall.SIGTERM) + defer stop() + pool := postgres.New(cfg.Postgres) + defer pool.Close() + telemetry.ObservePool(pool, c.Concurrency) + tm := txmanager.New(pool) + repo := &storage.AttachmentsRepo{Pool: pool} + rabbit := &broker.Rabbit{URL: c.BrokerURL} + ob := outbox.New(&storage.OutboxRepo{Pool: pool}, tm, rabbit) + svc := attachment.New(repo, objectstore.New(a), a, nil) + svc.Configure(tm, ob, c) + ob.OnDead = svc.PublishFailed + if args[0] == "jobs" || args[0] == "outbox" { + if len(args) < 2 { + return errors.New("operator action required") + } + fs := flag.NewFlagSet(args[1], flag.ContinueOnError) + idText := fs.String("id", "", "job ID (attachment ID for reprocess)") + version := fs.Int("pipeline-version", 2, "new pipeline version") + if e = fs.Parse(args[2:]); e != nil { + return e + } + if args[1] == "list" { + jobs, e := repo.ListJobs(ctx) + if e != nil { + return e + } + return json.NewEncoder(os.Stdout).Encode(jobs) + } + id, e := uuid.Parse(*idText) + if e != nil { + return e + } + switch args[1] { + case "redeliver": + if os.Getenv("LAB5_FAULTS") != "1" { + return errors.New("redeliver is restricted to the explicit fault profile") + } + var payload []byte + e = tm.RunInTransaction(ctx, nil, func(tx pgx.Tx) error { + j, e := repo.JobTx(ctx, tx, id) + if e != nil { + return e + } + payload, e = json.Marshal(j.Message) + return e + }) + if e == nil { + e = rabbit.Publish(ctx, outbox.DeliveryMessage{ID: uuid.New(), AggregateID: id, Payload: payload}) + } + case "replay": + e = svc.Replay(ctx, id) + case "reprocess": + e = svc.Reprocess(ctx, id, *version) + default: + return errors.New("unknown operator action") + } + if e != nil { + return e + } + return json.NewEncoder(os.Stdout).Encode(map[string]any{"accepted": true, "id": id}) + } + if c.BrokerURL == "" { + return errors.New("BROKER_URL_FILE is required") + } + if args[0] == "worker" { + // Recovery continues while the consumer is disconnected or a delivery is paused. + recoveryDone := make(chan struct{}) + defer func() { stop(); <-recoveryDone }() + go func() { + defer close(recoveryDone) + tick := time.NewTicker(time.Second) + defer tick.Stop() + for { + select { + case <-ctx.Done(): + return + case <-tick.C: + if e := svc.Recover(ctx); e != nil { + slog.Warn("job_recovery_failed") + } + if e := svc.Cleanup(ctx); e != nil { + slog.Warn("source_cleanup_failed") + } + } + } + }() + for ctx.Err() == nil { + if e = rabbit.Consume(ctx, c.Concurrency, c.Prefetch, svc.Process); e != nil && ctx.Err() == nil { + slog.Warn("consumer_reconnecting") + } + select { + case <-ctx.Done(): + case <-time.After(time.Second): + } + } + return nil + } + observer := &observation.Service{Repo: &storage.ObservationRepo{Pool: pool}, Publish: func(s observation.Snapshot) { + telemetry.PublishObservation(s.Queued, s.Outbox, s.Good, s.Bad, s.Oldest, s.Locked) + }} + telemetry.RegisterObservation() + nextObservation := time.Time{} + for ctx.Err() == nil { + if time.Now().After(nextObservation) { + if err := observer.Observe(ctx); err != nil { + slog.Warn("observation_unavailable") + } + depth, err := rabbit.Depth(ctx) + if err == nil { + telemetry.QueueDepth.Store(int64(depth)) + } + nextObservation = time.Now().Add(time.Second) + } + _, e = ob.ProcessOnce(ctx) + if e != nil { + slog.Warn("dispatcher_retry") + } + select { + case <-ctx.Done(): + case <-time.After(100 * time.Millisecond): + } + } + return nil +} diff --git a/service/cmd/app/storage.go b/service/cmd/app/storage.go new file mode 100644 index 0000000..09d459c --- /dev/null +++ b/service/cmd/app/storage.go @@ -0,0 +1,183 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "flag" + "fmt" + "log/slog" + "net/http" + "os" + "strings" + "time" + + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/aws/smithy-go" + "github.com/google/uuid" + + "gitea.privcloud.tech/mtusi2026/internal/adapter/repository/storage" + "gitea.privcloud.tech/mtusi2026/internal/config" + "gitea.privcloud.tech/mtusi2026/internal/pkg/objectstore" + "gitea.privcloud.tech/mtusi2026/internal/pkg/postgres" + "gitea.privcloud.tech/mtusi2026/internal/pkg/postgres/txmanager" + "gitea.privcloud.tech/mtusi2026/internal/service/attachment" + "gitea.privcloud.tech/mtusi2026/internal/service/outbox" +) + +func runStorage(cfg *config.Config, c *config.Attachment, args []string) error { + slog.SetDefault(slog.New(slog.NewJSONHandler(os.Stderr, &slog.HandlerOptions{Level: slog.LevelWarn}))) + if len(args) == 0 { + return fmt.Errorf("storage command required") + } + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute) + defer cancel() + store := objectstore.New(c) + if args[0] == "bootstrap" { + var err error + for i := 0; i < 60; i++ { + _, err = store.Client.CreateBucket(ctx, &s3.CreateBucketInput{Bucket: &c.Bucket}) + var api smithy.APIError + if err == nil || (errors.As(err, &api) && (api.ErrorCode() == "BucketAlreadyOwnedByYou" || api.ErrorCode() == "BucketAlreadyExists")) { + if store.Ping(ctx) == nil { + // HEAD alone can succeed before the restarted master/volume write path. + // This constant key is outside the application's immutable originals prefix. + key := "bootstrap/write-check" + if store.Put(ctx, key, "application/octet-stream", strings.NewReader("ready"), 5) == nil && store.Delete(ctx, key) == nil { + return json.NewEncoder(os.Stdout).Encode(map[string]any{"bucket_ready": true, "write_checked": true}) + } + } + } + time.Sleep(time.Second) + } + return fmt.Errorf("storage bootstrap failed: %w", err) + } + if args[0] == "private-check" { + // Unsigned requests must reach the S3 server and receive an auth failure, + // rather than passing because a network error hid a public object. + fs := flag.NewFlagSet("private-check", flag.ContinueOnError) + key := fs.String("key", "", "known object key") + if e := fs.Parse(args[1:]); e != nil { + return e + } + client := http.Client{Timeout: 5 * time.Second, CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }} + resp, e := client.Get(c.Endpoint + "/" + c.Bucket + "/" + *key) + if e != nil { + return e + } + defer resp.Body.Close() + if resp.StatusCode != 401 && resp.StatusCode != 403 { + return fmt.Errorf("unsigned GET returned %d", resp.StatusCode) + } + _, e = store.Client.CreateBucket(ctx, &s3.CreateBucketInput{Bucket: new("forbidden-" + uuid.NewString())}) + var api smithy.APIError + if !errors.As(e, &api) || api.ErrorCode() != "AccessDenied" { + return fmt.Errorf("API key bucket creation was not denied") + } + return json.NewEncoder(os.Stdout).Encode(map[string]any{"anonymous_status": resp.StatusCode, "api_create_bucket_denied": true}) + } + if args[0] == "object-check" { + fs := flag.NewFlagSet("object-check", flag.ContinueOnError) + key := fs.String("key", "", "synthetic object key") + if e := fs.Parse(args[1:]); e != nil { + return e + } + head, e := store.Head(ctx, *key) + if objectstore.Missing(e) { + return json.NewEncoder(os.Stdout).Encode(map[string]any{"exists": false}) + } + if e != nil { + return e + } + return json.NewEncoder(os.Stdout).Encode(map[string]any{"exists": true, "last_modified": head.LastModified, "size_bytes": head.ContentLength}) + } + + pool := postgres.New(cfg.Postgres) + defer pool.Close() + svc := attachment.New(&storage.AttachmentsRepo{Pool: pool}, store, c, nil) + jobs, e := config.LoadPipeline() + if e != nil { + return e + } + tm := txmanager.New(pool) + svc.Configure(tm, outbox.New(&storage.OutboxRepo{Pool: pool}, tm, nil), jobs) + switch args[0] { + case "reconcile": + fs := flag.NewFlagSet("reconcile", flag.ContinueOnError) + apply := fs.Bool("apply", false, "apply safe changes") + if e := fs.Parse(args[1:]); e != nil { + return e + } + result, e := svc.Reconcile(ctx, *apply) + _ = json.NewEncoder(os.Stdout).Encode(result) + return e + case "restore": + fs := flag.NewFlagSet("restore", flag.ContinueOnError) + id := fs.String("id", "", "attachment id") + source := fs.String("source", "", "original file") + if e := fs.Parse(args[1:]); e != nil { + return e + } + uid, e := uuid.Parse(*id) + if e != nil { + return e + } + if e = svc.Restore(ctx, uid, *source); e != nil { + return e + } + return json.NewEncoder(os.Stdout).Encode(map[string]any{"restored": uid}) + case "seed": + fs := flag.NewFlagSet("seed", flag.ContinueOnError) + count := fs.Int("count", 1, "synthetic orphan count") + keyOverride := fs.String("key", "", "explicit key for a single synthetic object") + if e := fs.Parse(args[1:]); e != nil { + return e + } + if *count < 1 || *count > 10000 { + return fmt.Errorf("count must be 1..10000") + } + keys := []string{} + for i := 0; i < *count; i++ { + key := attachment.Prefix + uuid.NewString() + if *keyOverride != "" { + if *count != 1 { + return fmt.Errorf("explicit key requires count=1") + } + key = *keyOverride + } + if e := store.Put(ctx, key, "image/png", strings.NewReader("synthetic-orphan"), 16); e != nil { + return e + } + if *count == 1 { + keys = append(keys, key) + } + } + return json.NewEncoder(os.Stdout).Encode(map[string]any{"seeded": *count, "keys": keys}) + case "remove-object": + fs := flag.NewFlagSet("remove-object", flag.ContinueOnError) + id := fs.String("id", "", "synthetic attachment id") + if e := fs.Parse(args[1:]); e != nil { + return e + } + uid, e := uuid.Parse(*id) + if e != nil { + return e + } + a, e := svc.Repo.Get(ctx, uid) + if e != nil { + return e + } + conn, unlock, e := svc.Repo.Lock(ctx, a.Key) + _ = conn + if e != nil { + return e + } + defer unlock() + if e = store.Delete(ctx, a.Key); e != nil { + return e + } + return json.NewEncoder(os.Stdout).Encode(map[string]any{"removed_object": uid}) + default: + return fmt.Errorf("unknown storage command") + } +} diff --git a/service/cmd/benchmark/main.go b/service/cmd/benchmark/main.go new file mode 100644 index 0000000..80efdf8 --- /dev/null +++ b/service/cmd/benchmark/main.go @@ -0,0 +1,260 @@ +// Command benchmark generates paced lab5 traffic through the public load balancer. +package main + +import ( + "bytes" + "encoding/json" + "flag" + "fmt" + "io" + "math" + "net/http" + "os" + "slices" + "strings" + "sync" + "sync/atomic" + "time" + + "github.com/google/uuid" +) + +type client struct { + base, token string + http *http.Client +} + +func (c *client) call(method, path string, body any, key string) (map[string]any, int, error) { + var data io.Reader + if body != nil { + b, e := json.Marshal(body) + if e != nil { + return nil, 0, e + } + data = bytes.NewReader(b) + } + req, e := http.NewRequest(method, c.base+path, data) + if e != nil { + return nil, 0, e + } + if body != nil { + req.Header.Set("Content-Type", "application/json") + } + if c.token != "" { + req.Header.Set("Authorization", "Bearer "+c.token) + } + if key != "" { + req.Header.Set("Idempotency-Key", key) + } + // net/http otherwise treats Idempotency-Key + GetBody as permission to retry POST. + if method == http.MethodPost { + req.GetBody = nil + } + resp, e := c.http.Do(req) + if e != nil { + return nil, 0, e + } + defer resp.Body.Close() + b, e := io.ReadAll(io.LimitReader(resp.Body, 1<<20)) + if e != nil { + return nil, resp.StatusCode, e + } + if resp.StatusCode != 200 && resp.StatusCode != 201 { + return nil, resp.StatusCode, fmt.Errorf("HTTP %d", resp.StatusCode) + } + var out map[string]any + e = json.Unmarshal(b, &out) + return out, resp.StatusCode, e +} +func (c *client) setup(seed int) (string, error) { + username := "bench_" + strings.ReplaceAll(uuid.NewString(), "-", "")[:20] + password := "Synthetic-benchmark-123" + user, _, e := c.call("POST", "/api/v1/users", map[string]string{"username": username, "display_name": "Benchmark", "password": password}, "") + if e != nil { + return "", e + } + req, _ := http.NewRequest("POST", c.base+"/api/v1/auth/sessions", nil) + req.SetBasicAuth(username, password) + resp, e := c.http.Do(req) + if e != nil { + return "", e + } + var session struct { + Token string `json:"token"` + } + e = json.NewDecoder(resp.Body).Decode(&session) + resp.Body.Close() + if e != nil || resp.StatusCode != 201 || session.Token == "" { + return "", fmt.Errorf("login status %d: %v", resp.StatusCode, e) + } + c.token = session.Token + chat, _, e := c.call("POST", "/api/v1/chats", map[string]any{"title": "Benchmark", "member_ids": []any{user["id"]}}, "") + if e != nil { + return "", e + } + path := "/api/v1/chats/" + chat["id"].(string) + "/messages" + // Bounded concurrent setup is excluded from the measured interval. + var group sync.WaitGroup + slots := make(chan struct{}, 8) + errs := make(chan error, seed) + for i := range seed { + slots <- struct{}{} + group.Go(func() { + defer func() { <-slots }() + _, _, e := c.call("POST", path, map[string]string{"text": fmt.Sprintf("seed-%d", i)}, uuid.NewString()) + if e != nil { + errs <- e + } + }) + } + group.Wait() + close(errs) + for e := range errs { + if e != nil { + return "", e + } + } + return path, nil +} + +type result struct { + MeasurementUnix float64 `json:"measurement_started_unix"` + Label string `json:"label"` + TargetRate int `json:"target_rps"` + Duration float64 `json:"duration_seconds"` + Elapsed float64 `json:"elapsed_including_drain_seconds"` + Scheduled int `json:"scheduled"` + Dropped int `json:"generator_dropped"` + Success int `json:"successful_requests"` + Errors int `json:"failed_requests"` + ReadSuccess int `json:"read_success"` + WriteSuccess int `json:"write_success"` + SuccessfulRPS float64 `json:"successful_rps"` + ErrorRatio float64 `json:"error_ratio_including_drops"` + Latency map[string]float64 `json:"successful_latency_ms"` + Statuses map[int]int `json:"statuses"` + PeakInFlight int64 `json:"peak_in_flight"` + Note string `json:"note"` +} + +func phase(c *client, path string, rate int, duration time.Duration, maxInFlight int, label string) result { + out := result{Label: label, TargetRate: rate, Duration: duration.Seconds(), Statuses: map[int]int{}, Latency: map[string]float64{}, Note: "Paced open-loop, 10ms dispatch batches; 90% first-page GET, 10% POST; no POST retries. Percentiles of successes include scheduler delay; failures and capacity drops reported separately. RPS denominator includes drain."} + start := time.Now() + out.MeasurementUnix = float64(start.UnixNano()) / 1e9 + end := start.Add(duration) + slots := make(chan struct{}, maxInFlight) + var group sync.WaitGroup + var mu sync.Mutex + var active atomic.Int64 + var peak atomic.Int64 + latencies := make([]float64, 0, rate*int(duration.Seconds())) + index := 0 + for tick := 0; ; tick++ { + now := time.Now() + if !now.Before(end) { + break + } + target := start.Add(time.Duration(tick) * 10 * time.Millisecond) + if target.After(now) { + time.Sleep(time.Until(target)) + } + until := min(int(time.Since(start).Seconds()*float64(rate))+1, int(duration.Seconds()*float64(rate))) + for index < until { + i := index + index++ + out.Scheduled++ + scheduled := start.Add(time.Duration(float64(i) / float64(rate) * float64(time.Second))) + select { + case slots <- struct{}{}: + group.Go(func() { + n := active.Add(1) + for old := peak.Load(); n > old; old = peak.Load() { + if peak.CompareAndSwap(old, n) { + break + } + } + defer func() { active.Add(-1); <-slots }() + method, p := "GET", path+"?limit=50" + var body any + key := "" + if i%10 == 0 { + method = "POST" + p = path + key = uuid.NewString() + body = map[string]string{"text": key} + } + _, status, err := c.call(method, p, body, key) + latency := float64(time.Since(scheduled).Microseconds()) / 1000 + mu.Lock() + defer mu.Unlock() + out.Statuses[status]++ + if err != nil { + out.Errors++ + } else { + out.Success++ + if method == "GET" { + out.ReadSuccess++ + } else { + out.WriteSuccess++ + } + latencies = append(latencies, max(0, latency)) + } + }) + default: + out.Dropped++ + } + } + } + group.Wait() + out.Elapsed = time.Since(start).Seconds() + out.PeakInFlight = peak.Load() + out.SuccessfulRPS = float64(out.Success) / out.Elapsed + if out.Scheduled > 0 { + out.ErrorRatio = float64(out.Errors+out.Dropped) / float64(out.Scheduled) + } + slices.Sort(latencies) + for name, p := range map[string]float64{"p50": .5, "p95": .95, "p99": .99} { + if len(latencies) > 0 { + out.Latency[name] = latencies[max(0, int(math.Ceil(p*float64(len(latencies))))-1)] + } + } + return out +} +func main() { + base := flag.String("base-url", "http://localhost:18086", "public LB URL") + rate := flag.Int("rate", 100, "scheduled requests per second") + duration := flag.Duration("duration", 30*time.Second, "measured time") + warmup := flag.Duration("warmup", 5*time.Second, "warmup excluded from results") + seed := flag.Int("seed", 1000, "initial messages") + maxFlight := flag.Int("max-in-flight", 256, "bounded in-flight limit") + label := flag.String("label", "manual", "case label") + output := flag.String("output", "", "JSON destination") + flag.Parse() + if *rate < 1 || *rate > 100000 || *duration <= 0 || *warmup < 0 || *maxFlight < 1 || *seed < 0 { + fmt.Fprintln(os.Stderr, "invalid benchmark settings") + os.Exit(2) + } + c := &client{base: *base, http: &http.Client{Timeout: 2 * time.Second, Transport: &http.Transport{MaxIdleConns: 512, MaxIdleConnsPerHost: 512, MaxConnsPerHost: *maxFlight, IdleConnTimeout: 60 * time.Second}}} + path, e := c.setup(*seed) + if e != nil { + fmt.Fprintln(os.Stderr, "setup:", e) + os.Exit(1) + } + if *warmup > 0 { + phase(c, path, *rate, *warmup, *maxFlight, "warmup") + } + row := phase(c, path, *rate, *duration, *maxFlight, *label) + b, e := json.MarshalIndent(row, "", " ") + if e != nil { + panic(e) + } + if *output != "" { + if e = os.WriteFile(*output, append(b, '\n'), 0644); e != nil { + panic(e) + } + } + fmt.Println(string(b)) + if row.Success == 0 { + os.Exit(1) + } +} diff --git a/service/cmd/benchmark/main_test.go b/service/cmd/benchmark/main_test.go new file mode 100644 index 0000000..63e1034 --- /dev/null +++ b/service/cmd/benchmark/main_test.go @@ -0,0 +1,42 @@ +package main + +import ( + "io" + "net/http" + "net/http/httptest" + "sync/atomic" + "testing" + "time" +) + +func TestLostPostResponseIsNotRetried(t *testing.T) { + var posts atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method == http.MethodGet { + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{}`)) + return + } + posts.Add(1) + _, _ = io.Copy(io.Discard, r.Body) + conn, _, err := w.(http.Hijacker).Hijack() + if err != nil { + t.Error(err) + return + } + _ = conn.Close() + })) + defer server.Close() + transport := &http.Transport{MaxIdleConnsPerHost: 1} + defer transport.CloseIdleConnections() + c := &client{base: server.URL, http: &http.Client{Timeout: time.Second, Transport: transport}} + if _, _, err := c.call(http.MethodGet, "/warm", nil, ""); err != nil { + t.Fatal(err) + } + if _, _, err := c.call(http.MethodPost, "/message", map[string]string{"text": "uncertain result"}, "same-key"); err == nil { + t.Fatal("lost response must be reported as an error") + } + if posts.Load() != 1 { + t.Fatalf("transport sent %d POSTs, want exactly 1", posts.Load()) + } +} diff --git a/service/cmd/securityprobe/main.go b/service/cmd/securityprobe/main.go new file mode 100644 index 0000000..0848016 --- /dev/null +++ b/service/cmd/securityprobe/main.go @@ -0,0 +1,335 @@ +// securityprobe exercises real dependency authorization and emits no secrets. +package main + +import ( + "bytes" + "context" + "crypto/tls" + "crypto/x509" + "encoding/binary" + "encoding/json" + "errors" + "fmt" + "gitea.privcloud.tech/mtusi2026/internal/config" + "gitea.privcloud.tech/mtusi2026/internal/pkg/encrypting" + "gitea.privcloud.tech/mtusi2026/internal/pkg/objectstore" + "gitea.privcloud.tech/mtusi2026/internal/pkg/openbao" + "gitea.privcloud.tech/mtusi2026/internal/pkg/transportsecurity" + "github.com/aws/smithy-go" + "github.com/jackc/pgx/v5" + amqp "github.com/rabbitmq/amqp091-go" + "github.com/redis/go-redis/v9" + "io" + "net" + "net/http" + "os" + "sort" + "strings" + "syscall" + "time" +) + +func main() { + if err := run(); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } +} +func run() error { + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) + defer cancel() + mode := os.Getenv("PROBE_MODE") + target := os.Getenv("PROBE_TARGET") + expect := os.Getenv("PROBE_EXPECT") + if mode == "hash" || mode == "kdf" { + encryptor := encrypting.New() + if mode == "hash" { + h, e := encryptor.EncryptSecretUsingArgon2id(ctx, os.Getenv("PROBE_PASSWORD")) + if e != nil { + return e + } + fmt.Print(h) + return nil + } + times := []float64{} + for i := 0; i < 12; i++ { + start := time.Now() + _, e := encryptor.EncryptSecretUsingArgon2id(ctx, "synthetic-benchmark-password") + if e != nil { + return e + } + times = append(times, float64(time.Since(start).Microseconds())/1000) + } + sort.Float64s(times) + return json.NewEncoder(os.Stdout).Encode(map[string]any{"samples": 12, "memory_kib": 65536, "iterations": 3, "parallelism": 1, "max_parallel_kdf": 2, "p50_ms": times[6], "p95_ms": times[11], "max_ms": times[11]}) + } + tc, err := transportsecurity.Client() + if err != nil { + return fmt.Errorf("probe TLS material unavailable") + } + if os.Getenv("PROBE_UNTRUST_SERVER") == "1" { + tc.RootCAs = x509.NewCertPool() + } + if os.Getenv("PROBE_NO_CERT") == "1" { + tc.Certificates = nil + } + if mode == "telemetry" { + client := http.Client{Timeout: 5 * time.Second, Transport: &http.Transport{TLSClientConfig: tc}} + endpoints := map[string]string{"metrics": "https://prometheus:9443/api/v1/query?query=up", "logs": "https://loki:9443/loki/api/v1/query_range?query=%7Bservice_name%3D%22messenger-worker%22%7D&limit=5", "traces": "https://tempo:9443/api/traces/" + os.Getenv("PROBE_TRACE_ID")} + result := map[string]any{} + for name, url := range endpoints { + r, e := client.Get(url) + if e != nil { + return fmt.Errorf("telemetry %s transport failed", name) + } + b, e := io.ReadAll(io.LimitReader(r.Body, 2<<20)) + r.Body.Close() + if e != nil || r.StatusCode != 200 { + return fmt.Errorf("telemetry %s query failed", name) + } + var data map[string]json.RawMessage + if json.Unmarshal(b, &data) != nil { + return fmt.Errorf("invalid telemetry response") + } + result[name] = json.RawMessage(b) + } + return json.NewEncoder(os.Stdout).Encode(result) + } + if mode == "db-tls" { + raw, e := net.DialTimeout("tcp", "postgres:5432", 3*time.Second) + if e != nil { + return fmt.Errorf("DB listener unreachable") + } + defer raw.Close() + raw.SetDeadline(time.Now().Add(3 * time.Second)) + packet := make([]byte, 8) + binary.BigEndian.PutUint32(packet, 8) + binary.BigEndian.PutUint32(packet[4:], 80877103) + if _, e = raw.Write(packet); e != nil { + return e + } + answer := make([]byte, 1) + if _, e = io.ReadFull(raw, answer); e != nil || answer[0] != 'S' { + return fmt.Errorf("Postgres SSLRequest failed") + } + tc.ServerName = "postgres" + if v := os.Getenv("PROBE_SERVER_NAME"); v != "" { + tc.ServerName = v + } + e = tls.Client(raw, tc).HandshakeContext(ctx) + if (e != nil) != (expect == "deny") { + return fmt.Errorf("DB TLS expectation failed") + } + return json.NewEncoder(os.Stdout).Encode(map[string]any{"mode": mode, "matched": true}) + } + if mode == "amqp-tls" { + conn, e := amqp.DialConfig("amqps://broker:5671/attachments", amqp.Config{TLSClientConfig: tc, SASL: []amqp.Authentication{&amqp.ExternalAuth{}}}) + if conn != nil { + defer conn.Close() + } + if (e != nil) != (expect == "deny") { + return fmt.Errorf("AMQP identity expectation failed") + } + return json.NewEncoder(os.Stdout).Encode(map[string]any{"mode": mode, "expect": expect, "matched": true}) + } + if mode == "tls" { + if name := os.Getenv("PROBE_SERVER_NAME"); name != "" { + tc.ServerName = name + } + req, _ := http.NewRequestWithContext(ctx, "GET", target, nil) + if strings.Contains(target, "otel-collector") && strings.Contains(target, "/v1/") { + req, _ = http.NewRequestWithContext(ctx, "POST", target, strings.NewReader("{}")) + req.Header.Set("Content-Type", "application/json") + } + if method := os.Getenv("PROBE_METHOD"); method != "" { + req.Method = method + } + client := http.Client{Timeout: 5 * time.Second, Transport: &http.Transport{TLSClientConfig: tc}} + r, e := client.Do(req) + denied := e != nil + if e != nil { + var n net.Error + if errors.As(e, &n) && n.Timeout() { + return fmt.Errorf("timeout is not identity denial") + } + if errors.Is(e, syscall.ECONNREFUSED) { + return fmt.Errorf("connection refused is not identity denial") + } + } + if r != nil { + defer r.Body.Close() + io.Copy(io.Discard, r.Body) + denied = r.StatusCode == 401 || r.StatusCode == 403 + } + if expect == "deny" && !denied { + return fmt.Errorf("TLS/identity denial missing") + } + if expect != "deny" && denied { + return fmt.Errorf("authorized TLS request failed") + } + // Prove a reachable TLS listener independently: a failed TCP connect is never accepted as an identity denial. + if denied { + address := strings.TrimPrefix(strings.Split(target[8:], "/")[0], "https://") + c, e := net.DialTimeout("tcp", address, 2*time.Second) + if e != nil { + return fmt.Errorf("network failure is not a security denial") + } + c.Close() + } + return json.NewEncoder(os.Stdout).Encode(map[string]any{"mode": mode, "expect": expect, "matched": true}) + } + stop, err := openbao.Start(ctx) + if err != nil { + return err + } + defer stop() + role := os.Getenv("SERVICE_ROLE") + out := map[string]any{"mode": mode, "role": role} + check := func(name string, allowed bool, err error) error { + ok := (err == nil) == allowed + out[name] = ok + if !ok { + return fmt.Errorf("authorization mismatch: %s", name) + } + return nil + } + switch mode { + case "db": + cfg, e := config.New() + if e != nil { + return e + } + conn, e := pgx.Connect(ctx, cfg.Postgres.GetURI()) + if e != nil { + return fmt.Errorf("DB connection failed") + } + defer conn.Close(ctx) + for _, q := range []struct { + name, sql string + allow bool + }{{"jobs_read", "SELECT id FROM attachment_jobs LIMIT 0", true}, {"password_read_denied", "SELECT password_hash FROM users LIMIT 0", false}, {"session_read_denied", "SELECT token_hash FROM refresh_tokens LIMIT 0", false}, {"messages_read_denied", "SELECT id FROM messages LIMIT 0", false}, {"ddl_denied", "CREATE TABLE security_probe_should_not_exist(id int)", false}} { + _, e := conn.Exec(ctx, q.sql) + if e = check(q.name, q.allow, e); e != nil { + return e + } + } + case "s3": + cfg, e := config.LoadAttachment() + if e != nil { + return e + } + s := objectstore.New(cfg) + key := "results/security-probe/" + fmt.Sprint(time.Now().UnixNano()) + e = s.Put(ctx, key, "application/octet-stream", bytes.NewReader([]byte("synthetic")), 9) + if os.Getenv("PROBE_OLD_SECRET") == "1" { + var denied smithy.APIError + if !errors.As(e, &denied) || (denied.ErrorCode() != "InvalidAccessKeyId" && denied.ErrorCode() != "AccessDenied" && denied.ErrorCode() != "SignatureDoesNotMatch") { + return fmt.Errorf("old S3 secret was not explicitly denied") + } + return json.NewEncoder(os.Stdout).Encode(map[string]any{"old_s3_secret_rejected": true}) + } + if e = check("results_write", true, e); e != nil { + return e + } + _, e = s.Get(ctx, key) + if e = check("results_read", true, e); e != nil { + return e + } + defer s.Delete(ctx, key) + for _, key := range []string{"originals/security-denied", "forbidden/security-denied"} { + if os.Getenv("PROBE_BASELINE") == "1" { + defer s.Delete(ctx, key) + } + e = s.Put(ctx, key, "application/octet-stream", bytes.NewReader([]byte("x")), 1) + if e = check(key, os.Getenv("PROBE_BASELINE") == "1", e); e != nil { + return e + } + } + case "redis": + cfg, e := config.New() + if e != nil { + return e + } + r := redis.NewClient(cfg.Redis.BuildRedisOptions()) + defer r.Close() + if e = check("ping", true, r.Ping(ctx).Err()); e != nil { + return e + } + if e = check("foreign_key_denied", false, r.Get(ctx, "forbidden:security-probe").Err()); e != nil { + return e + } + if e = check("config_denied", false, r.ConfigGet(ctx, "*").Err()); e != nil { + return e + } + case "amqp": + dial := func() (*amqp.Connection, *amqp.Channel, error) { + conn, e := amqp.DialConfig("amqps://broker:5671/attachments", amqp.Config{TLSClientConfig: tc, SASL: []amqp.Authentication{&amqp.ExternalAuth{}}}) + if e != nil { + return nil, nil, e + } + ch, e := conn.Channel() + return conn, ch, e + } + conn, ch, e := dial() + if e != nil { + return fmt.Errorf("authorized AMQP connection failed") + } + defer conn.Close() + _, e = ch.QueueInspect("attachments.v1") + if e = check("queue_read", true, e); e != nil { + return e + } + ch.Close() + ch, _ = conn.Channel() + baseline := os.Getenv("PROBE_BASELINE") == "1" + queue := "attachments.security-probe" + _, e = ch.QueueDeclare(queue, true, false, false, false, nil) + if e = check("configure_profile", baseline, e); e != nil { + return e + } + ch, _ = conn.Channel() + destination := "attachments.v1" + if baseline { + destination = queue + } + // Synchronous confirm/error reporting avoids accepting an async ACL failure as success. + if e = ch.Confirm(false); e != nil { + return e + } + confirms := ch.NotifyPublish(make(chan amqp.Confirmation, 1)) + closed := ch.NotifyClose(make(chan *amqp.Error, 1)) + e = ch.PublishWithContext(ctx, "", destination, false, false, amqp.Publishing{Body: []byte("synthetic-probe")}) + if e == nil { + select { + case ack := <-confirms: + if !ack.Ack { + e = fmt.Errorf("not confirmed") + } + case <-closed: + e = fmt.Errorf("broker denied publish") + case <-ctx.Done(): + return fmt.Errorf("broker probe timed out") + } + } + time.Sleep(200 * time.Millisecond) + if e == nil && ch.IsClosed() { + e = fmt.Errorf("denied") + } + if e = check("publish_profile", baseline, e); e != nil { + return e + } + if baseline { + cleanup, _ := conn.Channel() + defer cleanup.Close() + _, e = cleanup.QueueDelete(queue, false, false, false) + if e != nil { + return e + } + } + default: + return fmt.Errorf("unknown probe mode") + } + return json.NewEncoder(os.Stdout).Encode(out) +} + +var _ = tls.VersionTLS12 diff --git a/service/go.mod b/service/go.mod new file mode 100644 index 0000000..3246192 --- /dev/null +++ b/service/go.mod @@ -0,0 +1,96 @@ +module gitea.privcloud.tech/mtusi2026 + +go 1.27.4 + +require ( + github.com/aws/aws-sdk-go-v2 v1.47.8 + github.com/aws/aws-sdk-go-v2/credentials v1.20.16 + github.com/aws/aws-sdk-go-v2/service/s3 v1.114.9 + github.com/aws/smithy-go v1.28.8 + github.com/go-chi/chi/v5 v5.5.0 + github.com/go-playground/validator/v10 v10.30.5 + github.com/golang-jwt/jwt/v5 v5.3.1 + github.com/google/uuid v1.6.0 + github.com/jackc/pgerrcode v0.0.0-20250907135507-afb5586c32a6 + github.com/jackc/pgx/v5 v5.11.0 + github.com/jackc/tern/v2 v2.4.3 + github.com/oapi-codegen/runtime v1.6.0 + github.com/rabbitmq/amqp091-go v1.14.0 + github.com/redis/go-redis/v9 v9.23.0 + github.com/shirou/gopsutil/v4 v4.26.10 + github.com/spf13/viper v1.21.0 + github.com/vgarvardt/pgx-google-uuid/v5 v5.6.0 + go.opentelemetry.io/contrib/bridges/otelslog v0.21.0 + go.opentelemetry.io/otel v1.47.0 + go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.23.0 + go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.47.0 + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.47.0 + go.opentelemetry.io/otel/metric v1.47.0 + go.opentelemetry.io/otel/sdk v1.47.0 + go.opentelemetry.io/otel/sdk/log v1.47.0 + go.opentelemetry.io/otel/sdk/metric v1.47.0 + go.opentelemetry.io/otel/trace v1.47.0 + go.uber.org/multierr v1.11.0 + golang.org/x/crypto v0.59.0 + golang.org/x/sync v0.25.0 + google.golang.org/grpc v1.85.0 +) + +require ( + dario.cat/mergo v1.0.2 // indirect + github.com/Masterminds/goutils v1.1.1 // indirect + github.com/Masterminds/semver/v3 v3.5.0 // indirect + github.com/Masterminds/sprig/v3 v3.3.0 // indirect + github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.26 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.11 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.11 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.11 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.24 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.12 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.11 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.11 // indirect + github.com/cenkalti/backoff/v5 v5.0.3 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/ebitengine/purego v0.11.2 // indirect + github.com/fsnotify/fsnotify v1.9.0 // indirect + github.com/gabriel-vasile/mimetype v1.4.15 // indirect + github.com/go-logr/logr v1.4.4 // indirect + github.com/go-logr/stdr v1.2.2 // indirect + github.com/go-ole/go-ole v1.2.6 // indirect + github.com/go-playground/locales v0.14.1 // indirect + github.com/go-playground/universal-translator v0.18.1 // indirect + github.com/go-viper/mapstructure/v2 v2.5.0 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.30.0 // indirect + github.com/huandu/xstrings v1.5.0 // indirect + github.com/jackc/pgpassfile v1.0.0 // indirect + github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect + github.com/jackc/puddle/v2 v2.2.2 // indirect + github.com/leodido/go-urn v1.5.0 // indirect + github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect + github.com/mitchellh/copystructure v1.2.0 // indirect + github.com/mitchellh/reflectwalk v1.0.2 // indirect + github.com/pelletier/go-toml/v2 v2.4.3-0.20260624044523-35f78d5908f1 // indirect + github.com/power-devops/perfstat v0.0.0-20260805114148-88456608a4f6 // indirect + github.com/sagikazarmark/locafero v0.11.0 // indirect + github.com/shopspring/decimal v1.4.0 // indirect + github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8 // indirect + github.com/spf13/afero v1.15.0 // indirect + github.com/spf13/cast v1.10.0 // indirect + github.com/spf13/pflag v1.0.10 // indirect + github.com/subosito/gotenv v1.6.0 // indirect + github.com/tklauser/go-sysconf v0.4.0 // indirect + github.com/tklauser/numcpus v0.12.0 // indirect + github.com/yusufpapurcu/wmi v1.2.4 // indirect + go.opentelemetry.io/auto/sdk v1.2.1 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.47.0 // indirect + go.opentelemetry.io/otel/log v1.47.0 // indirect + go.opentelemetry.io/proto/otlp v1.11.0 // indirect + go.uber.org/atomic v1.12.0 // indirect + go.yaml.in/yaml/v3 v3.0.5 // indirect + golang.org/x/net v0.62.0 // indirect + golang.org/x/sys v0.50.0 // indirect + golang.org/x/text v0.44.0 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20260928230214-8a89bd6388cc // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20260928230214-8a89bd6388cc // indirect + google.golang.org/protobuf v1.36.12 // indirect +) diff --git a/service/go.sum b/service/go.sum new file mode 100644 index 0000000..7499f5d --- /dev/null +++ b/service/go.sum @@ -0,0 +1,230 @@ +dario.cat/mergo v1.0.2 h1:85+piFYR1tMbRrLcDwR18y4UKJ3aH1Tbzi24VRW1TK8= +dario.cat/mergo v1.0.2/go.mod h1:E/hbnu0NxMFBjpMIE34DRGLWqDy0g5FuKDhCb31ngxA= +github.com/Masterminds/goutils v1.1.1 h1:5nUrii3FMTL5diU80unEVvNevw1nH4+ZV4DSLVJLSYI= +github.com/Masterminds/goutils v1.1.1/go.mod h1:8cTjp+g8YejhMuvIA5y2vz3BpJxksy863GQaJW2MFNU= +github.com/Masterminds/semver/v3 v3.5.0 h1:kQceYJfbupGfZOKZQg0kou0DgAKhzDg2NZPAwZ/2OOE= +github.com/Masterminds/semver/v3 v3.5.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM= +github.com/Masterminds/sprig/v3 v3.3.0 h1:mQh0Yrg1XPo6vjYXgtf5OtijNAKJRNcTdOOGZe3tPhs= +github.com/Masterminds/sprig/v3 v3.3.0/go.mod h1:Zy1iXRYNqNLUolqCpL4uhk6SHUMAOSCzdgBfDb35Lz0= +github.com/aws/aws-sdk-go-v2 v1.47.8 h1:fuClo7Un+Bi0EDTnSIPWxVEoJcze+wkGzNA4Vo3jRcQ= +github.com/aws/aws-sdk-go-v2 v1.47.8/go.mod h1:I8lFSH4MGrZiw2m7miCrRYHsgpC1+HW8uiVGOiPYPcw= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.26 h1:SJNA4XaM8Z09adeTHf/vME93jzl44lJV70pSawIBXac= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.26/go.mod h1:l0ME5pd2QytH8UZzETcrt90x9Bm+3TyBVeWfVEcZ82c= +github.com/aws/aws-sdk-go-v2/credentials v1.20.16 h1:cqRWTY4fcctJGqZedMC1Khu4h2VsgRODUOIZwn39n3Y= +github.com/aws/aws-sdk-go-v2/credentials v1.20.16/go.mod h1:x8JgCOAnbnEVQOLVX6lOtsyTrG2EHgiRNqzku9jiVxA= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.11 h1:LTxHSJ/rwTFfaXqi5mlKKHJOW+shFB+HQ7NrCa4OQ48= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.11/go.mod h1:GF0bVKZVQX4UeUtPmTuWZj3gw0eRCdkVhW0xFnSRLmU= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.11 h1:DGUM0yztI3Zz5kePulT/mMP/gVZFGTTISUiCcqSQtT0= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.11/go.mod h1:kbcdJgN9cgSzZlUqsMveDl9IfexJqOzCihXjQXTNaPk= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.11 h1:kOHCwgWFyfrz9GqnJct8knQw7Apyl3xHH+OGnome77s= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.11/go.mod h1:854cZkT2fdkdrzQklD49uzxkZnajXoelpVNgT/yk4J0= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.24 h1:jGzHjc2jDx08Rji1JTBtMxuXUDG7muTEw+ExYXqFzPg= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.24/go.mod h1:uJt75pK2cEyY80F5C+DK9YkCkJWsMkJViHxLYZRHl8Y= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.12 h1:hfO8FblXX93ZKHXSCqCjehwul1EtNJa9G8iFSOiJ+6w= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.12/go.mod h1:r8Eypc7ZRNdS7BsaLBUUdcXIEuKE6cVHZJ29u+IGaIc= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.11 h1:PO7Tx443ecMI00nIxDrT99DVcbeMBQ6BUtJlDMRnY24= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.11/go.mod h1:n/9RixikO12YvNomZLP8k4TgUSoEkT1W23QIrpFAPvw= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.11 h1:jNP7wZylDeSJhjc1nmjZBuG5deotp3cEYIfMazmzp+Y= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.11/go.mod h1:1US+8JBnF0xNm6GE6wM5u5ugFwSZpkd2686gMuql5tI= +github.com/aws/aws-sdk-go-v2/service/s3 v1.114.9 h1:MLHGyNy4melTyk9e63s+9QgLKCHPDJrP+1NfJhtU1g4= +github.com/aws/aws-sdk-go-v2/service/s3 v1.114.9/go.mod h1:zfUkiNK2JiYlR1/SnU8aJ8Gj1nMFLkPZ69rVLv6jWPo= +github.com/aws/smithy-go v1.28.8 h1:aEmdAq7gVCoGCnvf1NnELlu3I6D/nhqrC321FyFiQkQ= +github.com/aws/smithy-go v1.28.8/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc= +github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs= +github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c= +github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA= +github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0= +github.com/cenkalti/backoff/v5 v5.0.3 h1:ZN+IMa753KfX5hd8vVaMixjnqRZ3y8CuJKRKj1xcsSM= +github.com/cenkalti/backoff/v5 v5.0.3/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/ebitengine/purego v0.11.2 h1:WWiTx8rD8zYdOwSj8O/NiNbsOvSAgJMTlyV1XlbUSeU= +github.com/ebitengine/purego v0.11.2/go.mod h1:DCHPP08djqhNSoTfImcnHYQRZmd0qhakvrozqaEYhGQ= +github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= +github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= +github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k= +github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= +github.com/gabriel-vasile/mimetype v1.4.15 h1:05iP/CYtZ/w455R/KZM6rZ5ieAdh99UPtd+d3YzLmaI= +github.com/gabriel-vasile/mimetype v1.4.15/go.mod h1:azpTcoLcDZRNgFou5j+APrqQx9HqVPWa6ijYQIIVswQ= +github.com/go-chi/chi/v5 v5.5.0 h1:xdKQ1dJmVJsmgahrXSKhUx8NyVPzpwdlyrrQFpMPJgg= +github.com/go-chi/chi/v5 v5.5.0/go.mod h1:d1IG7GtKPSjPyBP365PH8sAGVHk3qfB/IhjFNRH1wWM= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.4 h1:tG4xh9yMsRCAiodLVTxyrkzSZ9+o0L1Kg/+cPVcbP/8= +github.com/go-logr/logr v1.4.4/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= +github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= +github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s= +github.com/go-playground/assert/v2 v2.2.0/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= +github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA= +github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY= +github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY= +github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= +github.com/go-playground/validator/v10 v10.30.5 h1:YyCXvVShZbs2Sm3Mb53eNOlhRXctSOzW5QJAouCTZL4= +github.com/go-playground/validator/v10 v10.30.5/go.mod h1:wEqiaov48pXX1kjhc3Da8y0M0Dtg/BK7gurFBLgwFrQ= +github.com/go-viper/mapstructure/v2 v2.5.0 h1:vM5IJoUAy3d7zRSVtIwQgBj7BiWtMPfmPEgAXnvj1Ro= +github.com/go-viper/mapstructure/v2 v2.5.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= +github.com/golang-jwt/jwt/v5 v5.3.1 h1:kYf81DTWFe7t+1VvL7eS+jKFVWaUnK9cB1qbwn63YCY= +github.com/golang-jwt/jwt/v5 v5.3.1/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.30.0 h1:/Tnpcb2E0Pz/tN9s3bfEY2Q8ePCEX9iuS+cneUwncnw= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.30.0/go.mod h1:zOBXOsUaBSjKgmH4OGzV1esUpR3oUSCPYVd2cUBjKYY= +github.com/huandu/xstrings v1.5.0 h1:2ag3IFq9ZDANvthTwTiqSSZLjDc+BedvHPAp5tJy2TI= +github.com/huandu/xstrings v1.5.0/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE= +github.com/jackc/pgerrcode v0.0.0-20250907135507-afb5586c32a6 h1:D/V0gu4zQ3cL2WKeVNVM4r2gLxGGf6McLwgXzRTo2RQ= +github.com/jackc/pgerrcode v0.0.0-20250907135507-afb5586c32a6/go.mod h1:a/s9Lp5W7n/DD0VrVoyJ00FbP2ytTPDVOivvn2bMlds= +github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= +github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= +github.com/jackc/pgx/v5 v5.11.0 h1:IzBBtyK9AHqf98cctWFifYSci2hgQR/cd56wB4p+ogg= +github.com/jackc/pgx/v5 v5.11.0/go.mod h1:mal1tBGAFfLHvZzaYh77YS/eC6IX9OWbRV1QIIM0Jn4= +github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo= +github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= +github.com/jackc/tern/v2 v2.4.3 h1:g293d3OZgW7OFEhsYXgEv0C21jea2boNr0VR4K5I7OY= +github.com/jackc/tern/v2 v2.4.3/go.mod h1:rMpMuRYcff5wWLptoTSO1qcDxJ4OodysvK17i2SVBys= +github.com/klauspost/cpuid/v2 v2.4.0 h1:S6Hrbc7+ywsr0r+RLapfGBHfyefhCTwEh3A0tV913Dw= +github.com/klauspost/cpuid/v2 v2.4.0/go.mod h1:19jmZ9mjzoF//ddRSUsv0zfBTJWh3QJh9FNxZTMrGxU= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/leodido/go-urn v1.5.0 h1:pLqT2kq1zpHW/1D18QMjMpdtX7cekxqtJJjg5ANyWw0= +github.com/leodido/go-urn v1.5.0/go.mod h1:9BORnCDhdPBJNDEX+w1bJisa8yOKYi116VeO96s4ifE= +github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 h1:6E+4a0GO5zZEnZ81pIr0yLvtUWk2if982qA3F3QD6H4= +github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I= +github.com/mitchellh/copystructure v1.2.0 h1:vpKXTN4ewci03Vljg/q9QvCGUDttBOGBIa15WveJJGw= +github.com/mitchellh/copystructure v1.2.0/go.mod h1:qLl+cE2AmVv+CoeAwDPye/v+N2HKCj9FbZEVFJRxO9s= +github.com/mitchellh/reflectwalk v1.0.2 h1:G2LzWKi524PWgd3mLHV8Y5k7s6XUvT0Gef6zxSIeXaQ= +github.com/mitchellh/reflectwalk v1.0.2/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= +github.com/oapi-codegen/runtime v1.6.0 h1:7Xx+GlueD6nRuyKoCPzL434Jfi3BetbiJOrzCHp/VPU= +github.com/oapi-codegen/runtime v1.6.0/go.mod h1:GwV7hC2hviaMzj+ITfHVRESK5J2W/GefVwIND/bMGvU= +github.com/pelletier/go-toml/v2 v2.4.3-0.20260624044523-35f78d5908f1 h1:gIZQQ2+wbsdQGWLPDSdJFptR25tBCjSC0KbSIQZ1rgE= +github.com/pelletier/go-toml/v2 v2.4.3-0.20260624044523-35f78d5908f1/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/power-devops/perfstat v0.0.0-20260805114148-88456608a4f6 h1:jL3a8soXdzuTCcRnKhOmtcsVOObdDTFf4O2B403HPRU= +github.com/power-devops/perfstat v0.0.0-20260805114148-88456608a4f6/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= +github.com/rabbitmq/amqp091-go v1.10.0 h1:STpn5XsHlHGcecLmMFCtg7mqq0RnD+zFr4uzukfVhBw= +github.com/rabbitmq/amqp091-go v1.10.0/go.mod h1:Hy4jKW5kQART1u+JkDTF9YYOQUHXqMuhrgxOEeS7G4o= +github.com/rabbitmq/amqp091-go v1.14.0 h1:RSaT7aOKt/OrkVUyswPDW29lnRz9psuGmfZFBmLqLek= +github.com/rabbitmq/amqp091-go v1.14.0/go.mod h1:Hy4jKW5kQART1u+JkDTF9YYOQUHXqMuhrgxOEeS7G4o= +github.com/redis/go-redis/v9 v9.23.0 h1:/TTVdJa8BRNsybwwGGnmiYktHNlUXtbOQxlWpHBPL3s= +github.com/redis/go-redis/v9 v9.23.0/go.mod h1:EWP2UUk+XDaI1s5nQB5mablZN6vccXlmBQTyfnF87D0= +github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= +github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= +github.com/sagikazarmark/locafero v0.11.0 h1:1iurJgmM9G3PA/I+wWYIOw/5SyBtxapeHDcg+AAIFXc= +github.com/sagikazarmark/locafero v0.11.0/go.mod h1:nVIGvgyzw595SUSUE6tvCp3YYTeHs15MvlmU87WwIik= +github.com/shirou/gopsutil/v4 v4.26.10 h1:MJN6euesJx53MVEjscYeyRSRcrdIcy3sz5k9nosjldg= +github.com/shirou/gopsutil/v4 v4.26.10/go.mod h1:XpkIrcUMQhwte/dGxu57KtnMMFsbKelI7PP23zweKBk= +github.com/shopspring/decimal v1.4.0 h1:bxl37RwXBklmTi0C79JfXCEBD1cqqHt0bbgBAGFp81k= +github.com/shopspring/decimal v1.4.0/go.mod h1:gawqmDU56v4yIKSwfBSFip1HdCCXN8/+DMd9qYNcwME= +github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8 h1:+jumHNA0Wrelhe64i8F6HNlS8pkoyMv5sreGx2Ry5Rw= +github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8/go.mod h1:3n1Cwaq1E1/1lhQhtRK2ts/ZwZEhjcQeJQ1RuC6Q/8U= +github.com/spf13/afero v1.15.0 h1:b/YBCLWAJdFWJTN9cLhiXXcD7mzKn9Dm86dNnfyQw1I= +github.com/spf13/afero v1.15.0/go.mod h1:NC2ByUVxtQs4b3sIUphxK0NioZnmxgyCrfzeuq8lxMg= +github.com/spf13/cast v1.10.0 h1:h2x0u2shc1QuLHfxi+cTJvs30+ZAHOGRic8uyGTDWxY= +github.com/spf13/cast v1.10.0/go.mod h1:jNfB8QC9IA6ZuY2ZjDp0KtFO2LZZlg4S/7bzP6qqeHo= +github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk= +github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/viper v1.21.0 h1:x5S+0EU27Lbphp4UKm1C+1oQO+rKx36vfCoaVebLFSU= +github.com/spf13/viper v1.21.0/go.mod h1:P0lhsswPGWD/1lZJ9ny3fYnVqxiegrlNrEmgLjbTCAY= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.12.1 h1:EuwCh5fleGS7H32xRwO3wRGT7DxrDhLAT6FF8MpWDWE= +github.com/stretchr/testify v1.12.1/go.mod h1:MDEgiDPPsNp5cuIrHPPCyornHKgEVbtFUmoNlxoYthg= +github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8= +github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= +github.com/tklauser/go-sysconf v0.4.0 h1:7H0uAN+7RkwWRaxhYXDLqa5V3LPrJeV8wmD9dRUgPQU= +github.com/tklauser/go-sysconf v0.4.0/go.mod h1:8mTNWyog7H+MpKijp4VmKJAd2bbYQ2zuUwkYRbUArPI= +github.com/tklauser/numcpus v0.12.0 h1:NR85qdvHA9pFse3x3weVZ0r0ST8R6l5RHbZrlRaqob4= +github.com/tklauser/numcpus v0.12.0/go.mod h1:ABHeXzJnr/qqwguhClkZKT1/8VABcYrsyUiUGobwWJg= +github.com/vgarvardt/pgx-google-uuid/v5 v5.6.0 h1:EhPtK0mgrgaTMXpegE69hvoSOVC1Ahk8+QJ9B8b+OdU= +github.com/vgarvardt/pgx-google-uuid/v5 v5.6.0/go.mod h1:5LtFrNEkgzxHvXPO9eOvcXsSn9/KeKYgx9kjeI2oXQI= +github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0= +github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= +github.com/zeebo/xxh3 v1.1.0 h1:s7DLGDK45Dyfg7++yxI0khrfwq9661w9EN78eP/UZVs= +github.com/zeebo/xxh3 v1.1.0/go.mod h1:IisAie1LELR4xhVinxWS5+zf1lA4p0MW4T+w+W07F5s= +go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= +go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= +go.opentelemetry.io/contrib/bridges/otelslog v0.21.0 h1:/CdgE52i1OEPHhsGUqrggYNRaZy4v4YgVenGd52dRBI= +go.opentelemetry.io/contrib/bridges/otelslog v0.21.0/go.mod h1:9pBdbVbuehIDzxN4accnzzxEUPT8Nq7grbZMo+J1VN4= +go.opentelemetry.io/otel v1.47.0 h1:j7ALJ/zgkS7Z6aeJW09p8VC9804bC+PpeTfCD4XPnOM= +go.opentelemetry.io/otel v1.47.0/go.mod h1:8wS9O2qfXrYrzp6hIF/HOYJJf/wIhFPhR2xLuP+iXQU= +go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.23.0 h1:5WVp40k2fJHuc5BtNTgKk+EugKFT8Eg3lywFfCPI/nE= +go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.23.0/go.mod h1:59nH0RYCGHRlwt15WyioeSYINBbuObeBpTowPhEKWHY= +go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.47.0 h1:7kszRN4Ze5GeAkUnBjBSUIunVBMA+mZIG04auueqguM= +go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.47.0/go.mod h1:CLzakYy8jOdkBfcveCWYkeKJxujcF4xPhh2Z7IfE98U= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.47.0 h1:julhjPeUH/q/7hinbSdDdqt5h7Zw9YWmRlWRhI0jd54= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.47.0/go.mod h1:Ao2mz688LH/tFf0yMAenidq6k2YNSx6SIY2q6jDACck= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.47.0 h1:aXZXsZ012wOgVkqaiQv+Z/d8V0xKjmg70F6m4CE94lc= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.47.0/go.mod h1:ziu1gUIJhAaxM3EV1e3Ralk5AAyL1UOYz3lJJ4VSI38= +go.opentelemetry.io/otel/log v1.47.0 h1:cOTS1CcLbSQeZKanGJ+0JpF/+t4PELi3O3bbl2lqCcI= +go.opentelemetry.io/otel/log v1.47.0/go.mod h1:9byitSQ5pLC6PpqwGXjqdMKya6ZTswHRZh2vvXT33nw= +go.opentelemetry.io/otel/metric v1.47.0 h1:4PptaldXx3Eat1XjMZ68pPJEs5wrhlemctZE9a3UdWY= +go.opentelemetry.io/otel/metric v1.47.0/go.mod h1:ADGSXxRrXM6bjbvLo535EstVFlPpPYZm4LBKixjDHwU= +go.opentelemetry.io/otel/metric/x v0.69.0 h1:DjRLr15H83v+hCW7JA9NoJvOkYTtmq5YoDRbe9deYpM= +go.opentelemetry.io/otel/metric/x v0.69.0/go.mod h1:uVvsMPMFFyj/HUQfrUnH3JjnOQ1dwFDorgFLRBasM0k= +go.opentelemetry.io/otel/sdk v1.47.0 h1:zWXEr4j2lFefG87TU6Yg8a7ngfohIKFZHKp0Hf5hC6I= +go.opentelemetry.io/otel/sdk v1.47.0/go.mod h1:VUc24kiOeoGsxG8G9ULx3fWKvB7jMhnGE8Oi607lgR0= +go.opentelemetry.io/otel/sdk/log v1.47.0 h1:W4N/ZfgpkRRnmARnqYoQ6BrWA0mrj8IzVDNLlpmUJZY= +go.opentelemetry.io/otel/sdk/log v1.47.0/go.mod h1:Wu74Z2EmOKXi2FleSNAb0Hyau2G+r5kubSEMJw5aAMs= +go.opentelemetry.io/otel/sdk/log/logtest v0.23.0 h1:j3pm42QCrKYYLLkI6OrV1QHFkUfDA6/4EP+Y1PzggLg= +go.opentelemetry.io/otel/sdk/log/logtest v0.23.0/go.mod h1:zt0gEnuTZgDvqvczHwYswz93eqMaIr6upTTsdV/HUlU= +go.opentelemetry.io/otel/sdk/metric v1.47.0 h1:lfISg2j93VT6yqdk9OfUaZmw/GfcZqCCV3jdXtsPnKw= +go.opentelemetry.io/otel/sdk/metric v1.47.0/go.mod h1:ypLp+mW1Nt2x+Szt3b5/i1syodyts49lMOwxpDI3VGw= +go.opentelemetry.io/otel/trace v1.47.0 h1:JOjX/Oci8K94QHddo+bbfya/Ai/nf6/dt9ZfrFNWSrM= +go.opentelemetry.io/otel/trace v1.47.0/go.mod h1:jNaSLa2PZEYFG6fRjJABAu+bw4FS08uDmPg28lTghu0= +go.opentelemetry.io/proto/otlp v1.11.0 h1:5rrYs0Ykyj50sdU/JU0x8etU+LubXWb+gED6TbEdMIk= +go.opentelemetry.io/proto/otlp v1.11.0/go.mod h1:SmVizdCOAm3XBtG1g1NnOdhW6jtddT72hLMhv8VwA8E= +go.uber.org/atomic v1.12.0 h1:BvcXdFKuviU4fTL/f+SxdQ5qJX/Jix8pAkgdUcb3XOE= +go.uber.org/atomic v1.12.0/go.mod h1:I6c4cg+6HCxRjfjSsYtApoFILnpc0CGUdGkXVqbYVNk= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw= +go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg= +golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M= +golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA= +golang.org/x/crypto v0.59.0 h1:xS2jhqYbzNrpvkWP2YTDf4TJwBy3ykR03i8vFDxEc7c= +golang.org/x/crypto v0.59.0/go.mod h1:veDyUwATGs3c+vN3VfTbzViMGXeTBk3WyhBOkn+F/4U= +golang.org/x/net v0.60.0 h1:79p50tfZlm0J9YfoDsSi639qSXNGVwEzOPLCxM2FsYU= +golang.org/x/net v0.60.0/go.mod h1:2DA/G1UfVbCpQPeWTmMPGY7Cs2PkBkwu743bVX5PIVg= +golang.org/x/net v0.62.0 h1:18ELalfM5+XNRwzM2nYqXsZQ9ByS1uhj2O/U7+5jqRY= +golang.org/x/net v0.62.0/go.mod h1:WgDCOTH8iwtB66T4/GfI94AHo/hhdb7Hah9NDseQfiI= +golang.org/x/sync v0.23.0 h1:KameEIfc1IkluZyXWLn39Wd4tURc6GbCiISGiZm2bQk= +golang.org/x/sync v0.23.0/go.mod h1:sUUOizhqBxiL6pEWpqNLUiaJn1ShEbZ6BBqskPbjZm0= +golang.org/x/sync v0.25.0 h1:dJnifr0nMV8/SBnBvdo4eSGGnrc3O4FFx6IqvtLcmRs= +golang.org/x/sync v0.25.0/go.mod h1:sUUOizhqBxiL6pEWpqNLUiaJn1ShEbZ6BBqskPbjZm0= +golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo= +golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= +golang.org/x/sys v0.50.0 h1:dkKZEFTEKaLr1sjhs0WiKqoi8Jj5nHbS3Njmx1UoMJs= +golang.org/x/sys v0.50.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= +golang.org/x/text v0.42.0 h1:JbOZXgfeCPU9gacVtYliJqOhD+zhrEqK4LfdpmlUZqI= +golang.org/x/text v0.42.0/go.mod h1:ojzP1Z+2QtioaF8DTtO8K5q7JWVVYwZKenzujK0Zd0E= +golang.org/x/text v0.44.0 h1:YItdp/mQNGnsmrgpcdA5yQ3bnRxCfo/Tj/iByGW9GGo= +golang.org/x/text v0.44.0/go.mod h1:S4FR1waQ9LbjQTls97SyRjDaUUVkaM+cXwwuG/G+MwY= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= +gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= +google.golang.org/genproto/googleapis/api v0.0.0-20260928230214-8a89bd6388cc h1:rZHRz0ogq4Pid1IKtA/ivHi8XONVF5tqUQXA2TtgpkE= +google.golang.org/genproto/googleapis/api v0.0.0-20260928230214-8a89bd6388cc/go.mod h1:K/n0XLazCJHjD73YuMyNJsN38BPPy1kfFnKxUXaThFU= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260928230214-8a89bd6388cc h1:4bNTbnb44EqGVy9HaQxSY2jnafSUdgV3qHtedvNpDKg= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260928230214-8a89bd6388cc/go.mod h1:OaIUM3+LpYcK2GXM4FTmhWoIq371Owdr+Cc7/BsYHHc= +google.golang.org/grpc v1.85.0 h1:BPBCfuVRQ1qvBhKDOq9uvQllMvO59rKsYhXpp7Dd8vw= +google.golang.org/grpc v1.85.0/go.mod h1:0rQrOD58yjal/0EaVbGBANindixsCZNQPDV3VKtaKuQ= +google.golang.org/protobuf v1.36.12 h1:pJOKDDOyeXErUroCihFAd5LQuwXBSpVnKGrj5o/fwxc= +google.golang.org/protobuf v1.36.12/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/service/internal/adapter/controller/http/attachment.go b/service/internal/adapter/controller/http/attachment.go new file mode 100644 index 0000000..d02cfda --- /dev/null +++ b/service/internal/adapter/controller/http/attachment.go @@ -0,0 +1 @@ +package http diff --git a/service/internal/adapter/controller/http/attachmentHandler.go b/service/internal/adapter/controller/http/attachmentHandler.go new file mode 100644 index 0000000..d286d53 --- /dev/null +++ b/service/internal/adapter/controller/http/attachmentHandler.go @@ -0,0 +1,111 @@ +package http + +import ( + "gitea.privcloud.tech/mtusi2026/internal/dto" + "gitea.privcloud.tech/mtusi2026/internal/pkg/apperr" + "gitea.privcloud.tech/mtusi2026/internal/pkg/auth/jwtprovider" + "gitea.privcloud.tech/mtusi2026/internal/pkg/rest/framework" + "gitea.privcloud.tech/mtusi2026/internal/pkg/rest/router" + "gitea.privcloud.tech/mtusi2026/internal/service/attachment" + "io" + "mime" + "net/http" + "strconv" + "strings" +) + +type AttachmentHandler struct { + router router.Router + provider jwtprovider.Provider + svc *attachment.Service +} + +func NewAttachmentHandler(r router.Router, p jwtprovider.Provider, s *attachment.Service) Handler { + return &AttachmentHandler{r, p, s} +} +func (h *AttachmentHandler) RegisterRoutes() { + h.router.Private("POST", "/api/v1/chats/{chat_id}/attachments", h.Upload) + h.router.Private("GET", "/api/v1/attachments/{attachment_id}", h.Get) + h.router.Private("GET", "/api/v1/attachments/{attachment_id}/content", h.Get) +} +func (h *AttachmentHandler) Upload(w http.ResponseWriter, r *http.Request) { + user, e := framework.ParseToken(r, h.provider) + if e != nil { + framework.JSONError(r.Context(), w, e) + return + } + id, e := framework.ParseIDFromPathParam(r.Context(), w, r, "chat_id") + if e != nil { + return + } + values := r.URL.Query()["filename"] + if len(values) != 1 { + framework.JSONError(r.Context(), w, apperr.New(apperr.CodeBadRequest, "expected one filename")) + return + } + if !attachment.ValidFilename(values[0]) { + framework.JSONError(r.Context(), w, apperr.New(apperr.CodeBadRequest, "invalid filename")) + return + } + mt, _, e := mime.ParseMediaType(r.Header.Get("Content-Type")) + if e != nil || (mt != "image/png" && mt != "image/jpeg") { + framework.JSONError(r.Context(), w, apperr.New(apperr.CodeUnsupportedMediaType, "expected PNG or JPEG")) + return + } + if r.ContentLength > h.svc.Config.MaxBytes { + framework.JSONError(r.Context(), w, apperr.New(apperr.CodePayloadTooLarge, "upload too large")) + return + } + a, e := h.svc.Upload(r.Context(), dto.UploadAttachment{UserID: user.UserID, ChatID: id, Filename: values[0], ContentType: mt, Body: io.LimitReader(r.Body, h.svc.Config.MaxBytes+1)}) + if e != nil { + if apperr.CodeFromError(e) == apperr.CodeTooManyRequests { + w.Header().Set("Retry-After", "1") + } + framework.JSONError(r.Context(), w, e) + return + } + framework.JSONResponse(w, http.StatusAccepted, a) +} +func (h *AttachmentHandler) Get(w http.ResponseWriter, r *http.Request) { + user, e := framework.ParseToken(r, h.provider) + if e != nil { + framework.JSONError(r.Context(), w, e) + return + } + id, e := framework.ParseIDFromPathParam(r.Context(), w, r, "attachment_id") + if e != nil { + return + } + + if !strings.HasSuffix(r.URL.Path, "/content") { + a, e := h.svc.Get(r.Context(), user.UserID, id) + if e != nil { + framework.JSONError(r.Context(), w, e) + return + } + framework.JSONResponse(w, 200, a.Public) + return + } + v := r.URL.Query().Get("variant") + if v == "" { + v = "original" + } + if v != "original" && v != "thumbnail" && v != "crop" && v != "optimized" { + framework.JSONError(r.Context(), w, apperr.New(apperr.CodeBadRequest, "invalid variant")) + return + } + obj, e := h.svc.Content(r.Context(), user.UserID, id, v) + if e != nil { + framework.JSONError(r.Context(), w, e) + return + } + defer obj.Body.Close() + w.Header().Set("Content-Type", obj.ContentType) + w.Header().Set("Content-Length", strconv.FormatInt(obj.Size, 10)) + w.Header().Set("X-Content-Type-Options", "nosniff") + // Once streaming has started a transport failure terminates the response; no JSON + // error can be appended to an image. Content-Length exposes incomplete downloads. + if _, e = io.Copy(w, obj.Body); e != nil { + panic(http.ErrAbortHandler) + } +} diff --git a/service/internal/adapter/controller/http/auth.go b/service/internal/adapter/controller/http/auth.go new file mode 100644 index 0000000..b84563c --- /dev/null +++ b/service/internal/adapter/controller/http/auth.go @@ -0,0 +1,106 @@ +package http + +import ( + "net" + "net/http" + "strings" + "unicode/utf8" + + "gitea.privcloud.tech/mtusi2026/internal/dto" + "gitea.privcloud.tech/mtusi2026/internal/gen/contracts" + "gitea.privcloud.tech/mtusi2026/internal/pkg/apperr" + "gitea.privcloud.tech/mtusi2026/internal/pkg/rest/framework" + "gitea.privcloud.tech/mtusi2026/internal/pkg/rest/router" + "gitea.privcloud.tech/mtusi2026/internal/service/auth" +) + +type AuthHandler struct { + router router.Router + + auth auth.API +} + +func NewAuthHandler(r router.Router, api auth.API) Handler { + return &AuthHandler{ + router: r, + auth: api, + } +} + +func (h *AuthHandler) RegisterRoutes() { + h.router.Public(http.MethodPost, "/api/v1/auth/sessions", h.Login) + h.router.Private(http.MethodDelete, "/api/v1/auth/sessions/current", h.Logout) +} + +func (h *AuthHandler) Login(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + + username, password, ok := r.BasicAuth() + if !ok || username == "" || utf8.RuneCountInString(password) < 12 || utf8.RuneCountInString(password) > 128 || !utf8.ValidString(password) { + w.Header().Set( + "WWW-Authenticate", + `Basic realm="messenger", charset="UTF-8"`, + ) + framework.JSONError(ctx, w, apperr.New(apperr.CodeUnauthorized, "invalid credentials")) + return + } + + if r.ContentLength != 0 { + framework.JSONError(ctx, w, apperr.New(apperr.CodeBadRequest, "login body must be empty")) + return + } + clientIP, _, _ := net.SplitHostPort(r.RemoteAddr) + // This listener requires the LB client identity; LB overwrites this header. + if r.TLS != nil { + if forwarded := r.Header.Get("X-Forwarded-For"); net.ParseIP(strings.TrimSpace(forwarded)) != nil { + clientIP = strings.TrimSpace(forwarded) + } + } + resp, err := h.auth.Login(ctx, dto.Login{ + Username: username, + ClientIP: clientIP, + Password: password, + }) + if err != nil { + if apperr.HTTPStatusFromError(err) == 429 { + w.Header().Set("Retry-After", "60") + } + if apperr.HTTPStatusFromError(err) == http.StatusUnauthorized { + w.Header().Set("WWW-Authenticate", `Basic realm="messenger", charset="UTF-8"`) + } + framework.JSONError(ctx, w, err) + return + } + + framework.JSONResponse(w, http.StatusCreated, contracts.Session{ + ExpiresAt: resp.ExpiresAt, + Token: resp.Tokens.RefreshToken.Payload, + TokenType: resp.TokenType, + User: contracts.User{ + CreatedAt: resp.User.CreatedAt, + DisplayName: resp.User.DisplayName, + Id: resp.User.ID, + Username: resp.User.Username, + }, + }) +} + +func (h *AuthHandler) Logout(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + + token, exists := framework.Token(r) + if !exists { + framework.JSONError(ctx, w, apperr.New(apperr.CodeUnauthorized, "missing token in header")) + return + } + + err := h.auth.Logout(ctx, dto.RefreshToken{ + Payload: token, + }) + if err != nil { + framework.JSONError(ctx, w, err) + return + } + + framework.JSONResponse(w, http.StatusNoContent, nil) +} diff --git a/service/internal/adapter/controller/http/chat.go b/service/internal/adapter/controller/http/chat.go new file mode 100644 index 0000000..dcf583d --- /dev/null +++ b/service/internal/adapter/controller/http/chat.go @@ -0,0 +1,287 @@ +package http + +import ( + "bytes" + "encoding/json" + "net/http" + "regexp" + "strings" + "unicode/utf8" + + "github.com/google/uuid" + + "gitea.privcloud.tech/mtusi2026/internal/dto" + "gitea.privcloud.tech/mtusi2026/internal/gen/contracts" + "gitea.privcloud.tech/mtusi2026/internal/pkg/apperr" + "gitea.privcloud.tech/mtusi2026/internal/pkg/auth/jwtprovider" + "gitea.privcloud.tech/mtusi2026/internal/pkg/rest/framework" + "gitea.privcloud.tech/mtusi2026/internal/pkg/rest/router" + "gitea.privcloud.tech/mtusi2026/internal/pkg/validator" + "gitea.privcloud.tech/mtusi2026/internal/service/chat" +) + +var messageKeyPattern = regexp.MustCompile(`^[A-Za-z0-9._:-]{1,128}$`) + +type ChatHandler struct { + router router.Router + provider jwtprovider.Provider + validator *validator.ValidatorImpl + + chat chat.API +} + +func NewChatHandler(r router.Router, provider jwtprovider.Provider, v *validator.ValidatorImpl, api chat.API) Handler { + return &ChatHandler{ + router: r, + provider: provider, + validator: v, + chat: api, + } +} + +func (h *ChatHandler) RegisterRoutes() { + h.router.Private(http.MethodPost, "/api/v1/chats", h.Create) + h.router.Private(http.MethodGet, "/api/v1/chats", h.Get) + h.router.Private(http.MethodGet, "/api/v1/chats/{chat_id}", h.GetByID) + h.router.Private(http.MethodPost, "/api/v1/chats/{chat_id}/messages", h.SendMessage) + h.router.Private(http.MethodGet, "/api/v1/chats/{chat_id}/messages", h.GetMessages) +} + +func (h *ChatHandler) Create(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + + userData, err := framework.ParseToken(r, h.provider) + if err != nil { + framework.JSONError(ctx, w, err) + return + } + + var req contracts.CreateChat + if !framework.Decode(w, r, &req) || !framework.Validate(w, r, h.validator, &req) { + return + } + + if userData.UserID == uuid.Nil || + !validText(req.Title, 100) || + len(req.MemberIds) < 1 || + len(req.MemberIds) > 100 { + framework.JSONError(ctx, w, apperr.New(apperr.CodeBadRequest, "invalid chat fields")) + return + } + + resp, err := h.chat.CreateChat(ctx, dto.CreateChat{ + CreatorID: userData.UserID, + Title: req.Title, + MemberIDs: req.MemberIds, + }) + if err != nil { + framework.JSONError(ctx, w, err) + return + } + + framework.JSONResponse(w, http.StatusCreated, contracts.Chat{ + Id: resp.ID, + Title: resp.Title, + MemberIds: resp.MemberIDs, + CreatedAt: resp.CreatedAt, + }) +} + +func (h *ChatHandler) Get(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + + userData, err := framework.ParseToken(r, h.provider) + if err != nil { + framework.JSONError(ctx, w, err) + return + } + + limit, afterID, ok := framework.PageParams(w, r) + if !ok { + return + } + + if userData.UserID == uuid.Nil { + framework.JSONError(ctx, w, apperr.New(apperr.CodeBadRequest, "invalid user id")) + return + } + if afterID != nil && *afterID == uuid.Nil { + framework.JSONError(ctx, w, apperr.New(apperr.CodeBadRequest, "invalid cursor")) + return + } + + page, err := h.chat.ListForMember(ctx, dto.ChatFilters{ + UserID: userData.UserID, + Limit: limit, + AfterID: afterID, + }) + if err != nil { + framework.JSONError(ctx, w, err) + return + } + + items := make([]contracts.Chat, 0, len(page.Items)) + for _, item := range page.Items { + items = append(items, contracts.Chat{ + CreatedAt: item.CreatedAt, + Id: item.ID, + MemberIds: item.MemberIDs, + Title: item.Title, + }) + } + + framework.JSONResponse(w, http.StatusOK, contracts.ChatPage{ + Items: items, + NextCursor: cursorString(page.NextCursorID), + }) +} + +func (h *ChatHandler) GetByID(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + + userData, err := framework.ParseToken(r, h.provider) + if err != nil { + framework.JSONError(ctx, w, err) + return + } + + chatID, err := framework.ParseIDFromPathParam(ctx, w, r, "chat_id") + if err != nil { + return + } + + resp, err := h.chat.GetForMember(ctx, dto.ChatMember{ + UserID: userData.UserID, + ChatID: chatID, + }) + if err != nil { + framework.JSONError(ctx, w, err) + return + } + + framework.JSONResponse(w, http.StatusOK, contracts.Chat{ + CreatedAt: resp.CreatedAt, + Id: resp.ID, + MemberIds: resp.MemberIDs, + Title: resp.Title, + }) +} + +func (h *ChatHandler) SendMessage(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + + userData, err := framework.ParseToken(r, h.provider) + if err != nil { + framework.JSONError(ctx, w, err) + return + } + + chatID, err := framework.ParseIDFromPathParam(ctx, w, r, "chat_id") + if err != nil { + return + } + + keyValues := r.Header.Values("Idempotency-Key") + if len(keyValues) != 1 || !messageKeyPattern.MatchString(keyValues[0]) { + framework.JSONError(ctx, w, apperr.New(apperr.CodeBadRequest, "invalid idempotency key")) + return + } + + var req struct { + Text *string `json:"text"` + AttachmentIDs json.RawMessage `json:"attachment_ids"` + } + if !framework.Decode(w, r, &req) || !framework.Validate(w, r, h.validator, &req) { + return + } + + if req.Text == nil { + framework.JSONError(ctx, w, apperr.New(apperr.CodeBadRequest, "text is required")) + return + } + + ids := []uuid.UUID{} + if len(req.AttachmentIDs) > 0 { + if bytes.Equal(bytes.TrimSpace(req.AttachmentIDs), []byte("null")) || json.Unmarshal(req.AttachmentIDs, &ids) != nil { + framework.JSONError(ctx, w, apperr.New(apperr.CodeBadRequest, "invalid attachment_ids")) + return + } + } + + message, err := h.chat.CreateMessage(ctx, dto.CreateMessage{ + IdempotencyKey: keyValues[0], + ChatID: chatID, + SenderID: userData.UserID, + Text: *req.Text, + AttachmentIDs: ids, + }) + if err != nil { + framework.JSONError(ctx, w, err) + return + } + + framework.JSONResponse(w, http.StatusCreated, contracts.Message{ + Id: message.ID, + ChatId: message.ChatID, + SenderId: message.SenderID, + Text: message.Text, + CreatedAt: message.CreatedAt, + AttachmentIds: message.AttachmentIDs, + }) +} + +func (h *ChatHandler) GetMessages(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + + userData, err := framework.ParseToken(r, h.provider) + if err != nil { + framework.JSONError(ctx, w, err) + return + } + + chatID, err := framework.ParseIDFromPathParam(ctx, w, r, "chat_id") + if err != nil { + return + } + + limit, afterID, ok := framework.PageParams(w, r) + if !ok { + return + } + + page, err := h.chat.ListForChat(ctx, dto.MessageFilters{ + ChatID: chatID, + UserID: userData.UserID, + Limit: limit, + AfterID: afterID, + }) + if err != nil { + framework.JSONError(ctx, w, err) + return + } + + items := make([]contracts.Message, 0, len(page.Items)) + for _, message := range page.Items { + items = append(items, contracts.Message{ + Id: message.ID, + ChatId: message.ChatID, + SenderId: message.SenderID, + Text: message.Text, + CreatedAt: message.CreatedAt, + AttachmentIds: message.AttachmentIDs, + }) + } + framework.JSONResponse(w, http.StatusOK, contracts.MessagePage{Items: items, NextCursor: cursorString(page.NextCursorID)}) +} + +func cursorString(id *uuid.UUID) *string { + if id == nil { + return nil + } + value := id.String() + return &value +} + +func validText(value string, maxLength int) bool { + return utf8.ValidString(value) && strings.TrimSpace(value) != "" && utf8.RuneCountInString(value) <= maxLength +} diff --git a/service/internal/adapter/controller/http/controller.go b/service/internal/adapter/controller/http/controller.go new file mode 100644 index 0000000..18064af --- /dev/null +++ b/service/internal/adapter/controller/http/controller.go @@ -0,0 +1,113 @@ +package http + +import ( + "context" + "crypto/tls" + "errors" + "gitea.privcloud.tech/mtusi2026/internal/pkg/transportsecurity" + "log/slog" + "net" + "net/http" + "sync/atomic" + + "gitea.privcloud.tech/mtusi2026/internal/config" + "gitea.privcloud.tech/mtusi2026/internal/pkg/rest/router" +) + +type Server interface { + ShouldBeRunning() bool + Ready() bool + Start(ctx context.Context) error + Stop(ctx context.Context) error +} + +type server struct { + server http.Server + cfg *config.Server + ready atomic.Bool +} + +func New(cfg *config.Config, router router.Router) Server { + s := server{ + server: http.Server{ + Addr: cfg.Server.BindAddress(), + Handler: router, + ReadTimeout: cfg.Server.ReadTimeout, + WriteTimeout: cfg.Server.WriteTimeout, + IdleTimeout: cfg.Server.IdleTimeout, + }, + cfg: cfg.Server, + ready: atomic.Bool{}, + } + + if err := s.validate(); err != nil { + panic(err) + } + + return &s +} + +func (s *server) Start(ctx context.Context) error { + if !s.ShouldBeRunning() { + slog.Info("server should not be running: skip starting") + return nil + } + listener, err := net.Listen("tcp", s.server.Addr) + if err != nil { + return err + } + + if transportsecurity.Enabled() { + cfg, e := transportsecurity.Server("lb") + if e != nil { + listener.Close() + return e + } + listener = tls.NewListener(listener, cfg) + } + s.server.BaseContext = func(net.Listener) context.Context { return ctx } + s.ready.Store(true) + go func() { + defer s.ready.Store(false) + if err := s.server.Serve(listener); err != nil && + !errors.Is(err, http.ErrServerClosed) { + slog.Error("rest server failed", "error", err) + } + }() + return nil +} + +func (s *server) Stop(ctx context.Context) error { + if !s.ShouldBeRunning() { + slog.Info("server should not be running: skip stopping") + return nil + } + + slog.Info("shutting down rest server...", "address", s.server.Addr) + s.ready.Store(false) + if err := s.server.Shutdown(ctx); err != nil { + slog.Error("rest server shutdown failed", "error", err) + _ = s.server.Close() + return err + } + slog.Info("rest server shutdown successfully") + return nil +} + +func (s *server) ShouldBeRunning() bool { + return s.cfg.Enabled +} + +func (s *server) Ready() bool { + return s.ready.Load() +} + +func (s *server) validate() error { + if s.cfg == nil { + slog.Error("failed to validate server: provided nil config") + return errors.New("failed to validate server: provided nil config") + } + + slog.Info("server successfully validated") + return nil +} diff --git a/service/internal/adapter/controller/http/handler.go b/service/internal/adapter/controller/http/handler.go new file mode 100644 index 0000000..275bee0 --- /dev/null +++ b/service/internal/adapter/controller/http/handler.go @@ -0,0 +1,5 @@ +package http + +type Handler interface { + RegisterRoutes() +} diff --git a/service/internal/adapter/controller/http/health.go b/service/internal/adapter/controller/http/health.go new file mode 100644 index 0000000..4d021a7 --- /dev/null +++ b/service/internal/adapter/controller/http/health.go @@ -0,0 +1,60 @@ +package http + +import ( + "context" + "net/http" + "time" + + "gitea.privcloud.tech/mtusi2026/internal/gen/contracts" + "gitea.privcloud.tech/mtusi2026/internal/pkg/apperr" + "gitea.privcloud.tech/mtusi2026/internal/pkg/rest/framework" + "gitea.privcloud.tech/mtusi2026/internal/pkg/rest/router" +) + +type databasePinger interface{ Ping(context.Context) error } + +type HealthHandler struct { + router router.Router + server Server + database databasePinger + storage []databasePinger +} + +func NewHealthHandler(r router.Router, server Server, database databasePinger, storage ...databasePinger) Handler { + return &HealthHandler{ + router: r, + server: server, + database: database, + storage: storage, + } +} + +func (h *HealthHandler) RegisterRoutes() { + h.router.Public(http.MethodGet, "/health/ready", h.Ready) + h.router.Public(http.MethodGet, "/health/live", h.Live) +} + +func (h *HealthHandler) Live(w http.ResponseWriter, _ *http.Request) { + framework.JSONResponse(w, http.StatusOK, contracts.Health{Status: contracts.Ok}) +} + +func (h *HealthHandler) Ready(w http.ResponseWriter, r *http.Request) { + if !h.server.Ready() { + framework.JSONError(r.Context(), w, apperr.New(apperr.CodeUnavailable, "server is not ready")) + return + } + ctx, cancel := context.WithTimeout(r.Context(), 200*time.Millisecond) + defer cancel() + if err := h.database.Ping(ctx); err != nil { + framework.JSONError(r.Context(), w, apperr.New(apperr.CodeUnavailable, "database unavailable")) + return + } + for _, dependency := range h.storage { + if err := dependency.Ping(ctx); err != nil { + framework.JSONError(r.Context(), w, apperr.New(apperr.CodeUnavailable, "storage unavailable")) + return + } + } + + framework.JSONResponse(w, http.StatusOK, contracts.Health{Status: contracts.Ok}) +} diff --git a/service/internal/adapter/controller/http/router.go b/service/internal/adapter/controller/http/router.go new file mode 100644 index 0000000..d02cfda --- /dev/null +++ b/service/internal/adapter/controller/http/router.go @@ -0,0 +1 @@ +package http diff --git a/service/internal/adapter/controller/http/user.go b/service/internal/adapter/controller/http/user.go new file mode 100644 index 0000000..7fdb640 --- /dev/null +++ b/service/internal/adapter/controller/http/user.go @@ -0,0 +1,113 @@ +package http + +import ( + "net/http" + "regexp" + "strings" + "unicode/utf8" + + "github.com/google/uuid" + + "gitea.privcloud.tech/mtusi2026/internal/dto" + "gitea.privcloud.tech/mtusi2026/internal/gen/contracts" + "gitea.privcloud.tech/mtusi2026/internal/pkg/apperr" + "gitea.privcloud.tech/mtusi2026/internal/pkg/auth/jwtprovider" + "gitea.privcloud.tech/mtusi2026/internal/pkg/rest/framework" + "gitea.privcloud.tech/mtusi2026/internal/pkg/rest/router" + "gitea.privcloud.tech/mtusi2026/internal/pkg/validator" + "gitea.privcloud.tech/mtusi2026/internal/service/user" +) + +type UserHandler struct { + router router.Router + provider jwtprovider.Provider + validator *validator.ValidatorImpl + + user user.API +} + +func NewUserHandler(r router.Router, provider jwtprovider.Provider, v *validator.ValidatorImpl, api user.API) Handler { + return &UserHandler{ + router: r, + provider: provider, + validator: v, + user: api, + } +} + +var usernamePattern = regexp.MustCompile(`^[a-z0-9_]{3,32}$`) + +func (h *UserHandler) RegisterRoutes() { + h.router.Public(http.MethodPost, "/api/v1/users", h.Create) + h.router.Private(http.MethodGet, "/api/v1/users/me", h.GetMe) +} + +func (h *UserHandler) Create(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + + var req contracts.CreateUser + if !framework.Decode(w, r, &req) || !framework.Validate(w, r, h.validator, &req) { + return + } + + toCreate := dto.CreateUser{ + Username: req.Username, + DisplayName: req.DisplayName, + } + + if req.Password != nil { + toCreate.Password = *req.Password + } + + if !usernamePattern.MatchString(req.Username) || !utf8.ValidString(req.DisplayName) || + strings.TrimSpace(req.DisplayName) == "" || utf8.RuneCountInString(req.DisplayName) > 100 { + framework.JSONError(ctx, w, apperr.New(apperr.CodeBadRequest, "invalid user fields")) + return + } + passwordLen := utf8.RuneCountInString(toCreate.Password) + if !utf8.ValidString(toCreate.Password) || strings.TrimSpace(toCreate.Password) == "" || passwordLen < 12 || passwordLen > 128 { + framework.JSONError(ctx, w, apperr.New(apperr.CodeBadRequest, "invalid password")) + return + } + + resp, err := h.user.Create(ctx, toCreate) + if err != nil { + framework.JSONError(ctx, w, err) + return + } + + framework.JSONResponse(w, http.StatusCreated, contracts.User{ + CreatedAt: resp.CreatedAt, + DisplayName: resp.DisplayName, + Id: resp.ID, + Username: resp.Username, + }) +} + +func (h *UserHandler) GetMe(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + + userData, err := framework.ParseToken(r, h.provider) + if err != nil { + framework.JSONError(ctx, w, err) + return + } + + if userData.UserID == uuid.Nil { + framework.JSONError(ctx, w, apperr.New(apperr.CodeBadRequest, "invalid user id")) + return + } + + resp, err := h.user.GetByID(ctx, userData.UserID) + if err != nil { + framework.JSONError(ctx, w, err) + return + } + + framework.JSONResponse(w, http.StatusOK, contracts.User{ + CreatedAt: resp.CreatedAt, + DisplayName: resp.DisplayName, + Id: resp.ID, + Username: resp.Username, + }) +} diff --git a/service/internal/adapter/repository/cache/outside/chat.go b/service/internal/adapter/repository/cache/outside/chat.go new file mode 100644 index 0000000..a5eb60c --- /dev/null +++ b/service/internal/adapter/repository/cache/outside/chat.go @@ -0,0 +1,78 @@ +package outside + +import ( + "context" + "encoding/json" + "fmt" + "time" + + "github.com/google/uuid" + + "gitea.privcloud.tech/mtusi2026/internal/dto" + "gitea.privcloud.tech/mtusi2026/internal/pkg/apperr" + "gitea.privcloud.tech/mtusi2026/internal/pkg/cache" +) + +const ( + // chatRedisKey equals "chat:{chat_id}" path + chatRedisKey = "chat:%s" +) + +type ChatCache struct { + client *cache.Client +} + +func NewChatCache(client *cache.Client) *ChatCache { + return &ChatCache{ + client: client, + } +} + +type chat struct { + ID uuid.UUID `json:"id"` + Title string `json:"title"` + MemberIDs []uuid.UUID `json:"member_ids"` + CreatedAt time.Time `json:"created_at"` +} + +func (r *ChatCache) Create(ctx context.Context, req dto.Chat) error { + data, err := json.Marshal(chat{ + ID: req.ID, + Title: req.Title, + MemberIDs: req.MemberIDs, + CreatedAt: req.CreatedAt, + }) + if err != nil { + return apperr.Wrap(err, apperr.CodeInternal, "failed to marshal chat cache") + } + + ctx, cancel := context.WithTimeout(ctx, cache.OperationTimeout) + defer cancel() + err = r.client.Set(ctx, fmt.Sprintf(chatRedisKey, req.ID), data, 0).Err() + if err != nil { + return apperr.WrapRedis(err, "failed to create chat cache") + } + + return nil +} + +func (r *ChatCache) Get(ctx context.Context, chatID uuid.UUID) (*dto.Chat, error) { + ctx, cancel := context.WithTimeout(ctx, cache.OperationTimeout) + defer cancel() + data, err := r.client.Get(ctx, fmt.Sprintf(chatRedisKey, chatID.String())).Bytes() + if err != nil { + return nil, apperr.WrapRedis(err, "failed to read chat cache") + } + + var item chat + if err = json.Unmarshal(data, &item); err != nil { + return nil, apperr.Wrap(err, apperr.CodeInternal, "failed to unmarshal chat cache") + } + + return &dto.Chat{ + ID: item.ID, + Title: item.Title, + MemberIDs: item.MemberIDs, + CreatedAt: item.CreatedAt, + }, nil +} diff --git a/service/internal/adapter/repository/storage/attachment_scan.go b/service/internal/adapter/repository/storage/attachment_scan.go new file mode 100644 index 0000000..5a10537 --- /dev/null +++ b/service/internal/adapter/repository/storage/attachment_scan.go @@ -0,0 +1,39 @@ +package storage + +import ( + "context" + "gitea.privcloud.tech/mtusi2026/internal/pkg/apperr" + "github.com/google/uuid" + "github.com/jackc/pgx/v5/pgxpool" +) + +func (r *AttachmentsRepo) Page(ctx context.Context, after uuid.UUID) ([]*AttachmentRecord, error) { + rows, e := r.Pool.Query(ctx, "SELECT "+AttachmentColumns+" FROM attachments WHERE id>$1 ORDER BY id LIMIT 100", after) + if e != nil { + return nil, apperr.WrapPostgres(e, "scan failed") + } + defer rows.Close() + out := make([]*AttachmentRecord, 0, 100) + for rows.Next() { + a, e := ScanAttachment(rows) + if e != nil { + return nil, e + } + out = append(out, a) + } + return out, rows.Err() +} +func (r *AttachmentsRepo) Exists(ctx context.Context, c *pgxpool.Conn, key string) (bool, error) { + var found bool + e := c.QueryRow(ctx, "SELECT EXISTS(SELECT 1 FROM attachment_variants WHERE object_key=$1 UNION ALL SELECT 1 FROM attachments WHERE object_key=$1 OR staging_key=$1 OR variant_keys @> jsonb_build_object('thumbnail',$1::text) OR variant_keys @> jsonb_build_object('crop',$1::text) OR variant_keys @> jsonb_build_object('optimized',$1::text))", key).Scan(&found) + return found, e +} +func (r *AttachmentsRepo) MarkMissing(ctx context.Context, c *pgxpool.Conn, id uuid.UUID, missing bool) (bool, error) { + tag, e := c.Exec(ctx, "UPDATE attachments SET integrity_missing=$2,updated_at=clock_timestamp() WHERE id=$1 AND integrity_missing<>$2", id, missing) + return tag.RowsAffected() == 1, e +} + +func (r *AttachmentsRepo) CheckConn(ctx context.Context, c *pgxpool.Conn) error { + _, e := c.Exec(ctx, "SELECT 1") + return e +} diff --git a/service/internal/adapter/repository/storage/attachments.go b/service/internal/adapter/repository/storage/attachments.go new file mode 100644 index 0000000..0a3b2bb --- /dev/null +++ b/service/internal/adapter/repository/storage/attachments.go @@ -0,0 +1,65 @@ +package storage + +import ( + "context" + "time" + + "github.com/google/uuid" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" + + "encoding/json" + "gitea.privcloud.tech/mtusi2026/internal/dto" + "gitea.privcloud.tech/mtusi2026/internal/pkg/apperr" +) + +type AttachmentRecord = dto.AttachmentRecord +type AttachmentsRepo struct{ Pool *pgxpool.Pool } + +const AttachmentColumns = "id,chat_id,owner_id,filename,content_type,size_bytes,sha256,created_at,updated_at,object_key,integrity_missing,status,error_code,pipeline_version,staging_key,variants,variant_keys" + +func ScanAttachment(row pgx.Row) (*AttachmentRecord, error) { + a := new(AttachmentRecord) + p := &a.Public + var variants, keys []byte + if err := row.Scan(&p.Id, &p.ChatId, &p.OwnerId, &p.Filename, &p.ContentType, &p.SizeBytes, &p.Sha256, &p.CreatedAt, &p.UpdatedAt, &a.Key, &a.Missing, &p.Status, &p.ErrorCode, &a.PipelineVersion, &a.StagingKey, &variants, &keys); err != nil { + return nil, apperr.WrapPostgres(err, "attachment unavailable") + } + if err := json.Unmarshal(variants, &p.Variants); err != nil { + return nil, err + } + if err := json.Unmarshal(keys, &a.VariantKeys); err != nil { + return nil, err + } + return a, nil +} +func (r *AttachmentsRepo) Get(ctx context.Context, id uuid.UUID) (*AttachmentRecord, error) { + return ScanAttachment(r.Pool.QueryRow(ctx, "SELECT "+AttachmentColumns+" FROM attachments WHERE id=$1", id)) +} + +// Session locks span S3 work and the database commit. A lost connection releases +// the lock; the mandatory grace period also exceeds the maximum upload lifetime. +func (r *AttachmentsRepo) Lock(ctx context.Context, key string) (*pgxpool.Conn, func(), error) { + c, e := r.Pool.Acquire(ctx) + if e != nil { + return nil, nil, apperr.WrapPostgres(e, "database unavailable") + } + if _, e = c.Exec(ctx, "SELECT pg_advisory_lock(hashtextextended($1, 42))", key); e != nil { + // The server may have acquired the session lock before cancellation was + // observed by the client. Never return that uncertain session to the pool. + clean, cancel := context.WithTimeout(context.Background(), 2*time.Second) + _ = c.Conn().Close(clean) + cancel() + c.Release() + return nil, nil, apperr.WrapPostgres(e, "lock unavailable") + } + return c, func() { + // Close rather than return a possibly locked session when unlock fails. + clean, cancel := context.WithTimeout(context.Background(), 2*time.Second) + defer cancel() + if _, e := c.Exec(clean, "SELECT pg_advisory_unlock(hashtextextended($1, 42))", key); e != nil { + _ = c.Conn().Close(clean) + } + c.Release() + }, nil +} diff --git a/service/internal/adapter/repository/storage/chats.go b/service/internal/adapter/repository/storage/chats.go new file mode 100644 index 0000000..15a7298 --- /dev/null +++ b/service/internal/adapter/repository/storage/chats.go @@ -0,0 +1,91 @@ +package storage + +import ( + "context" + "errors" + + "github.com/google/uuid" + "github.com/jackc/pgerrcode" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgconn" + "github.com/jackc/pgx/v5/pgxpool" + + "gitea.privcloud.tech/mtusi2026/internal/dto" + "gitea.privcloud.tech/mtusi2026/internal/pkg/apperr" +) + +type ChatsRepo struct { + pool *pgxpool.Pool +} + +func NewChatsRepo(pool *pgxpool.Pool) *ChatsRepo { + return &ChatsRepo{pool: pool} +} + +func (r *ChatsRepo) CreateTx(ctx context.Context, tx pgx.Tx, req dto.Chat) error { + const insertChat = `INSERT INTO chats (id, title, created_at) VALUES ($1, $2, $3)` + if _, err := tx.Exec(ctx, insertChat, req.ID, req.Title, req.CreatedAt); err != nil { + return apperr.WrapPostgres(err, "failed to create chat") + } + const insertMembers = `INSERT INTO chat_members (chat_id, user_id) +SELECT $1, user_id FROM unnest($2::uuid[]) AS members(user_id)` + if _, err := tx.Exec(ctx, insertMembers, req.ID, req.MemberIDs); err != nil { + if pgErr, ok := errors.AsType[*pgconn.PgError](err); ok && + pgErr.Code == pgerrcode.ForeignKeyViolation && pgErr.ConstraintName == "chat_members_user_fk" { + return apperr.Wrap(err, apperr.CodeBadRequest, "unknown chat member") + } + return apperr.WrapPostgres(err, "failed to create chat members") + } + return nil +} + +func (r *ChatsRepo) GetForMember(ctx context.Context, req dto.ChatMember) (*dto.Chat, error) { + const query = `SELECT c.id, c.title, c.created_at, + ARRAY(SELECT m.user_id FROM chat_members m WHERE m.chat_id = c.id ORDER BY m.user_id) +FROM chats c +WHERE c.id = $1 + AND EXISTS (SELECT 1 FROM chat_members access WHERE access.chat_id = c.id AND access.user_id = $2)` + return scanChat(r.pool.QueryRow(ctx, query, req.ChatID, req.UserID)) +} + +func (r *ChatsRepo) ListForMember(ctx context.Context, userID uuid.UUID, page dto.PageQuery) ([]dto.Chat, error) { + query := `WITH page AS MATERIALIZED ( + SELECT c.id, c.title, c.created_at + FROM chats c JOIN chat_members access ON access.chat_id = c.id + WHERE access.user_id = $1` + args := []any{userID, page.Limit} + if page.After != nil { + query += ` AND (c.created_at, c.id) > ($3, $4)` + args = append(args, page.After.CreatedAt, page.After.ID) + } + query += ` ORDER BY c.created_at, c.id LIMIT $2 +) +SELECT p.id, p.title, p.created_at, + ARRAY(SELECT m.user_id FROM chat_members m WHERE m.chat_id = p.id ORDER BY m.user_id) +FROM page p ORDER BY p.created_at, p.id` + rows, err := r.pool.Query(ctx, query, args...) + if err != nil { + return nil, apperr.WrapPostgres(err, "failed to list chats") + } + defer rows.Close() + chats := make([]dto.Chat, 0) + for rows.Next() { + chat, err := scanChat(rows) + if err != nil { + return nil, err + } + chats = append(chats, *chat) + } + if err = rows.Err(); err != nil { + return nil, apperr.WrapPostgres(err, "failed to list chats") + } + return chats, nil +} + +func scanChat(row pgx.Row) (*dto.Chat, error) { + var chat dto.Chat + if err := row.Scan(&chat.ID, &chat.Title, &chat.CreatedAt, &chat.MemberIDs); err != nil { + return nil, apperr.WrapPostgres(err, "failed to get chat") + } + return &chat, nil +} diff --git a/service/internal/adapter/repository/storage/idempotency_test.go b/service/internal/adapter/repository/storage/idempotency_test.go new file mode 100644 index 0000000..db5164c --- /dev/null +++ b/service/internal/adapter/repository/storage/idempotency_test.go @@ -0,0 +1,303 @@ +package storage + +import ( + "context" + "crypto/sha256" + "errors" + "fmt" + "os" + "strings" + "testing" + "time" + + "gitea.privcloud.tech/mtusi2026/internal/config" + "gitea.privcloud.tech/mtusi2026/internal/dto" + "gitea.privcloud.tech/mtusi2026/internal/pkg/apperr" + "gitea.privcloud.tech/mtusi2026/internal/pkg/migrator" + "gitea.privcloud.tech/mtusi2026/internal/pkg/postgres" + "gitea.privcloud.tech/mtusi2026/internal/pkg/postgres/txmanager" + chatsvc "gitea.privcloud.tech/mtusi2026/internal/service/chat" + "github.com/google/uuid" +) + +// The explicit runner supplies a newly created database; never alter the API database. +func TestIdempotencyTransactionAndExpiry(t *testing.T) { + database := os.Getenv("TEST_DATABASE_NAME") + if database == "" { + t.Skip("run make test-storage for an isolated PostgreSQL database") + } + if database == os.Getenv("POSTGRES_DATABASE") { + t.Fatal("test database must differ from application database") + } + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + cfg, err := config.New() + if err != nil { + t.Fatal(err) + } + cfg.Postgres.Database = database + cfg.Postgres.RunMigrations = true + if err = migrator.New(cfg).Start(ctx); err != nil { + t.Fatal(err) + } + pool := postgres.New(cfg.Postgres) + defer pool.Close() + repo := NewMessagesRepo(pool) + user, chat := uuid.New(), uuid.New() + if _, err = pool.Exec(ctx, `INSERT INTO users(id,name,username,password_hash) VALUES($1,'Synthetic','atomic_test','unused')`, user); err != nil { + t.Fatal(err) + } + if _, err = pool.Exec(ctx, `INSERT INTO chats(id,title) VALUES($1,'Atomic test')`, chat); err != nil { + t.Fatal(err) + } + if _, err = pool.Exec(ctx, `INSERT INTO chat_members(chat_id,user_id) VALUES($1,$2)`, chat, user); err != nil { + t.Fatal(err) + } + req := dto.CreateMessage{ChatID: chat, SenderID: user, Text: "atomic", IdempotencyKey: "atomic-key"} + tx, err := pool.Begin(ctx) + if err != nil { + t.Fatal(err) + } + if _, err = chatsvc.CreateIdempotentTx(ctx, tx, repo, req); err != nil { + t.Fatal(err) + } + if err = tx.Rollback(ctx); err != nil { + t.Fatal(err) + } + var messages, keys int + if err = pool.QueryRow(ctx, `SELECT (SELECT count(*) FROM messages),(SELECT count(*) FROM message_idempotency)`).Scan(&messages, &keys); err != nil { + t.Fatal(err) + } + if messages != 0 || keys != 0 { + t.Fatalf("rollback left %d messages, %d keys", messages, keys) + } + tx, err = pool.Begin(ctx) + if err != nil { + t.Fatal(err) + } + first, err := chatsvc.CreateIdempotentTx(ctx, tx, repo, req) + if err != nil { + t.Fatal(err) + } + if err = tx.Commit(ctx); err != nil { + t.Fatal(err) + } + // Expire the test record only: no shortened production TTL and no wall-clock wait. + if _, err = pool.Exec(ctx, `UPDATE message_idempotency SET expires_at=clock_timestamp()-interval '1 second' WHERE user_id=$1`, user); err != nil { + t.Fatal(err) + } + req.Text = "new after expiry" + tx, err = pool.Begin(ctx) + if err != nil { + t.Fatal(err) + } + second, err := chatsvc.CreateIdempotentTx(ctx, tx, repo, req) + if err != nil { + t.Fatal(err) + } + if err = tx.Commit(ctx); err != nil { + t.Fatal(err) + } + if first.ID == second.ID { + t.Fatal("expired key replayed old message") + } + if err = pool.QueryRow(ctx, `SELECT (SELECT count(*) FROM messages),(SELECT count(*) FROM message_idempotency)`).Scan(&messages, &keys); err != nil { + t.Fatal(err) + } + if messages != 2 || keys != 1 { + t.Fatalf("expiry left %d messages, %d keys; want 2,1", messages, keys) + } + if _, err = pool.Exec(ctx, `CREATE FUNCTION reject_test_key() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN IF NEW.key='failure-key' THEN RAISE EXCEPTION 'synthetic failure'; END IF; RETURN NEW; END $$; CREATE TRIGGER reject_test_key BEFORE INSERT ON message_idempotency FOR EACH ROW EXECUTE FUNCTION reject_test_key()`); err != nil { + t.Fatal(err) + } + req.IdempotencyKey = "failure-key" + tx, err = pool.Begin(ctx) + if err != nil { + t.Fatal(err) + } + if _, err = chatsvc.CreateIdempotentTx(ctx, tx, repo, req); err == nil { + t.Fatal("expected injected failure") + } + if err = tx.Rollback(ctx); err != nil { + t.Fatal(err) + } + if err = pool.QueryRow(ctx, `SELECT (SELECT count(*) FROM messages),(SELECT count(*) FROM message_idempotency)`).Scan(&messages, &keys); err != nil { + t.Fatal(err) + } + if messages != 2 || keys != 1 { + t.Fatal("failed result persisted a side effect") + } + if _, err = pool.Exec(ctx, `DROP TRIGGER reject_test_key ON message_idempotency; DROP FUNCTION reject_test_key()`); err != nil { + t.Fatal(err) + } + tx, err = pool.Begin(ctx) + if err != nil { + t.Fatal(err) + } + if _, err = chatsvc.CreateIdempotentTx(ctx, tx, repo, req); err != nil { + t.Fatal("transient failure was cached", err) + } + if err = tx.Commit(ctx); err != nil { + t.Fatal(err) + } + + t.Run("service owns message content rules", func(t *testing.T) { + svc := chatsvc.New(txmanager.New(pool), NewChatsRepo(pool), repo, uncachedChats{}) + own, foreign, missing := uuid.New(), uuid.New(), uuid.New() + otherChat := uuid.New() + if _, err := pool.Exec(ctx, `INSERT INTO chats(id,title) VALUES($1,'Other chat');`, otherChat); err != nil { + t.Fatal(err) + } + if _, err := pool.Exec(ctx, `INSERT INTO chat_members(chat_id,user_id) VALUES($1,$2)`, otherChat, user); err != nil { + t.Fatal(err) + } + for _, fixture := range []struct { + id, chat uuid.UUID + missing bool + }{{own, chat, false}, {foreign, otherChat, false}, {missing, chat, true}} { + if _, err := pool.Exec(ctx, `INSERT INTO attachments(id,chat_id,owner_id,filename,object_key,content_type,size_bytes,sha256,integrity_missing,created_at,updated_at) VALUES($1,$2,$3,'test.png',$4,'image/png',1,$5,$6,now(),now())`, fixture.id, fixture.chat, user, "originals/"+fixture.id.String(), strings.Repeat("a", 64), fixture.missing); err != nil { + t.Fatal(err) + } + } + counts := func() string { + t.Helper() + var messages, links, keys int + if err := pool.QueryRow(ctx, `SELECT (SELECT count(*) FROM messages),(SELECT count(*) FROM message_attachments),(SELECT count(*) FROM message_idempotency)`).Scan(&messages, &links, &keys); err != nil { + t.Fatal(err) + } + return fmt.Sprintf("%d/%d/%d", messages, links, keys) + } + for _, tc := range []struct { + name, text string + ids []uuid.UUID + code apperr.Code + }{ + {"empty", "", nil, apperr.CodeBadRequest}, + {"whitespace", " \t\n\u2003", nil, apperr.CodeBadRequest}, + {"invalid UTF8", string([]byte{0xff}), nil, apperr.CodeBadRequest}, + {"nul", "\x00", nil, apperr.CodeBadRequest}, + {"too long", strings.Repeat("я", 4001), nil, apperr.CodeBadRequest}, + {"duplicate", "", []uuid.UUID{own, own}, apperr.CodeBadRequest}, + {"too many", "text", make([]uuid.UUID, 11), apperr.CodeBadRequest}, + {"nil id", "", []uuid.UUID{uuid.Nil}, apperr.CodeBadRequest}, + {"unknown", "", []uuid.UUID{uuid.New()}, apperr.CodeNotFound}, + {"foreign chat", "", []uuid.UUID{foreign}, apperr.CodeNotFound}, + {"missing object", "", []uuid.UUID{missing}, apperr.CodeUnavailable}, + } { + t.Run(tc.name, func(t *testing.T) { + before := counts() + _, err := svc.CreateMessage(ctx, dto.CreateMessage{ChatID: chat, SenderID: user, IdempotencyKey: uuid.NewString(), Text: tc.text, AttachmentIDs: tc.ids}) + if err == nil || apperr.CodeFromError(err) != tc.code { + t.Fatalf("got %v, want code %v", err, tc.code) + } + if after := counts(); after != before { + t.Fatalf("rejected message changed counts: %s -> %s", before, after) + } + }) + } + for _, text := range []string{"", " \t\u2003"} { + request := dto.CreateMessage{ChatID: chat, SenderID: user, IdempotencyKey: uuid.NewString(), Text: text, AttachmentIDs: []uuid.UUID{own}} + first, err := svc.CreateMessage(ctx, request) + if err != nil { + t.Fatal(err) + } + replay, err := svc.CreateMessage(ctx, request) + if err != nil || replay.ID != first.ID || len(replay.AttachmentIDs) != 1 || replay.AttachmentIDs[0] != own { + t.Fatalf("attachment replay failed: %v, %v", replay, err) + } + page, err := svc.ListForChat(ctx, dto.MessageFilters{UserID: user, ChatID: chat}) + if err != nil { + t.Fatal(err) + } + found := false + for _, msg := range page.Items { + if msg.ID == first.ID { + found = len(msg.AttachmentIDs) == 1 && msg.AttachmentIDs[0] == own + } + } + if !found { + t.Fatal("history lost attachment") + } + request.AttachmentIDs = []uuid.UUID{foreign} + if _, err = svc.CreateMessage(ctx, request); apperr.CodeFromError(err) != apperr.CodeNotFound { + t.Fatalf("foreign replay: %v", err) + } + } + // Preserve legacy hashes written before the attachment migration. + legacy := dto.CreateMessage{ChatID: chat, SenderID: user, IdempotencyKey: uuid.NewString(), Text: "legacy"} + first, err := svc.CreateMessage(ctx, legacy) + if err != nil { + t.Fatal(err) + } + hash := sha256.Sum256([]byte(legacy.Text)) + if _, err = pool.Exec(ctx, `UPDATE message_idempotency SET request_hash=$1,hash_version=1 WHERE key=$2`, hash[:], legacy.IdempotencyKey); err != nil { + t.Fatal(err) + } + replay, err := svc.CreateMessage(ctx, legacy) + if err != nil || replay.ID != first.ID { + t.Fatalf("legacy replay failed: %v", err) + } + legacy.AttachmentIDs = []uuid.UUID{own} + if _, err = svc.CreateMessage(ctx, legacy); apperr.CodeFromError(err) != apperr.CodeConflict { + t.Fatalf("legacy hash ignored new attachment: %v", err) + } + textOnly := dto.CreateMessage{ChatID: chat, SenderID: user, IdempotencyKey: uuid.NewString(), Text: "text only"} + first, err = svc.CreateMessage(ctx, textOnly) + if err != nil { + t.Fatal(err) + } + textOnly.AttachmentIDs = []uuid.UUID{} + replay, err = svc.CreateMessage(ctx, textOnly) + if err != nil || replay.ID != first.ID { + t.Fatalf("nil and empty attachment lists differ: %v", err) + } + // Force link failure using the relational FK and prove all preceding writes roll back. + before := counts() + tx, err := pool.Begin(ctx) + if err != nil { + t.Fatal(err) + } + _, err = chatsvc.CreateIdempotentTx(ctx, tx, repo, dto.CreateMessage{ChatID: chat, SenderID: user, IdempotencyKey: uuid.NewString(), Text: "link rollback", AttachmentIDs: []uuid.UUID{own, uuid.New()}}) + if err == nil { + t.Fatal("expected missing attachment FK failure") + } + _ = tx.Rollback(ctx) + if after := counts(); after != before { + t.Fatalf("failed link changed counts: %s -> %s", before, after) + } + var triggers int + if err = pool.QueryRow(ctx, `SELECT count(*) FROM pg_trigger WHERE tgname='message_content'`).Scan(&triggers); err != nil || triggers != 0 { + t.Fatalf("message trigger remains: %d, %v", triggers, err) + } + }) + + t.Run("object lock cancellation does not leak a pooled lock", func(t *testing.T) { + attachments := &AttachmentsRepo{Pool: pool} + key := "originals/" + uuid.NewString() + _, unlock, err := attachments.Lock(ctx, key) + if err != nil { + t.Fatal(err) + } + waiting, cancel := context.WithTimeout(ctx, 50*time.Millisecond) + _, _, err = attachments.Lock(waiting, key) + cancel() + unlock() + if err == nil { + t.Fatal("contending lock ignored deadline") + } + _, release, err := attachments.Lock(ctx, key) + if err != nil { + t.Fatalf("lock leaked after cancellation: %v", err) + } + release() + }) + + t.Log("rollback removed both side effect and key; expired key created one new message") +} + +type uncachedChats struct{} + +func (uncachedChats) Create(context.Context, dto.Chat) error { return nil } +func (uncachedChats) Get(context.Context, uuid.UUID) (*dto.Chat, error) { + return nil, errors.New("cache disabled for storage test") +} diff --git a/service/internal/adapter/repository/storage/jobs.go b/service/internal/adapter/repository/storage/jobs.go new file mode 100644 index 0000000..887a59c --- /dev/null +++ b/service/internal/adapter/repository/storage/jobs.go @@ -0,0 +1,134 @@ +package storage + +import ( + "context" + "encoding/json" + "gitea.privcloud.tech/mtusi2026/internal/dto" + "github.com/google/uuid" + "github.com/jackc/pgx/v5" + "time" +) + +func (r *AttachmentsRepo) Ping(ctx context.Context) error { return r.Pool.Ping(ctx) } +func (r *AttachmentsRepo) GetTx(ctx context.Context, tx pgx.Tx, id uuid.UUID) (*dto.AttachmentRecord, error) { + return ScanAttachment(tx.QueryRow(ctx, "SELECT "+AttachmentColumns+" FROM attachments WHERE id=$1 FOR UPDATE", id)) +} +func (r *AttachmentsRepo) InsertTx(ctx context.Context, tx pgx.Tx, a *dto.AttachmentRecord) error { + p := a.Public + _, e := tx.Exec(ctx, `INSERT INTO attachments(id,chat_id,owner_id,filename,content_type,size_bytes,sha256,created_at,updated_at,object_key,staging_key,status) VALUES($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$10,$11)`, p.Id, p.ChatId, p.OwnerId, p.Filename, p.ContentType, p.SizeBytes, p.Sha256, p.CreatedAt, p.UpdatedAt, a.StagingKey, p.Status) + return e +} +func (r *AttachmentsRepo) UpdateTx(ctx context.Context, tx pgx.Tx, a *dto.AttachmentRecord) error { + variants, e := json.Marshal(a.Public.Variants) + if e != nil { + return e + } + keys, e := json.Marshal(a.VariantKeys) + if e != nil { + return e + } + _, e = tx.Exec(ctx, `UPDATE attachments SET status=$2,error_code=$3,pipeline_version=$4,staging_key=$5,object_key=$6,variants=$7,variant_keys=$8,updated_at=$9,integrity_missing=$10 WHERE id=$1`, a.Public.Id, a.Public.Status, a.Public.ErrorCode, a.PipelineVersion, a.StagingKey, a.Key, variants, keys, a.Public.UpdatedAt, a.Missing) + return e +} + +const jobColumns = `id,attachment_id,pipeline_version,attempt,created_at,state,token,lease_until,next_attempt_at,attempt_limit,error_code,updated_at,traceparent,tracestate` + +func scanJob(row pgx.Row) (*dto.AttachmentJob, error) { + j := new(dto.AttachmentJob) + m := &j.Message + err := row.Scan(&m.JobID, &m.AttachmentID, &m.PipelineVersion, &m.Attempt, &m.CreatedAt, &j.State, &j.Token, &j.LeaseUntil, &j.NextAttemptAt, &j.AttemptLimit, &j.ErrorCode, &j.UpdatedAt, &m.Traceparent, &m.Tracestate) + m.SchemaVersion = 1 + m.Type = "attachment.process.v1" + return j, err +} +func (r *AttachmentsRepo) JobTx(ctx context.Context, tx pgx.Tx, id uuid.UUID) (*dto.AttachmentJob, error) { + return scanJob(tx.QueryRow(ctx, `SELECT `+jobColumns+` FROM attachment_jobs WHERE id=$1 FOR UPDATE`, id)) +} +func (r *AttachmentsRepo) InsertJobTx(ctx context.Context, tx pgx.Tx, j *dto.AttachmentJob) error { + m := j.Message + _, e := tx.Exec(ctx, `INSERT INTO attachment_jobs(`+jobColumns+`) VALUES($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14)`, m.JobID, m.AttachmentID, m.PipelineVersion, m.Attempt, m.CreatedAt, j.State, j.Token, j.LeaseUntil, j.NextAttemptAt, j.AttemptLimit, j.ErrorCode, j.UpdatedAt, m.Traceparent, m.Tracestate) + return e +} +func (r *AttachmentsRepo) SaveJobTx(ctx context.Context, tx pgx.Tx, j *dto.AttachmentJob) error { + _, e := tx.Exec(ctx, `UPDATE attachment_jobs SET attempt=$2,state=$3,token=$4,lease_until=$5,next_attempt_at=$6,attempt_limit=$7,error_code=$8,updated_at=$9,traceparent=$10,tracestate=$11 WHERE id=$1`, j.Message.JobID, j.Message.Attempt, j.State, j.Token, j.LeaseUntil, j.NextAttemptAt, j.AttemptLimit, j.ErrorCode, j.UpdatedAt, j.Message.Traceparent, j.Message.Tracestate) + return e +} +func (r *AttachmentsRepo) ExpiredJobs(ctx context.Context) ([]uuid.UUID, error) { + rows, e := r.Pool.Query(ctx, `SELECT id FROM attachment_jobs WHERE state='processing' AND lease_until>'attempt')::integer=j.attempt LIMIT 100`) + if e != nil { + return nil, e + } + defer rows.Close() + var ids []uuid.UUID + for rows.Next() { + var id uuid.UUID + if e = rows.Scan(&id); e != nil { + return nil, e + } + ids = append(ids, id) + } + return ids, rows.Err() +} +func (r *AttachmentsRepo) RetiredSources(ctx context.Context, before time.Time) ([]*dto.AttachmentRecord, error) { + rows, e := r.Pool.Query(ctx, `SELECT `+AttachmentColumns+` FROM attachments WHERE staging_key<>'' AND (status='ready' OR (status='failed' AND updated_at<$1)) LIMIT 100`, before) + if e != nil { + return nil, e + } + defer rows.Close() + var out []*dto.AttachmentRecord + for rows.Next() { + a, e := ScanAttachment(rows) + if e != nil { + return nil, e + } + out = append(out, a) + } + return out, rows.Err() +} + +func (r *AttachmentsRepo) SaveVariantTx(ctx context.Context, tx pgx.Tx, id uuid.UUID, version int, name, key, sha string, size int64) error { + _, e := tx.Exec(ctx, `INSERT INTO attachment_variants(attachment_id,pipeline_version,variant,object_key,sha256,size_bytes) VALUES($1,$2,$3,$4,$5,$6)`, id, version, name, key, sha, size) + return e +} + +func (r *AttachmentsRepo) PublicationDeadTx(ctx context.Context, tx pgx.Tx, id uuid.UUID, attempt int) (bool, error) { + var dead bool + e := tx.QueryRow(ctx, `SELECT EXISTS(SELECT 1 FROM outbox WHERE job_id=$1 AND status='dead_letter' AND (payload->>'attempt')::integer=$2)`, id, attempt).Scan(&dead) + return dead, e +} diff --git a/service/internal/adapter/repository/storage/messages.go b/service/internal/adapter/repository/storage/messages.go new file mode 100644 index 0000000..0c1b6f9 --- /dev/null +++ b/service/internal/adapter/repository/storage/messages.go @@ -0,0 +1,135 @@ +package storage + +import ( + "context" + + "errors" + "time" + + "github.com/google/uuid" + "github.com/jackc/pgerrcode" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgconn" + "github.com/jackc/pgx/v5/pgxpool" + + "gitea.privcloud.tech/mtusi2026/internal/dto" + "gitea.privcloud.tech/mtusi2026/internal/pkg/apperr" +) + +type MessagesRepo struct { + pool *pgxpool.Pool +} + +func NewMessagesRepo(pool *pgxpool.Pool) *MessagesRepo { + return &MessagesRepo{pool: pool} +} + +// LockKeyTx serializes persistence for one idempotency scope. +func (r *MessagesRepo) LockKeyTx(ctx context.Context, tx pgx.Tx, scope string) error { + _, err := tx.Exec(ctx, `SELECT pg_advisory_xact_lock(hashtextextended($1,0))`, scope) + if err != nil { + return apperr.WrapPostgres(err, "failed to lock message key") + } + return nil +} +func (r *MessagesRepo) ReplayTx(ctx context.Context, tx pgx.Tx, req dto.CreateMessage) (*dto.MessageReplay, error) { + v := new(dto.MessageReplay) + err := tx.QueryRow(ctx, `SELECT request_hash,message_id,hash_version FROM message_idempotency WHERE user_id=$1 AND method='POST' AND chat_id=$2 AND key=$3 AND expires_at>clock_timestamp()`, req.SenderID, req.ChatID, req.IdempotencyKey).Scan(&v.Hash, &v.MessageID, &v.HashVersion) + if errors.Is(err, pgx.ErrNoRows) { + return nil, nil + } + if err != nil { + return nil, apperr.WrapPostgres(err, "failed to read message key") + } + return v, nil +} +func (r *MessagesRepo) MessageTx(ctx context.Context, tx pgx.Tx, id uuid.UUID) (*dto.Message, error) { + return scanMessage(tx.QueryRow(ctx, `SELECT `+messageColumns+` FROM messages WHERE id=$1`, id)) +} +func (r *MessagesRepo) InsertMessageTx(ctx context.Context, tx pgx.Tx, message dto.Message) error { + _, err := tx.Exec(ctx, `INSERT INTO messages(id,chat_id,sender_id,text,created_at) VALUES($1,$2,$3,$4,$5)`, message.ID, message.ChatID, message.SenderID, message.Text, message.CreatedAt) + if err != nil { + if pgErr, ok := errors.AsType[*pgconn.PgError](err); ok && pgErr.Code == pgerrcode.ForeignKeyViolation { + return apperr.Wrap(err, apperr.CodeNotFound, "chat not found") + } + return apperr.WrapPostgres(err, "failed to create message") + } + for position, id := range message.AttachmentIDs { + if _, err = tx.Exec(ctx, `INSERT INTO message_attachments(message_id,chat_id,attachment_id,position) VALUES($1,$2,$3,$4)`, message.ID, message.ChatID, id, position); err != nil { + return apperr.WrapPostgres(err, "failed to link attachment") + } + } + return nil +} +func (r *MessagesRepo) SaveKeyTx(ctx context.Context, tx pgx.Tx, req dto.CreateMessage, hash []byte, messageID uuid.UUID, expires time.Time) error { + _, err := tx.Exec(ctx, `INSERT INTO message_idempotency(user_id,method,chat_id,key,request_hash,message_id,expires_at,hash_version) VALUES($1,'POST',$2,$3,$4,$5,$6,2) ON CONFLICT(user_id,method,chat_id,key) DO UPDATE SET request_hash=EXCLUDED.request_hash,message_id=EXCLUDED.message_id,expires_at=EXCLUDED.expires_at,hash_version=EXCLUDED.hash_version`, req.SenderID, req.ChatID, req.IdempotencyKey, hash, messageID, expires) + if err != nil { + return apperr.WrapPostgres(err, "failed to save message key") + } + return nil +} + +func (r *MessagesRepo) GetByIDForChat(ctx context.Context, chatID, messageID uuid.UUID) (*dto.Message, error) { + const query = `SELECT ` + messageColumns + ` FROM messages WHERE chat_id = $1 AND id = $2` + return scanMessage(r.pool.QueryRow(ctx, query, chatID, messageID)) +} + +func (r *MessagesRepo) ListForChat(ctx context.Context, chatID uuid.UUID, page dto.PageQuery) ([]dto.Message, error) { + query := `SELECT ` + messageColumns + ` FROM messages WHERE chat_id = $1` + args := []any{chatID, page.Limit} + if page.After != nil { + query += ` AND (created_at, id) > ($3, $4)` + args = append(args, page.After.CreatedAt, page.After.ID) + } + query += ` ORDER BY created_at, id LIMIT $2` + rows, err := r.pool.Query(ctx, query, args...) + if err != nil { + return nil, apperr.WrapPostgres(err, "failed to list messages") + } + defer rows.Close() + messages := make([]dto.Message, 0) + for rows.Next() { + message, err := scanMessage(rows) + if err != nil { + return nil, err + } + messages = append(messages, *message) + } + if err = rows.Err(); err != nil { + return nil, apperr.WrapPostgres(err, "failed to list messages") + } + return messages, nil +} + +func scanMessage(row pgx.Row) (*dto.Message, error) { + var message dto.Message + if err := row.Scan(&message.ID, &message.ChatID, &message.SenderID, &message.Text, &message.CreatedAt, &message.AttachmentIDs); err != nil { + return nil, apperr.WrapPostgres(err, "failed to get message") + } + return &message, nil +} + +const messageColumns = `id, chat_id, sender_id, text, created_at, + ARRAY(SELECT attachment_id FROM message_attachments WHERE message_id=messages.id ORDER BY position)` + +func (r *MessagesRepo) AttachmentsForMessageTx(ctx context.Context, tx pgx.Tx, ids []uuid.UUID) ([]dto.MessageAttachment, error) { + // Prevent a concurrent integrity update from invalidating the checked rows + // until the message and its links have committed together. + rows, err := tx.Query(ctx, `SELECT id, chat_id, integrity_missing,status FROM attachments WHERE id=ANY($1) ORDER BY id FOR SHARE`, ids) + if err != nil { + return nil, apperr.WrapPostgres(err, "failed to read attachments") + } + defer rows.Close() + result := make([]dto.MessageAttachment, 0, len(ids)) + for rows.Next() { + var a dto.MessageAttachment + if err := rows.Scan(&a.ID, &a.ChatID, &a.Missing, &a.Status); err != nil { + return nil, apperr.WrapPostgres(err, "failed to read attachment") + } + result = append(result, a) + } + if err := rows.Err(); err != nil { + return nil, apperr.WrapPostgres(err, "failed to read attachments") + } + return result, nil +} diff --git a/service/internal/adapter/repository/storage/observation.go b/service/internal/adapter/repository/storage/observation.go new file mode 100644 index 0000000..2fe1527 --- /dev/null +++ b/service/internal/adapter/repository/storage/observation.go @@ -0,0 +1,51 @@ +package storage + +import ( + "context" + "gitea.privcloud.tech/mtusi2026/internal/service/observation" + "github.com/google/uuid" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" + "time" +) + +func (r *AttachmentsRepo) AcceptSLO(ctx context.Context, tx pgx.Tx, id uuid.UUID, accepted, deadline time.Time) error { + _, e := tx.Exec(ctx, `INSERT INTO job_slo(job_id,accepted_at,deadline_at)VALUES($1,$2,$3)`, id, accepted, deadline) + return e +} +func (r *AttachmentsRepo) ReadySLO(ctx context.Context, tx pgx.Tx, id uuid.UUID, ready time.Time) error { + _, e := tx.Exec(ctx, `UPDATE job_slo SET ready_at=COALESCE(ready_at,$2) WHERE job_id=$1`, id, ready) + return e +} + +type ObservationRepo struct{ Pool *pgxpool.Pool } + +func (r *ObservationRepo) Pending(ctx context.Context) ([]observation.Sample, error) { + rows, e := r.Pool.Query(ctx, `SELECT job_id,accepted_at,deadline_at,ready_at,failed_at FROM job_slo WHERE outcome IS NULL ORDER BY deadline_at LIMIT 500`) + if e != nil { + return nil, e + } + defer rows.Close() + out := []observation.Sample{} + for rows.Next() { + var x observation.Sample + if e = rows.Scan(&x.ID, &x.Accepted, &x.Deadline, &x.Ready, &x.Failed); e != nil { + return nil, e + } + out = append(out, x) + } + return out, rows.Err() +} +func (r *ObservationRepo) Decide(ctx context.Context, id uuid.UUID, outcome string, at time.Time) error { + _, e := r.Pool.Exec(ctx, `UPDATE job_slo SET outcome=$2,evaluated_at=$3 WHERE job_id=$1 AND outcome IS NULL`, id, outcome, at) + return e +} +func (r *ObservationRepo) Snapshot(ctx context.Context) (x observation.Snapshot, e error) { + e = r.Pool.QueryRow(ctx, `SELECT (SELECT count(*) FROM attachment_jobs WHERE state IN ('queued','processing')), (SELECT count(*) FROM outbox WHERE status IN ('pending','processing','failed')), COALESCE((SELECT EXTRACT(EPOCH FROM clock_timestamp()-min(created_at)) FROM attachment_jobs WHERE state IN ('queued','processing')),0), (SELECT count(*) FROM job_slo WHERE outcome='good'), (SELECT count(*) FROM job_slo WHERE outcome='bad'), (SELECT count(*) FROM pg_stat_activity WHERE wait_event_type='Lock')`).Scan(&x.Queued, &x.Outbox, &x.Oldest, &x.Good, &x.Bad, &x.Locked) + return +} + +func (r *AttachmentsRepo) FailSLO(ctx context.Context, tx pgx.Tx, id uuid.UUID, at time.Time) error { + _, e := tx.Exec(ctx, `UPDATE job_slo SET failed_at=COALESCE(failed_at,$2) WHERE job_id=$1`, id, at) + return e +} diff --git a/service/internal/adapter/repository/storage/outbox.go b/service/internal/adapter/repository/storage/outbox.go new file mode 100644 index 0000000..ffc3191 --- /dev/null +++ b/service/internal/adapter/repository/storage/outbox.go @@ -0,0 +1,52 @@ +package storage + +import ( + "context" + "errors" + "gitea.privcloud.tech/mtusi2026/internal/service/outbox" + "github.com/google/uuid" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" + "time" +) + +type OutboxRepo struct{ Pool *pgxpool.Pool } + +func (r *OutboxRepo) CreateTx(ctx context.Context, tx pgx.Tx, q outbox.CreateMessageReq) (outbox.CreateMessageRes, error) { + _, e := tx.Exec(ctx, `INSERT INTO outbox(id,job_id,payload,status,max_attempts,token,next_attempt_at,created_at,updated_at) VALUES($1,$2,$3,$4,$5,$6,$7,$7,$7)`, q.ID, q.AggregateID, q.Payload, q.Status, q.MaxAttempts, uuid.New(), q.CreatedAt) + return outbox.CreateMessageRes{ID: q.ID, CreatedAt: q.CreatedAt}, e +} +func scanOutbox(rows pgx.Rows) (out []outbox.AcquiredMessage, err error) { + defer rows.Close() + for rows.Next() { + var m outbox.AcquiredMessage + if err = rows.Scan(&m.ID, &m.AggregateID, &m.Payload, &m.Attempts, &m.MaxAttempts, &m.Token); err != nil { + return + } + out = append(out, m) + } + err = rows.Err() + return +} +func (r *OutboxRepo) AcquireManyForProcessing(ctx context.Context, q outbox.AcquireMessagesForProcessingReq) ([]outbox.AcquiredMessage, error) { + rows, e := r.Pool.Query(ctx, `WITH picked AS (SELECT id FROM outbox WHERE status IN ('pending','failed') AND next_attempt_at<=clock_timestamp() ORDER BY next_attempt_at,id LIMIT $1 FOR UPDATE SKIP LOCKED) + UPDATE outbox o SET status='processing',attempts=attempts+1,token=$2,locked_until=clock_timestamp()+$3::interval,updated_at=clock_timestamp() FROM picked p WHERE o.id=p.id RETURNING o.id,o.job_id,o.payload,o.attempts,o.max_attempts,o.token`, q.Limit, uuid.New(), q.LockDuration.String()) + if e != nil { + return nil, e + } + return scanOutbox(rows) +} +func (r *OutboxRepo) Expired(ctx context.Context) ([]outbox.AcquiredMessage, error) { + rows, e := r.Pool.Query(ctx, `SELECT id,job_id,payload,attempts,max_attempts,token FROM outbox WHERE status='processing' AND locked_until 10485760 || c.MaxPixels < 1 || c.MaxPixels > 40000000 || c.Timeout < time.Second || c.Timeout > 120*time.Second || c.Grace <= 2*c.Timeout { + return nil, fmt.Errorf("invalid attachment limits or grace period") + } + return c, nil +} + +func env(k, d string) string { + if v := os.Getenv(k); v != "" { + return v + } + return d +} diff --git a/service/internal/config/config.go b/service/internal/config/config.go new file mode 100644 index 0000000..b55dc2e --- /dev/null +++ b/service/internal/config/config.go @@ -0,0 +1,115 @@ +package config + +import ( + "os" + "strings" + "time" + + "github.com/spf13/viper" + "go.uber.org/multierr" + + "gitea.privcloud.tech/mtusi2026/internal/pkg/types/environment" +) + +var defaultConfig = &Config{ + Server: &Server{ + Host: "localhost", + Port: 8001, + ReadTimeout: 10 * time.Second, + WriteTimeout: 10 * time.Second, + IdleTimeout: 10 * time.Second, + ShutdownTimeout: 10 * time.Second, + Enabled: false, + }, + Postgres: &Postgres{ + Host: "localhost", + Port: 5432, + User: "postgres", + Password: "postgres", + Database: "postgres", + VersionTable: "versions", + RunMigrations: false, + MaxConnections: 10, + ConnectTimeout: 500 * time.Millisecond, + QueryTimeout: 2 * time.Second, + }, + Redis: &Redis{ + Host: "localhost", + Port: 6379, + Username: "", + Password: "", + DB: 0, + }, + JWT: &JWT{ + AccessTokenLifetime: 4 * time.Hour, + RefreshTokenLifetime: time.Hour, + PublicKeyPath: "certs/private.pem", + PrivateKeyPath: "certs/public.pem", + }, + Application: &Application{ + LogLevel: "info", + ShutdownTimeout: 10 * time.Second, + Environment: environment.ApplicationEnvironmentDev, + RequestRateLimit: 100, + }, +} + +type Config struct { + Server *Server + Postgres *Postgres + Redis *Redis + JWT *JWT + Application *Application +} + +func New() (*Config, error) { + v := viper.New() + v.SetEnvKeyReplacer(strings.NewReplacer(".", "_")) + v.AutomaticEnv() + + cfg := defaultConfig.copy() + + if err := cfg.loadConfig(v); err != nil { + return nil, err + } + + if err := cfg.Validate(); err != nil { + return nil, err + } + + return cfg, nil +} + +func (c *Config) Validate() error { + var jwtErr error + if os.Getenv("SERVICE_ROLE") != "worker" && os.Getenv("SERVICE_ROLE") != "dispatcher" { + jwtErr = c.JWT.Validate() + } + return multierr.Combine( + c.Server.Validate(), + c.Postgres.Validate(), + jwtErr, + c.Redis.Validate(), + c.Application.Validate(), + ) +} + +func (c *Config) loadConfig(v *viper.Viper) error { + return multierr.Combine( + c.Postgres.loadConfig(v), + c.Server.loadConfig(v), + c.JWT.loadConfig(v), + c.Redis.loadConfig(v), + c.Application.loadConfig(v), + ) +} + +func (c *Config) copy() *Config { + return &Config{ + Server: c.Server.copy(), + Postgres: c.Postgres.copy(), + Redis: c.Redis.copy(), + JWT: c.JWT.copy(), + Application: c.Application.copy(), + } +} diff --git a/service/internal/config/cookies.go b/service/internal/config/cookies.go new file mode 100644 index 0000000..b6f7d88 --- /dev/null +++ b/service/internal/config/cookies.go @@ -0,0 +1,65 @@ +package config + +import ( + "errors" + "net/http" + "strings" + + "github.com/spf13/viper" +) + +type Cookies struct { + Enabled bool + Domain string + Path string + Secure bool + SameSite http.SameSite + AllowedOrigins []string +} + +func (c *Cookies) Validate() error { + switch { + case c.Domain == "": + return wrapValidationError("cookies", errors.New("domain cannot be empty")) + case c.Path == "": + return wrapValidationError("cookies", errors.New("path cannot be empty")) + case c.SameSite == http.SameSiteDefaultMode: + return wrapValidationError("cookies", errors.New("same_site_set_to_none_mode cannot be empty")) + //case c.Enabled && !c.Secure: + // return wrapValidationError("cookies", errors.New("if cookies_enabled equals true then cookies_secure must be true")) + case c.AllowedOrigins == nil || len(c.AllowedOrigins) == 0: + return wrapValidationError("cookies", errors.New("allowed_origins cannot be empty")) + default: + return nil + } +} + +func (c *Cookies) loadConfig(v *viper.Viper) error { + v.SetEnvPrefix("COOKIES") + c.Enabled = v.GetBool("ENABLED") + c.Domain = v.GetString("DOMAIN") + c.Path = v.GetString("PATH") + c.Secure = v.GetBool("SECURE") + c.AllowedOrigins = v.GetStringSlice("ALLOWED_ORIGINS") + sameSite := v.GetString("SAME_SITE") + switch strings.ToLower(strings.TrimSpace(sameSite)) { + case "strict": + c.SameSite = http.SameSiteStrictMode + case "none": + c.SameSite = http.SameSiteNoneMode + default: + c.SameSite = http.SameSiteLaxMode + } + return nil +} + +func (c *Cookies) copy() *Cookies { + return &Cookies{ + Enabled: c.Enabled, + Domain: c.Domain, + Path: c.Path, + Secure: c.Secure, + SameSite: c.SameSite, + AllowedOrigins: c.AllowedOrigins, + } +} diff --git a/service/internal/config/jwt.go b/service/internal/config/jwt.go new file mode 100644 index 0000000..a48bdd6 --- /dev/null +++ b/service/internal/config/jwt.go @@ -0,0 +1,63 @@ +package config + +import ( + "errors" + "os" + "strconv" + "time" + + "github.com/spf13/viper" +) + +type JWT struct { + AccessTokenLifetime time.Duration + RefreshTokenLifetime time.Duration + PublicKeyPath string + PrivateKeyPath string +} + +func (c *JWT) Validate() error { + switch { + case c.AccessTokenLifetime <= 0: + return wrapValidationError("jwt", errors.New("access_token_lifetime cannot be empty")) + case c.RefreshTokenLifetime <= 0: + return wrapValidationError("jwt", errors.New("refresh_token_lifetime cannot be empty")) + case c.PrivateKeyPath == "": + return wrapValidationError("jwt", errors.New("private_key_path cannot be empty")) + case c.PublicKeyPath == "": + return wrapValidationError("jwt", errors.New("public_key_path cannot be empty")) + default: + return nil + } +} + +func (c *JWT) loadConfig(v *viper.Viper) error { + v.SetEnvPrefix("JWT") + c.AccessTokenLifetime = v.GetDuration("ACCESS_TOKEN_LIFETIME") + c.RefreshTokenLifetime = time.Hour + if value := os.Getenv("SESSION_TTL_SECONDS"); value != "" { + seconds, err := strconv.ParseInt(value, 10, 64) + if err != nil || seconds <= 0 || seconds > int64((1<<63-1)/time.Second) { + return wrapValidationError("session", errors.New("SESSION_TTL_SECONDS must be a positive integer duration in seconds")) + } + c.RefreshTokenLifetime = time.Duration(seconds) * time.Second + } else if legacy := v.GetString("REFRESH_TOKEN_LIFETIME"); legacy != "" { + duration, err := time.ParseDuration(legacy) + if err != nil { + return wrapValidationError("jwt", errors.New("invalid refresh_token_lifetime")) + } + c.RefreshTokenLifetime = duration + } + c.PublicKeyPath = v.GetString("PUBLIC_KEY_PATH") + c.PrivateKeyPath = v.GetString("PRIVATE_KEY_PATH") + return nil +} + +func (c *JWT) copy() *JWT { + return &JWT{ + AccessTokenLifetime: c.AccessTokenLifetime, + RefreshTokenLifetime: c.RefreshTokenLifetime, + PublicKeyPath: c.PublicKeyPath, + PrivateKeyPath: c.PrivateKeyPath, + } +} diff --git a/service/internal/config/pipeline.go b/service/internal/config/pipeline.go new file mode 100644 index 0000000..44a2cea --- /dev/null +++ b/service/internal/config/pipeline.go @@ -0,0 +1,48 @@ +package config + +import ( + "fmt" + "os" + "strconv" + "strings" + "time" +) + +type Pipeline struct { + Concurrency, Prefetch, MaxAttempts int + Timeout, Lease, Retention time.Duration + BrokerURL string +} + +func LoadPipeline() (*Pipeline, error) { + c := &Pipeline{Concurrency: 2, Prefetch: 2, MaxAttempts: 5, Timeout: 60 * time.Second, Lease: 90 * time.Second, Retention: 24 * time.Hour} + for k, p := range map[string]*int{"WORKER_CONCURRENCY": &c.Concurrency, "WORKER_PREFETCH": &c.Prefetch, "JOB_MAX_ATTEMPTS": &c.MaxAttempts} { + if v := os.Getenv(k); v != "" { + n, e := strconv.Atoi(v) + if e != nil || n < 1 || n > 64 { + return nil, fmt.Errorf("invalid %s", k) + } + *p = n + } + } + for k, p := range map[string]*time.Duration{"JOB_TIMEOUT": &c.Timeout, "JOB_LEASE": &c.Lease, "STAGING_RETENTION": &c.Retention} { + if v := os.Getenv(k); v != "" { + d, e := time.ParseDuration(v) + if e != nil || d <= 0 { + return nil, fmt.Errorf("invalid %s", k) + } + *p = d + } + } + if c.Lease <= c.Timeout || c.Prefetch > c.Concurrency { + return nil, fmt.Errorf("lease must exceed timeout; prefetch must not exceed concurrency") + } + if p := os.Getenv("BROKER_URL_FILE"); p != "" { + b, e := os.ReadFile(p) + if e != nil { + return nil, fmt.Errorf("read broker secret: %w", e) + } + c.BrokerURL = strings.TrimSpace(string(b)) + } + return c, nil +} diff --git a/service/internal/config/postgres.go b/service/internal/config/postgres.go new file mode 100644 index 0000000..a191875 --- /dev/null +++ b/service/internal/config/postgres.go @@ -0,0 +1,126 @@ +package config + +import ( + "errors" + "fmt" + "gitea.privcloud.tech/mtusi2026/internal/pkg/transportsecurity" + "math" + "net/url" + "os" + "strings" + "time" + + "github.com/spf13/viper" +) + +type Postgres struct { + Host string + Port int + User string + Password string + Database string + VersionTable string + RunMigrations bool + MaxConnections int + ConnectTimeout time.Duration + QueryTimeout time.Duration +} + +func (c *Postgres) Validate() error { + switch { + case c.Host == "": + return wrapValidationError("postgres", errors.New("host cannot be empty")) + case c.Port == 0: + return wrapValidationError("postgres", errors.New("port cannot be empty")) + case c.User == "": + return wrapValidationError("postgres", errors.New("user cannot be empty")) + case c.Password == "": + return wrapValidationError("postgres", errors.New("password cannot be empty")) + case c.Database == "": + return wrapValidationError("postgres", errors.New("database cannot be empty")) + case c.VersionTable == "": + return wrapValidationError("postgres", errors.New("version_table name cannot be empty")) + case c.MaxConnections <= 0 || int64(c.MaxConnections) > math.MaxInt32: + return wrapValidationError("postgres", errors.New("max_connections must be between 1 and 2147483647")) + case c.ConnectTimeout <= 0: + return wrapValidationError("postgres", errors.New("connect_timeout must be positive")) + case c.QueryTimeout < time.Millisecond: + return wrapValidationError("postgres", errors.New("query_timeout must be at least 1ms")) + default: + return nil + } +} + +func (c *Postgres) GetMaxConnections() int32 { return int32(c.MaxConnections) } +func (c *Postgres) GetConnectTimeout() time.Duration { return c.ConnectTimeout } +func (c *Postgres) GetQueryTimeout() time.Duration { return c.QueryTimeout } + +func (c *Postgres) GetVersionTableName() string { + if c == nil { + return "" + } + return c.VersionTable +} + +func (c *Postgres) GetURI() string { + if c == nil { + return "" + } + u := &url.URL{Scheme: "postgres", User: url.UserPassword(c.User, c.Password), Host: fmt.Sprintf("%s:%d", c.Host, c.Port), Path: "/" + c.Database} + q := url.Values{} + if transportsecurity.Enabled() { + q.Set("sslmode", "verify-full") + q.Set("sslrootcert", transportsecurity.Directory()+"/ca.crt") + } else { + q.Set("sslmode", "disable") + } + u.RawQuery = q.Encode() + return u.String() +} + +func (c *Postgres) GetRunMigrations() bool { + if c == nil { + return false + } + return c.RunMigrations +} + +func (c *Postgres) loadConfig(v *viper.Viper) error { + v.SetEnvPrefix("POSTGRES") + c.Host = v.GetString("HOST") + c.Port = v.GetInt("PORT") + c.User = v.GetString("USER") + c.Password = v.GetString("PASSWORD") + if path := v.GetString("PASSWORD_FILE"); path != "" { + b, e := os.ReadFile(path) + if e != nil { + return fmt.Errorf("read database credential: %w", e) + } + c.Password = strings.TrimSpace(string(b)) + } + c.Database = v.GetString("DATABASE") + c.VersionTable = v.GetString("VERSION_TABLE") + c.RunMigrations = v.GetBool("RUN_MIGRATIONS") + c.MaxConnections = v.GetInt("MAX_CONNECTIONS") + c.ConnectTimeout = v.GetDuration("CONNECT_TIMEOUT") + c.QueryTimeout = v.GetDuration("QUERY_TIMEOUT") + return nil +} + +func (c *Postgres) copy() *Postgres { + if c == nil { + return nil + } + return &Postgres{ + Host: c.Host, + Port: c.Port, + User: c.User, + Password: c.Password, + Database: c.Database, + MaxConnections: c.MaxConnections, + ConnectTimeout: c.ConnectTimeout, + QueryTimeout: c.QueryTimeout, + RunMigrations: c.RunMigrations, + VersionTable: c.VersionTable, + } +} diff --git a/service/internal/config/redis.go b/service/internal/config/redis.go new file mode 100644 index 0000000..43a4e06 --- /dev/null +++ b/service/internal/config/redis.go @@ -0,0 +1,90 @@ +package config + +import ( + "errors" + "fmt" + "gitea.privcloud.tech/mtusi2026/internal/pkg/transportsecurity" + "net" + "os" + "strings" + "time" + + "github.com/redis/go-redis/v9" + "github.com/spf13/viper" +) + +type Redis struct { + Host string + Port int + Username string + Password string + DB int +} + +func (c *Redis) Validate() error { + switch { + case c.Host == "": + return wrapValidationError("redis", errors.New("host cannot be empty")) + case c.Port == 0: + return wrapValidationError("redis", errors.New("port cannot be empty")) + default: + return nil + } +} + +func (c *Redis) BuildRedisOptions() *redis.Options { + if c == nil { + return nil + } + opts := &redis.Options{ + Addr: net.JoinHostPort(c.Host, fmt.Sprintf("%d", c.Port)), + Username: c.Username, + Password: c.Password, + DB: c.DB, + DialTimeout: 200 * time.Millisecond, + ReadTimeout: 200 * time.Millisecond, + WriteTimeout: 200 * time.Millisecond, + PoolTimeout: 200 * time.Millisecond, + MaxRetries: -1, + DialerRetries: 1, + ContextTimeoutEnabled: true, + } + if transportsecurity.Enabled() { + tlsConfig, err := transportsecurity.Client() + if err != nil { + panic("invalid Redis TLS configuration") + } + opts.TLSConfig = tlsConfig + } + return opts +} + +func (c *Redis) loadConfig(v *viper.Viper) error { + v.SetEnvPrefix("REDIS") + c.Host = v.GetString("HOST") + c.Port = v.GetInt("PORT") + c.DB = v.GetInt("DB") + c.Password = v.GetString("PASSWORD") + if path := v.GetString("PASSWORD_FILE"); path != "" { + b, e := os.ReadFile(path) + if e != nil { + return fmt.Errorf("read cache credential: %w", e) + } + c.Password = strings.TrimSpace(string(b)) + } + c.Username = v.GetString("USERNAME") + return nil +} + +func (c *Redis) copy() *Redis { + if c == nil { + return nil + } + return &Redis{ + Host: c.Host, + Port: c.Port, + Username: c.Username, + Password: c.Password, + DB: c.DB, + } +} diff --git a/service/internal/config/server.go b/service/internal/config/server.go new file mode 100644 index 0000000..d3f9fbe --- /dev/null +++ b/service/internal/config/server.go @@ -0,0 +1,69 @@ +package config + +import ( + "errors" + "fmt" + "time" + + "github.com/spf13/viper" +) + +type Server struct { + Host string + Port int + ReadTimeout time.Duration + WriteTimeout time.Duration + IdleTimeout time.Duration + ShutdownTimeout time.Duration + Enabled bool +} + +func (c *Server) Validate() error { + if !c.Enabled { + return nil + } + switch { + case c.Host == "": + return wrapValidationError("server", errors.New("host cannot be empty")) + case c.Port == 0: + return wrapValidationError("server", errors.New("port cannot be empty")) + case c.ReadTimeout == 0: + return wrapValidationError("server", errors.New("read_timeout cannot be empty")) + case c.WriteTimeout == 0: + return wrapValidationError("server", errors.New("write_timeout cannot be empty")) + case c.IdleTimeout == 0: + return wrapValidationError("server", errors.New("idle_timeout cannot be empty")) + case c.ShutdownTimeout == 0: + return wrapValidationError("server", errors.New("shutdown_timeout name cannot be empty")) + default: + return nil + } +} + +func (c *Server) loadConfig(v *viper.Viper) error { + v.SetEnvPrefix("APP") + c.Host = v.GetString("HOST") + c.Port = v.GetInt("PORT") + c.ReadTimeout = v.GetDuration("READ_TIMEOUT") + c.WriteTimeout = v.GetDuration("WRITE_TIMEOUT") + c.IdleTimeout = v.GetDuration("IDLE_TIMEOUT") + c.ShutdownTimeout = v.GetDuration("SHUTDOWN_TIMEOUT") + c.Enabled = v.GetBool("ENABLED") + return c.Validate() +} + +func (c *Server) BindAddress() string { + return fmt.Sprintf("%s:%d", c.Host, c.Port) +} + +func (c *Server) copy() *Server { + return &Server{ + Host: c.Host, + Port: c.Port, + ReadTimeout: c.ReadTimeout, + WriteTimeout: c.WriteTimeout, + IdleTimeout: c.IdleTimeout, + ShutdownTimeout: c.ShutdownTimeout, + Enabled: c.Enabled, + } +} diff --git a/service/internal/config/validation.go b/service/internal/config/validation.go new file mode 100644 index 0000000..aeb4502 --- /dev/null +++ b/service/internal/config/validation.go @@ -0,0 +1,11 @@ +package config + +import "fmt" + +func wrapValidationError(name string, err error) error { + if err == nil { + return nil + } + + return fmt.Errorf("%s validation error: %w", name, err) +} diff --git a/service/internal/dto/attachment.go b/service/internal/dto/attachment.go new file mode 100644 index 0000000..1469e14 --- /dev/null +++ b/service/internal/dto/attachment.go @@ -0,0 +1,48 @@ +package dto + +import ( + "gitea.privcloud.tech/mtusi2026/internal/gen/contracts" + "github.com/google/uuid" + "io" + "time" +) + +type AttachmentRecord struct { + Public contracts.Attachment + Key string + StagingKey string + PipelineVersion int + VariantKeys map[string]string + Missing bool +} +type UploadAttachment struct { + UserID, ChatID uuid.UUID + Filename, ContentType string + Body io.Reader +} +type AttachmentContent struct { + Body io.ReadCloser + ContentType string + Size int64 +} +type JobMessage struct { + Traceparent string `json:"traceparent,omitempty"` + Tracestate string `json:"tracestate,omitempty"` + SchemaVersion int `json:"schema_version"` + JobID uuid.UUID `json:"job_id"` + Type string `json:"type"` + AttachmentID uuid.UUID `json:"attachment_id"` + PipelineVersion int `json:"pipeline_version"` + Attempt int `json:"attempt"` + CreatedAt time.Time `json:"created_at"` +} +type AttachmentJob struct { + Message JobMessage + State string + Token uuid.UUID + LeaseUntil *time.Time + NextAttemptAt time.Time + AttemptLimit int + ErrorCode *string + UpdatedAt time.Time +} diff --git a/service/internal/dto/auth.go b/service/internal/dto/auth.go new file mode 100644 index 0000000..1b74891 --- /dev/null +++ b/service/internal/dto/auth.go @@ -0,0 +1,127 @@ +package dto + +import ( + "errors" + "net/netip" + "time" + + "github.com/google/uuid" + + "gitea.privcloud.tech/mtusi2026/internal/pkg/types/role" +) + +// ---- Tokens ---- + +type UserDataInToken struct { + TokenID uuid.UUID + UserID uuid.UUID + Role role.Role + IsAccess bool + + PhoneNumber *string + Email *string + + ExpiresAt time.Time + IssuedAt time.Time +} + +func (u UserDataInToken) Validate() error { + if u.TokenID == uuid.Nil { + return errors.New("invalid token id") + } + if u.UserID == uuid.Nil { + return errors.New("invalid user id") + } + if !u.Role.IsValid() { + return errors.New("invalid role") + } + if u.ExpiresAt.IsZero() { + return errors.New("invalid expiry") + } + if u.IssuedAt.IsZero() { + return errors.New("invalid issued") + } + return nil +} + +type Token struct { + Payload string +} + +type ( + AccessToken Token + RefreshToken Token +) + +type Tokens struct { + AccessToken AccessToken + RefreshToken RefreshToken +} + +func (t AccessToken) Token() *Token { + return new(Token(t)) +} + +func (t RefreshToken) Token() *Token { + return new(Token(t)) +} + +type StoredRefreshToken struct { + ID uuid.UUID + UserID uuid.UUID + TokenHash string + UserAgent string + IPAddress netip.Addr + Role role.Role + ExpiresAt time.Time + RevokedAt *time.Time + CreatedAt time.Time +} + +type RevokeRefreshToken struct { + ID uuid.UUID + RevokedAt time.Time +} + +type RevokeRefreshTokensByUserID struct { + UserID uuid.UUID + RevokedAt time.Time +} + +// ---- Auth ---- + +type Session struct { + ExpiresAt time.Time + Token string + TokenType string + User User +} + +type Login struct { + ClientIP string + Username string + Password string +} + +type LoginResult struct { + ExpiresAt time.Time + TokenType string + User User + Tokens Tokens +} + +// UserCredentials is internal authentication data, never a public user response. +type UserCredentials struct { + User User + PasswordHash string `json:"-"` +} + +type Logout struct { + Token string +} + +type Register struct { + Email string + Password string + RepeatPassword string +} diff --git a/service/internal/dto/chat.go b/service/internal/dto/chat.go new file mode 100644 index 0000000..eaf6bc8 --- /dev/null +++ b/service/internal/dto/chat.go @@ -0,0 +1,31 @@ +package dto + +import ( + "time" + + "github.com/google/uuid" +) + +type Chat struct { + ID uuid.UUID + Title string + MemberIDs []uuid.UUID + CreatedAt time.Time +} + +type CreateChat struct { + CreatorID uuid.UUID + Title string + MemberIDs []uuid.UUID +} + +type ChatMember struct { + UserID uuid.UUID + ChatID uuid.UUID +} + +type ChatFilters struct { + UserID uuid.UUID + Limit *int + AfterID *uuid.UUID +} diff --git a/service/internal/dto/message.go b/service/internal/dto/message.go new file mode 100644 index 0000000..48f33b6 --- /dev/null +++ b/service/internal/dto/message.go @@ -0,0 +1,45 @@ +package dto + +import ( + "time" + + "github.com/google/uuid" +) + +type Message struct { + AttachmentIDs []uuid.UUID + ID uuid.UUID + ChatID uuid.UUID + SenderID uuid.UUID + Text string + CreatedAt time.Time +} + +type CreateMessage struct { + AttachmentIDs []uuid.UUID + IdempotencyKey string + ChatID uuid.UUID + SenderID uuid.UUID + Text string +} + +type MessageFilters struct { + ChatID uuid.UUID + UserID uuid.UUID + Limit *int + AfterID *uuid.UUID +} + +// MessageAttachment contains only the data needed to authorize linking an attachment. +type MessageAttachment struct { + ID uuid.UUID + ChatID uuid.UUID + Missing bool + Status string +} + +type MessageReplay struct { + Hash []byte + HashVersion int + MessageID uuid.UUID +} diff --git a/service/internal/dto/page.go b/service/internal/dto/page.go new file mode 100644 index 0000000..2af3f60 --- /dev/null +++ b/service/internal/dto/page.go @@ -0,0 +1,25 @@ +package dto + +import ( + "time" + + "github.com/google/uuid" +) + +// Page is a page of records with the ID to use for the next page. +type Page[T any] struct { + Items []T + NextCursorID *uuid.UUID +} + +// PagePosition is a resolved database cursor, ordered by creation time and ID. +type PagePosition struct { + CreatedAt time.Time + ID uuid.UUID +} + +// PageQuery is prepared by a service; Limit includes the lookahead row. +type PageQuery struct { + Limit int + After *PagePosition +} diff --git a/service/internal/dto/user.go b/service/internal/dto/user.go new file mode 100644 index 0000000..b617e0a --- /dev/null +++ b/service/internal/dto/user.go @@ -0,0 +1,30 @@ +package dto + +import ( + "time" + + "github.com/google/uuid" +) + +type User struct { + ID uuid.UUID + Username string + DisplayName string + CreatedAt time.Time +} + +type CreateUser struct { + Username string + DisplayName string + Password string +} + +// CreateStoredUser contains the prepared database record, never a plaintext password. +type CreateStoredUser struct { + ID uuid.UUID + Username string + DisplayName string + PasswordHash string `json:"-"` + CreatedAt time.Time + UpdatedAt time.Time +} diff --git a/service/internal/gen/contracts/types.gen.go b/service/internal/gen/contracts/types.gen.go new file mode 100644 index 0000000..977fa8c --- /dev/null +++ b/service/internal/gen/contracts/types.gen.go @@ -0,0 +1,232 @@ +// Package contracts provides primitives to interact with the openapi HTTP API. +// +// Code generated by github.com/oapi-codegen/oapi-codegen/v2 version v2.5.1 DO NOT EDIT. +package contracts + +import ( + "time" + + openapi_types "github.com/oapi-codegen/runtime/types" +) + +const ( + BasicAuthScopes = "basicAuth.Scopes" + BearerAuthScopes = "bearerAuth.Scopes" +) + +// Defines values for AttachmentContentType. +const ( + AttachmentContentTypeImagejpeg AttachmentContentType = "image/jpeg" + AttachmentContentTypeImagepng AttachmentContentType = "image/png" +) + +// Defines values for AttachmentStatus. +const ( + Failed AttachmentStatus = "failed" + Processing AttachmentStatus = "processing" + Queued AttachmentStatus = "queued" + Ready AttachmentStatus = "ready" +) + +// Defines values for ErrorErrorCode. +const ( + Conflict ErrorErrorCode = "conflict" + InternalError ErrorErrorCode = "internal_error" + InvalidRequest ErrorErrorCode = "invalid_request" + NotFound ErrorErrorCode = "not_found" + PayloadTooLarge ErrorErrorCode = "payload_too_large" + RateLimited ErrorErrorCode = "rate_limited" + Unauthorized ErrorErrorCode = "unauthorized" + Unavailable ErrorErrorCode = "unavailable" + UnsupportedMediaType ErrorErrorCode = "unsupported_media_type" +) + +// Defines values for HealthStatus. +const ( + Ok HealthStatus = "ok" +) + +// Defines values for VariantContentType. +const ( + VariantContentTypeImagejpeg VariantContentType = "image/jpeg" + VariantContentTypeImagepng VariantContentType = "image/png" + VariantContentTypeImagewebp VariantContentType = "image/webp" +) + +// Defines values for VariantName. +const ( + VariantNameCrop VariantName = "crop" + VariantNameOptimized VariantName = "optimized" + VariantNameThumbnail VariantName = "thumbnail" +) + +// Defines values for DownloadAttachmentParamsVariant. +const ( + DownloadAttachmentParamsVariantCrop DownloadAttachmentParamsVariant = "crop" + DownloadAttachmentParamsVariantOptimized DownloadAttachmentParamsVariant = "optimized" + DownloadAttachmentParamsVariantOriginal DownloadAttachmentParamsVariant = "original" + DownloadAttachmentParamsVariantThumbnail DownloadAttachmentParamsVariant = "thumbnail" +) + +// Attachment defines model for Attachment. +type Attachment struct { + ChatId openapi_types.UUID `json:"chat_id"` + ContentType AttachmentContentType `json:"content_type"` + CreatedAt time.Time `json:"created_at"` + ErrorCode *string `json:"error_code"` + Filename string `json:"filename"` + Id openapi_types.UUID `json:"id"` + OwnerId openapi_types.UUID `json:"owner_id"` + Sha256 string `json:"sha256"` + SizeBytes int `json:"size_bytes"` + Status AttachmentStatus `json:"status"` + UpdatedAt time.Time `json:"updated_at"` + Variants []Variant `json:"variants"` +} + +// AttachmentContentType defines model for Attachment.ContentType. +type AttachmentContentType string + +// AttachmentStatus defines model for Attachment.Status. +type AttachmentStatus string + +// Chat defines model for Chat. +type Chat struct { + CreatedAt time.Time `json:"created_at"` + Id openapi_types.UUID `json:"id"` + MemberIds []openapi_types.UUID `json:"member_ids"` + Title string `json:"title"` +} + +// ChatPage defines model for ChatPage. +type ChatPage struct { + Items []Chat `json:"items"` + NextCursor *string `json:"next_cursor"` +} + +// CreateChat defines model for CreateChat. +type CreateChat struct { + MemberIds []openapi_types.UUID `json:"member_ids"` + Title string `json:"title"` +} + +// CreateMessage defines model for CreateMessage. +type CreateMessage struct { + AttachmentIds *[]openapi_types.UUID `json:"attachment_ids,omitempty"` + Text *string `json:"text,omitempty"` +} + +// CreateUser defines model for CreateUser. +type CreateUser struct { + DisplayName string `json:"display_name"` + Password *string `json:"password,omitempty"` + Username string `json:"username"` +} + +// Error defines model for Error. +type Error struct { + Error struct { + Code ErrorErrorCode `json:"code"` + Message string `json:"message"` + RequestId string `json:"request_id"` + } `json:"error"` +} + +// ErrorErrorCode defines model for Error.Error.Code. +type ErrorErrorCode string + +// Health defines model for Health. +type Health struct { + Status HealthStatus `json:"status"` +} + +// HealthStatus defines model for Health.Status. +type HealthStatus string + +// Message defines model for Message. +type Message struct { + AttachmentIds []openapi_types.UUID `json:"attachment_ids"` + ChatId openapi_types.UUID `json:"chat_id"` + CreatedAt time.Time `json:"created_at"` + Id openapi_types.UUID `json:"id"` + SenderId openapi_types.UUID `json:"sender_id"` + Text string `json:"text"` +} + +// MessagePage defines model for MessagePage. +type MessagePage struct { + Items []Message `json:"items"` + NextCursor *string `json:"next_cursor"` +} + +// Session defines model for Session. +type Session struct { + ExpiresAt time.Time `json:"expires_at"` + Token string `json:"token"` + TokenType string `json:"token_type"` + User User `json:"user"` +} + +// User defines model for User. +type User struct { + CreatedAt time.Time `json:"created_at"` + DisplayName string `json:"display_name"` + Id openapi_types.UUID `json:"id"` + Username string `json:"username"` +} + +// Variant defines model for Variant. +type Variant struct { + ContentType VariantContentType `json:"content_type"` + Height int `json:"height"` + Name VariantName `json:"name"` + Sha256 string `json:"sha256"` + SizeBytes int `json:"size_bytes"` + Width int `json:"width"` +} + +// VariantContentType defines model for Variant.ContentType. +type VariantContentType string + +// VariantName defines model for Variant.Name. +type VariantName string + +// DownloadAttachmentParams defines parameters for DownloadAttachment. +type DownloadAttachmentParams struct { + Variant *DownloadAttachmentParamsVariant `form:"variant,omitempty" json:"variant,omitempty"` +} + +// DownloadAttachmentParamsVariant defines parameters for DownloadAttachment. +type DownloadAttachmentParamsVariant string + +// ListChatsParams defines parameters for ListChats. +type ListChatsParams struct { + Limit *int `form:"limit,omitempty" json:"limit,omitempty"` + Cursor *string `form:"cursor,omitempty" json:"cursor,omitempty"` +} + +// UploadAttachmentParams defines parameters for UploadAttachment. +type UploadAttachmentParams struct { + Filename string `form:"filename" json:"filename"` +} + +// ListMessagesParams defines parameters for ListMessages. +type ListMessagesParams struct { + Limit *int `form:"limit,omitempty" json:"limit,omitempty"` + Cursor *string `form:"cursor,omitempty" json:"cursor,omitempty"` +} + +// CreateMessageParams defines parameters for CreateMessage. +type CreateMessageParams struct { + // IdempotencyKey Обязателен с lab3. Область: пользователь + метод + путь + ключ; хранить результат не менее 24 ч. Повтор того же JSON возвращает исходный 201 и то же сообщение, другой JSON — 409. + IdempotencyKey string `json:"Idempotency-Key"` +} + +// CreateChatJSONRequestBody defines body for CreateChat for application/json ContentType. +type CreateChatJSONRequestBody = CreateChat + +// CreateMessageJSONRequestBody defines body for CreateMessage for application/json ContentType. +type CreateMessageJSONRequestBody = CreateMessage + +// RegisterUserJSONRequestBody defines body for RegisterUser for application/json ContentType. +type RegisterUserJSONRequestBody = CreateUser diff --git a/service/internal/pkg/apperr/code.go b/service/internal/pkg/apperr/code.go new file mode 100644 index 0000000..429d640 --- /dev/null +++ b/service/internal/pkg/apperr/code.go @@ -0,0 +1,274 @@ +package apperr + +import ( + "errors" + "net/http" + + "github.com/jackc/pgerrcode" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgconn" + grpcCodes "google.golang.org/grpc/codes" +) + +//go:generate stringer -type=Code -linecomment +type Code uint16 + +const ( + // CodeBadRequest = {AppError: BAD_REQUEST, HTTP:400} + CodeBadRequest Code = iota + // CodePayloadTooLarge = {AppError: REQUEST_ENTITY_TOO_LARGE, HTTP:413} + CodePayloadTooLarge + // CodeValidation = {AppError: VALIDATION, HTTP:400} + CodeValidation + // CodeUnauthorized = {AppError: UNAUTHORIZED, HTTP:401} + CodeUnauthorized + // CodeForbidden = {AppError: FORBIDDEN, HTTP:403} + CodeForbidden + // CodeNotFound = {AppError: NOT_FOUND, HTTP:404} + CodeNotFound + // CodeAlreadyExists = {AppError: ALREADY_EXISTS, HTTP:409} + CodeAlreadyExists + // CodeConflict = {AppError: CONFLICT, HTTP:409} + CodeConflict + // CodeTooManyRequests = {AppError: TOO_MANY_REQUESTS, HTTP:429} + CodeTooManyRequests + // CodeTimeout = {AppError: TIMEOUT, HTTP:504} + CodeTimeout + // CodeInternal = {AppError: INTERNAL, HTTP:500} + CodeInternal + // CodeServiceUnavailable = {AppError: SERVICE_UNAVAILABLE, HTTP:503} + CodeServiceUnavailable + // CodeOK = {AppError: OK, HTTP:200} + CodeOK + // CodeLocked = {AppError: LOCKED, HTTP:423} + CodeLocked + // CodeUnsupportedMediaType = {AppError: UNSUPPORTED_MEDIA_TYPE, HTTP:415} + CodeUnsupportedMediaType + // CodeUnavailable = {AppError: SERVICE_UNAVAILABLE, HTTP: 503} + CodeUnavailable +) + +func (c Code) HTTPStatus() int { + if status, ok := httpStatusByCode[c]; ok { + return status + } + return http.StatusInternalServerError +} + +func (c Code) GRPCCode() grpcCodes.Code { + if code, ok := grpcCodeByCode[c]; ok { + return code + } + return grpcCodes.Internal +} + +func (c Code) PostgresCodes() []string { + codes := postgresCodesByCode[c] + if len(codes) == 0 { + return nil + } + + cp := make([]string, len(codes)) + copy(cp, codes) + return cp +} + +func (c Code) PostgresCode() string { + codes := c.PostgresCodes() + if len(codes) == 0 { + return "" + } + return codes[0] +} + +func CodeFromError(err error) Code { + if err == nil { + return CodeInternal + } + + var coded interface { + Code() Code + } + if errors.As(err, &coded) { + return coded.Code() + } + + if code, ok := codeFromRedisError(err, false, false); ok { + return code + } + + return CodeFromPostgresError(err) +} + +func HTTPStatusFromError(err error) int { + return CodeFromError(err).HTTPStatus() +} + +func GRPCCodeFromError(err error) grpcCodes.Code { + return CodeFromError(err).GRPCCode() +} + +func CodeFromHTTPStatus(status int) Code { + if code, ok := codeByHTTPStatus[status]; ok { + return code + } + return CodeInternal +} + +func CodeFromGRPC(code grpcCodes.Code) Code { + if appCode, ok := codeByGRPCCode[code]; ok { + return appCode + } + return CodeInternal +} + +func CodeFromPostgres(pgCode string) Code { + if code, ok := codeByPostgresCode[pgCode]; ok { + return code + } + return CodeInternal +} + +func CodeFromPostgresError(err error) Code { + if err == nil { + return CodeInternal + } + + if code, ok := codeByPgxError[err]; ok { + return code + } + + if pgErr, ok := errors.AsType[*pgconn.PgError](err); ok { + return CodeFromPostgres(pgErr.Code) + } + + return CodeInternal +} + +var ( + httpStatusByCode = map[Code]int{ + CodeBadRequest: http.StatusBadRequest, + CodeUnauthorized: http.StatusUnauthorized, + CodeForbidden: http.StatusForbidden, + CodeNotFound: http.StatusNotFound, + CodeAlreadyExists: http.StatusConflict, + CodeConflict: http.StatusConflict, + CodePayloadTooLarge: http.StatusRequestEntityTooLarge, + CodeTooManyRequests: http.StatusTooManyRequests, + CodeValidation: http.StatusBadRequest, + CodeTimeout: http.StatusGatewayTimeout, + CodeInternal: http.StatusInternalServerError, + CodeServiceUnavailable: http.StatusServiceUnavailable, + CodeUnsupportedMediaType: http.StatusUnsupportedMediaType, + CodeUnavailable: http.StatusServiceUnavailable, + CodeOK: http.StatusOK, + CodeLocked: http.StatusLocked, + } + codeByHTTPStatus = map[int]Code{ + http.StatusBadRequest: CodeBadRequest, + http.StatusUnauthorized: CodeUnauthorized, + http.StatusForbidden: CodeForbidden, + http.StatusNotFound: CodeNotFound, + http.StatusConflict: CodeConflict, + http.StatusRequestEntityTooLarge: CodePayloadTooLarge, + http.StatusTooManyRequests: CodeTooManyRequests, + http.StatusUnprocessableEntity: CodeValidation, + http.StatusGatewayTimeout: CodeTimeout, + http.StatusInternalServerError: CodeInternal, + http.StatusServiceUnavailable: CodeServiceUnavailable, + http.StatusOK: CodeOK, + http.StatusLocked: CodeLocked, + http.StatusUnsupportedMediaType: CodeUnavailable, + } + grpcCodeByCode = map[Code]grpcCodes.Code{ + CodeBadRequest: grpcCodes.InvalidArgument, + CodePayloadTooLarge: grpcCodes.ResourceExhausted, + CodeValidation: grpcCodes.InvalidArgument, + CodeUnauthorized: grpcCodes.Unauthenticated, + CodeForbidden: grpcCodes.PermissionDenied, + CodeNotFound: grpcCodes.NotFound, + CodeAlreadyExists: grpcCodes.AlreadyExists, + CodeConflict: grpcCodes.Aborted, + CodeTooManyRequests: grpcCodes.ResourceExhausted, + CodeTimeout: grpcCodes.DeadlineExceeded, + CodeInternal: grpcCodes.Internal, + CodeServiceUnavailable: grpcCodes.Unavailable, + CodeOK: grpcCodes.OK, + CodeLocked: grpcCodes.FailedPrecondition, + } + codeByGRPCCode = map[grpcCodes.Code]Code{ + grpcCodes.InvalidArgument: CodeBadRequest, + grpcCodes.ResourceExhausted: CodeTooManyRequests, + grpcCodes.Unauthenticated: CodeUnauthorized, + grpcCodes.PermissionDenied: CodeForbidden, + grpcCodes.NotFound: CodeNotFound, + grpcCodes.AlreadyExists: CodeAlreadyExists, + grpcCodes.Aborted: CodeConflict, + grpcCodes.FailedPrecondition: CodeLocked, + grpcCodes.DeadlineExceeded: CodeTimeout, + grpcCodes.Internal: CodeInternal, + grpcCodes.Unimplemented: CodeInternal, + grpcCodes.Unavailable: CodeServiceUnavailable, + grpcCodes.OK: CodeOK, + } + postgresCodesByCode = map[Code][]string{ + CodeBadRequest: { + pgerrcode.InvalidTextRepresentation, + pgerrcode.InvalidParameterValue, + }, + CodeValidation: { + pgerrcode.NotNullViolation, + pgerrcode.CheckViolation, + pgerrcode.StringDataRightTruncationDataException, + pgerrcode.NumericValueOutOfRange, + }, + CodeNotFound: { + pgerrcode.ForeignKeyViolation, + }, + CodeAlreadyExists: { + pgerrcode.UniqueViolation, + }, + CodeConflict: { + pgerrcode.ExclusionViolation, + pgerrcode.SerializationFailure, + pgerrcode.DeadlockDetected, + }, + CodeLocked: { + pgerrcode.LockNotAvailable, + pgerrcode.ObjectInUse, + }, + CodeTooManyRequests: { + pgerrcode.TooManyConnections, + }, + CodeTimeout: { + pgerrcode.QueryCanceled, + }, + CodeInternal: { + pgerrcode.ConnectionException, + pgerrcode.ConnectionFailure, + pgerrcode.SQLClientUnableToEstablishSQLConnection, + pgerrcode.SQLServerRejectedEstablishmentOfSQLConnection, + pgerrcode.TransactionResolutionUnknown, + }, + CodeServiceUnavailable: { + pgerrcode.CannotConnectNow, + }, + } + codeByPostgresCode = buildCodeByPostgresCode(postgresCodesByCode) + codeByPgxError = map[error]Code{ + pgx.ErrNoRows: CodeNotFound, + pgx.ErrTxClosed: CodeInternal, + pgx.ErrTxCommitRollback: CodeConflict, + pgx.ErrTooManyRows: CodeInternal, + } +) + +func buildCodeByPostgresCode(src map[Code][]string) map[string]Code { + dst := make(map[string]Code, len(src)) + for code, pgCodes := range src { + for _, pgCode := range pgCodes { + dst[pgCode] = code + } + } + return dst +} diff --git a/service/internal/pkg/apperr/code_string.go b/service/internal/pkg/apperr/code_string.go new file mode 100644 index 0000000..be690bf --- /dev/null +++ b/service/internal/pkg/apperr/code_string.go @@ -0,0 +1,37 @@ +// Code generated by "stringer -type=Code -linecomment"; DO NOT EDIT. + +package apperr + +import "strconv" + +func _() { + // An "invalid array index" compiler error signifies that the constant values have changed. + // Re-run the stringer command to generate them again. + var x [1]struct{} + _ = x[CodeBadRequest-0] + _ = x[CodePayloadTooLarge-1] + _ = x[CodeValidation-2] + _ = x[CodeUnauthorized-3] + _ = x[CodeForbidden-4] + _ = x[CodeNotFound-5] + _ = x[CodeAlreadyExists-6] + _ = x[CodeConflict-7] + _ = x[CodeTooManyRequests-8] + _ = x[CodeTimeout-9] + _ = x[CodeInternal-10] + _ = x[CodeServiceUnavailable-11] + _ = x[CodeOK-12] + _ = x[CodeLocked-13] +} + +const _Code_name = "CodeBadRequestCodePayloadTooLargeCodeValidationCodeUnauthorizedCodeForbiddenCodeNotFoundCodeAlreadyExistsCodeConflictCodeTooManyRequestsCodeTimeoutCodeInternalCodeServiceUnavailableCodeOKCodeLocked" + +var _Code_index = [...]uint8{0, 14, 33, 47, 63, 76, 88, 105, 117, 136, 147, 159, 181, 187, 197} + +func (i Code) String() string { + idx := int(i) - 0 + if i < 0 || idx >= len(_Code_index)-1 { + return "Code(" + strconv.FormatInt(int64(i), 10) + ")" + } + return _Code_name[_Code_index[idx]:_Code_index[idx+1]] +} diff --git a/service/internal/pkg/apperr/error.go b/service/internal/pkg/apperr/error.go new file mode 100644 index 0000000..7b45fe0 --- /dev/null +++ b/service/internal/pkg/apperr/error.go @@ -0,0 +1,123 @@ +package apperr + +import ( + "errors" + "fmt" + + grpcCodes "google.golang.org/grpc/codes" +) + +type AppError struct { + err error + msg string + code Code + meta metadata +} + +func New(code Code, msg string) error { + return &AppError{ + msg: msg, + code: code, + meta: metadata{}, + } +} + +func Wrap(err error, code Code, msg string) error { + m := AppError{ + err: err, + msg: msg, + code: code, + meta: metadata{}, + } + return &m +} + +func WrapAppError(err error, msg string) error { + m := AppError{ + err: err, + msg: msg, + code: CodeInternal, + meta: metadata{}, + } + + if mistake := As(err); mistake != nil { + m.err = mistake + m.code = mistake.code + m.meta = mistake.meta + } + + return &m +} + +func As(err error) *AppError { + if mistakeErr, ok := errors.AsType[*AppError](err); ok { + return mistakeErr + } + return nil +} + +func CodeIs(err error, code Code) bool { + if appErr := As(err); appErr != nil && appErr.Code() == code { + return true + } + return false +} + +func (e *AppError) Error() string { + if e == nil { + return "" + } + + if e.err == nil { + if e.msg != "" { + return e.msg + } + return e.code.String() + } + + if e.msg == "" { + return e.err.Error() + } + + return fmt.Sprintf("%s: %s", e.msg, e.err.Error()) +} + +func (e *AppError) Unwrap() error { + if e == nil { + return nil + } + return e.err +} + +func (e *AppError) Code() Code { + if e == nil { + return CodeInternal + } + return e.code +} + +func (e *AppError) Message() string { + if e == nil { + return "" + } + return e.msg +} + +func (e *AppError) Cause() error { + if e == nil { + return nil + } + return e.err +} + +func (e *AppError) Metadata() Metadata { + return e.meta +} + +func (e *AppError) HTTPStatus() int { + return e.Code().HTTPStatus() +} + +func (e *AppError) GRPCCode() grpcCodes.Code { + return e.Code().GRPCCode() +} diff --git a/service/internal/pkg/apperr/meta.go b/service/internal/pkg/apperr/meta.go new file mode 100644 index 0000000..195666e --- /dev/null +++ b/service/internal/pkg/apperr/meta.go @@ -0,0 +1,42 @@ +package apperr + +import ( + "fmt" + "maps" + "strings" +) + +var _ Metadata = (*metadata)(nil) + +type Metadata interface { + Get(key metakey) string + Clone() map[metakey]string + CloneToAnyMap() map[string]any + String() string +} + +type metadata map[metakey]string + +func (m metadata) Get(key metakey) string { + return m[key] +} + +func (m metadata) Clone() map[metakey]string { + return maps.Clone(m) +} + +func (m metadata) CloneToAnyMap() map[string]any { + newM := make(map[string]any, len(m)) + for k, v := range m { + newM[k.String()] = v + } + return newM +} + +func (m metadata) String() string { + var b strings.Builder + for k, v := range m { + b.WriteString(fmt.Sprintf("%s: %s, ", k, v)) + } + return b.String() +} diff --git a/service/internal/pkg/apperr/metakey.go b/service/internal/pkg/apperr/metakey.go new file mode 100644 index 0000000..7dc091f --- /dev/null +++ b/service/internal/pkg/apperr/metakey.go @@ -0,0 +1,19 @@ +package apperr + +type metakey uint16 + +const ( + PgErrorCode metakey = iota + RedisErrorCode +) + +func (m metakey) String() string { + switch m { + case PgErrorCode: + return "pg_err_code" + case RedisErrorCode: + return "redis_err_code" + default: + return "unknown" + } +} diff --git a/service/internal/pkg/apperr/postgres.go b/service/internal/pkg/apperr/postgres.go new file mode 100644 index 0000000..9b82e2c --- /dev/null +++ b/service/internal/pkg/apperr/postgres.go @@ -0,0 +1,81 @@ +package apperr + +import ( + "context" + "errors" + "fmt" + "io" + "net" + "strings" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgconn" +) + +func WrapPostgres(err error, msg string) error { + m := AppError{ + err: err, + msg: msg, + code: CodeInternal, + meta: metadata{}, + } + + m.handlePostgres() + + return &m +} + +func (e *AppError) handlePostgres() { + if e.err == nil { + e.code = CodeOK + return + } + + var networkError net.Error + var connectError *pgconn.ConnectError + if errors.Is(e.err, context.DeadlineExceeded) || errors.Is(e.err, io.EOF) || + errors.Is(e.err, io.ErrUnexpectedEOF) || errors.As(e.err, &networkError) || errors.As(e.err, &connectError) { + e.code = CodeServiceUnavailable + e.msg = "database unavailable" + return + } + if pgErr, ok := errors.AsType[*pgconn.PgError](e.err); ok { + if strings.HasPrefix(pgErr.Code, "08") || pgErr.Code == "57014" || + pgErr.Code == "57P01" || pgErr.Code == "57P02" || pgErr.Code == "57P03" || pgErr.Code == "53300" { + e.code = CodeServiceUnavailable + e.msg = "database unavailable" + return + } + e.meta[PgErrorCode] = pgErr.Code + e.code = CodeFromPostgres(pgErr.Code) + return + } + switch { + case errors.Is(e.err, pgx.ErrNoRows): + e.msg = fmt.Sprintf("not found: %s", e.msg) + e.code = CodeNotFound + return + case errors.Is(e.err, pgx.ErrTooManyRows): + e.msg = fmt.Sprintf("got too many rows: %s", e.msg) + e.code = CodeInternal + return + case errors.Is(e.err, pgx.ErrTxClosed): + e.msg = fmt.Sprintf("transaction is closed: %s", e.msg) + e.code = CodeInternal + return + case errors.Is(e.err, pgx.ErrTxCommitRollback): + e.msg = fmt.Sprintf("transaction commit resulted in rollback: %s", e.msg) + e.code = CodeInternal + return + } +} + +func (e *AppError) PgCode() string { + if e == nil { + return "" + } + if e.meta == nil { + return "" + } + return e.meta.Get(PgErrorCode) +} diff --git a/service/internal/pkg/apperr/redis.go b/service/internal/pkg/apperr/redis.go new file mode 100644 index 0000000..f2afe56 --- /dev/null +++ b/service/internal/pkg/apperr/redis.go @@ -0,0 +1,170 @@ +package apperr + +import ( + "context" + "errors" + "fmt" + "net" + "strings" + + "github.com/redis/go-redis/v9" +) + +func WrapRedis(err error, msg string) error { + m := AppError{ + err: err, + msg: msg, + meta: metadata{}, + } + + m.handleRedis() + + return &m +} + +func CodeFromRedisError(err error) Code { + code, ok := codeFromRedisError(err, true, true) + if !ok { + return CodeInternal + } + return code +} + +func (e *AppError) handleRedis() { + if e.err == nil { + e.code = CodeOK + return + } + + if code := redisErrorCode(e.err); code != "" { + e.meta[RedisErrorCode] = code + } + + if errors.Is(e.err, redis.Nil) { + e.msg = fmt.Sprintf("not found: %s", e.msg) + } + + e.code = CodeFromRedisError(e.err) +} + +func (e *AppError) RedisCode() string { + if e == nil { + return "" + } + if e.meta == nil { + return "" + } + return e.meta.Get(RedisErrorCode) +} + +func codeFromRedisError(err error, includeTransport bool, includeProtocolString bool) (Code, bool) { + if err == nil { + return CodeInternal, false + } + + switch { + case errors.Is(err, redis.Nil): + return CodeNotFound, true + case errors.Is(err, redis.TxFailedErr): + return CodeConflict, true + case errors.Is(err, redis.ErrCrossSlot): + return CodeBadRequest, true + case errors.Is(err, redis.ErrPoolTimeout): + return CodeTimeout, true + case errors.Is(err, redis.ErrPoolExhausted): + return CodeTooManyRequests, true + case errors.Is(err, redis.ErrClosed): + return CodeServiceUnavailable, true + } + + if includeTransport { + if errors.Is(err, context.Canceled) || + errors.Is(err, context.DeadlineExceeded) { + return CodeTimeout, true + } + + if netErr, ok := errors.AsType[net.Error](err); ok { + if netErr.Timeout() { + return CodeTimeout, true + } + return CodeServiceUnavailable, true + } + } + + hasRedisError := isRedisError(err) + if !hasRedisError && !includeProtocolString { + return CodeInternal, false + } + + switch { + case redis.IsExecAbortError(err): + return CodeConflict, true + case redis.IsMaxClientsError(err): + return CodeTooManyRequests, true + case redis.IsLoadingError(err), + redis.IsReadOnlyError(err), + redis.IsClusterDownError(err), + redis.IsTryAgainError(err), + redis.IsMasterDownError(err), + redis.IsNoReplicasError(err), + redis.IsOOMError(err): + return CodeServiceUnavailable, true + case redis.IsAuthError(err), + redis.IsPermissionError(err): + return CodeInternal, true + } + + if _, ok := redis.IsMovedError(err); ok { + return CodeServiceUnavailable, true + } + if _, ok := redis.IsAskError(err); ok { + return CodeServiceUnavailable, true + } + + if hasRedisError { + return CodeInternal, true + } + + return CodeInternal, false +} + +func isRedisError(err error) bool { + if err == nil { + return false + } + + var redisErr redis.Error + return errors.As(err, &redisErr) +} + +func redisErrorCode(err error) string { + switch { + case err == nil: + return "" + case errors.Is(err, redis.Nil): + return "nil" + case errors.Is(err, redis.TxFailedErr): + return "tx_failed" + case errors.Is(err, redis.ErrCrossSlot): + return "CROSSSLOT" + case errors.Is(err, redis.ErrPoolTimeout): + return "pool_timeout" + case errors.Is(err, redis.ErrPoolExhausted): + return "pool_exhausted" + case errors.Is(err, redis.ErrClosed): + return "closed" + } + + var redisErr redis.Error + if !errors.As(err, &redisErr) { + return "" + } + + msg := strings.TrimSpace(redisErr.Error()) + msg = strings.TrimPrefix(msg, "ERR ") + parts := strings.Fields(msg) + if len(parts) == 0 { + return "" + } + return parts[0] +} diff --git a/service/internal/pkg/auth/jwtprovider/claims.go b/service/internal/pkg/auth/jwtprovider/claims.go new file mode 100644 index 0000000..bc8dfac --- /dev/null +++ b/service/internal/pkg/auth/jwtprovider/claims.go @@ -0,0 +1,145 @@ +package jwtprovider + +import ( + "fmt" + "log/slog" + "time" + + "github.com/golang-jwt/jwt/v5" + "github.com/google/uuid" + + "gitea.privcloud.tech/mtusi2026/internal/dto" + "gitea.privcloud.tech/mtusi2026/internal/pkg/types/role" +) + +var _ Claims = (*jwtClaims)(nil) + +type Claims interface { + GetTopTokenID() (uuid.UUID, bool) + GetTopUserID() (uuid.UUID, bool) + GetTopRole() (role.Role, bool) + GetTopIsAccess() (bool, bool) + GetTopExpiresAt() (time.Time, bool) + GetTopIssuedAt() (time.Time, bool) +} + +type claimsParseFunc func(token *dto.Token) (*jwtClaims, error) + +type jwtClaims struct { + jwt.RegisteredClaims + + Nonce string `json:"nonce,omitempty"` + IsAccess bool `json:"is_access"` + Role role.Role `json:"role"` +} + +func (claims *jwtClaims) GetTopTokenID() (uuid.UUID, bool) { + parsed, err := uuid.Parse(claims.ID) + if err != nil { + return uuid.Nil, false + } + return parsed, true +} + +func (claims *jwtClaims) GetTopUserID() (uuid.UUID, bool) { + parsed, err := uuid.Parse(claims.Subject) + if err != nil { + return uuid.Nil, false + } + return parsed, true +} + +func (claims *jwtClaims) GetTopRole() (role.Role, bool) { + return claims.Role, true +} + +func (claims *jwtClaims) GetTopIsAccess() (bool, bool) { + return claims.IsAccess, true +} + +func (claims *jwtClaims) GetTopExpiresAt() (time.Time, bool) { + if claims.ExpiresAt == nil { + return time.Time{}, false + } + return claims.ExpiresAt.Time, true +} + +func (claims *jwtClaims) GetTopIssuedAt() (time.Time, bool) { + if claims.IssuedAt == nil { + return time.Time{}, false + } + return claims.IssuedAt.Time, true +} + +func (claims *jwtClaims) userData() (*dto.UserDataInToken, error) { + var u dto.UserDataInToken + + tokenID, err := uuid.Parse(claims.ID) + if err != nil { + return nil, fmt.Errorf("failed to parse token id: %w", err) + } + u.TokenID = tokenID + + userID, err := uuid.Parse(claims.Subject) + if err != nil { + return nil, fmt.Errorf("failed to parse user id from token %q: %w", claims.Subject, err) + } + u.UserID = userID + + u.Role = claims.Role + u.IsAccess = claims.IsAccess + + if claims.ExpiresAt != nil { + u.ExpiresAt = claims.ExpiresAt.Time + } + + if claims.IssuedAt != nil { + u.IssuedAt = claims.IssuedAt.Time + } + + return &u, nil +} + +func (p *provider) parseClaimsFromProtectedJWT(token *dto.Token) (*jwtClaims, error) { + if token == nil { + return nil, ErrTokenIsMissing + } + + claims := &jwtClaims{} + + parsed, err := jwt.ParseWithClaims( + token.Payload, + claims, + p.readKeyFunc, + jwt.WithValidMethods([]string{jwt.SigningMethodRS256.Alg()}), + jwt.WithExpirationRequired(), + jwt.WithIssuedAt(), + ) + if err != nil { + slog.Error("failed to parse token with claims", "error", err) + return nil, ErrInvalidToken + } + + if !parsed.Valid { + slog.Error("failed to parse token: provided invalid token", "valid", parsed.Valid) + return nil, ErrInvalidToken + } + + return claims, nil +} + +func (p *provider) parseClaimsFromExposedJWT(token *dto.Token) (*jwtClaims, error) { + if token == nil { + return nil, ErrTokenIsMissing + } + + claims := &jwtClaims{} + + _, _, err := new(jwt.Parser).ParseUnverified(token.Payload, claims) + if err != nil { + slog.Error("failed to parse token: provided invalid token") + return nil, ErrInvalidToken + } + + return claims, nil +} diff --git a/service/internal/pkg/auth/jwtprovider/cookie.go b/service/internal/pkg/auth/jwtprovider/cookie.go new file mode 100644 index 0000000..8609585 --- /dev/null +++ b/service/internal/pkg/auth/jwtprovider/cookie.go @@ -0,0 +1,84 @@ +package jwtprovider + +import ( + "net/http" +) + +const ( + accessTokenCookieKey = "auth-access-token" + refreshTokenCookieKey = "auth-refresh-token" + expiresAtCookieKey = "auth-exp" +) + +type CookieBuilderFabric struct { + fabric func() http.Cookie +} + +func newCookieBuilder( + domain string, + path string, + secure bool, + sameSite http.SameSite, +) *CookieBuilderFabric { + return &CookieBuilderFabric{ + fabric: newCookieFabric(domain, path, secure, sameSite), + } +} + +func (b *CookieBuilderFabric) Builder() *CookieBuilder { + return &CookieBuilder{ + cookie: b.fabric(), + } +} + +func newCookieFabric( + _ string, + path string, + secure bool, + sameSite http.SameSite) func() http.Cookie { + return func() http.Cookie { + return http.Cookie{ + Path: path, + Secure: secure, + SameSite: sameSite, + } + } +} + +type CookieBuilder struct { + cookie http.Cookie +} + +func (b *CookieBuilder) AccessName() *CookieBuilder { + b.cookie.Name = accessTokenCookieKey + return b +} + +func (b *CookieBuilder) ExpiresName() *CookieBuilder { + b.cookie.Name = expiresAtCookieKey + return b +} + +func (b *CookieBuilder) RefreshName() *CookieBuilder { + b.cookie.Name = refreshTokenCookieKey + return b +} + +func (b *CookieBuilder) Value(value string) *CookieBuilder { + b.cookie.Value = value + return b +} + +func (b *CookieBuilder) HTTPOnly() *CookieBuilder { + b.cookie.HttpOnly = true + return b +} + +func (b *CookieBuilder) MaxAge(maxAge int) *CookieBuilder { + b.cookie.MaxAge = maxAge + return b +} + +func (b *CookieBuilder) Build() *http.Cookie { + return new(b.cookie) +} diff --git a/service/internal/pkg/auth/jwtprovider/error.go b/service/internal/pkg/auth/jwtprovider/error.go new file mode 100644 index 0000000..6fde812 --- /dev/null +++ b/service/internal/pkg/auth/jwtprovider/error.go @@ -0,0 +1,18 @@ +package jwtprovider + +import ( + "errors" +) + +var ( + ErrDecode = errors.New("failed to decode argument") + ErrNotFound = errors.New("not found") + ErrUnexpectedAlgorithm = errors.New("unexpected signing algorithm") + ErrTokenIsMissing = errors.New("token is missing") + ErrInvalidToken = errors.New("provided invalid token") + ErrInvalidUserDataInToken = errors.New("invalid user data in token") + ErrInvalidTokenClaims = errors.New("provided invalid token claims") + ErrInvalidBirthDateFormat = errors.New("provided invalid birth date format") + ErrTokenExpired = errors.New("token is expired") + ErrFailedToSign = errors.New("failed to sign jwt") +) diff --git a/service/internal/pkg/auth/jwtprovider/interface.go b/service/internal/pkg/auth/jwtprovider/interface.go new file mode 100644 index 0000000..ced83ee --- /dev/null +++ b/service/internal/pkg/auth/jwtprovider/interface.go @@ -0,0 +1,26 @@ +package jwtprovider + +import ( + "net/http" + + "github.com/google/uuid" + + "gitea.privcloud.tech/mtusi2026/internal/dto" + "gitea.privcloud.tech/mtusi2026/internal/pkg/types/role" +) + +type Provider interface { + Parse(token *dto.Token) (*dto.UserDataInToken, error) + CreateRefreshAndAccessTokensForUser(userID uuid.UUID, role role.Role) (*dto.Tokens, error) + ParseVerified(token *dto.Token) (*dto.UserDataInToken, error) + ParseUnverified(token *dto.Token) (*dto.UserDataInToken, error) + ValidateUserDataInToken(u *dto.UserDataInToken) error +} + +type Cookie interface { + Enabled() bool + Set(w http.ResponseWriter, tokens *dto.Tokens) + Clear(w http.ResponseWriter) + AccessToken(r *http.Request) *dto.AccessToken + RefreshToken(r *http.Request) *dto.RefreshToken +} diff --git a/service/internal/pkg/auth/jwtprovider/manager.go b/service/internal/pkg/auth/jwtprovider/manager.go new file mode 100644 index 0000000..baeda31 --- /dev/null +++ b/service/internal/pkg/auth/jwtprovider/manager.go @@ -0,0 +1,199 @@ +package jwtprovider + +import ( + "fmt" + "net/http" + "strconv" + "time" + + "gitea.privcloud.tech/mtusi2026/internal/config" + "gitea.privcloud.tech/mtusi2026/internal/dto" +) + +type manager struct { + enabled bool + cookieFabric *CookieBuilderFabric + prv Provider +} + +func NewCookieManager(cfg *config.Config, provider Provider) Cookie { + m := &manager{ + //enabled: cfg.Cookies.Enabled, + //cookieFabric: newCookieBuilder( + // cfg.Cookies.Domain, + // cfg.Cookies.Path, + // cfg.Cookies.Secure, + // cfg.Cookies.SameSite, + //), + prv: provider, + } + + if !m.enabled { + return m + } + + if err := m.cookieFabric. + Builder(). + AccessName(). + Value("test"). + HTTPOnly(). + MaxAge(0). + Build(). + Valid(); err != nil { + panic(fmt.Errorf("cookie validation failed: %w", err)) + } + + return m +} + +func (m *manager) Enabled() bool { + return m.enabled +} + +func (m *manager) Set(w http.ResponseWriter, tokens *dto.Tokens) { + if !m.enabled || tokens == nil { + return + } + + if tokens.AccessToken.Payload != "" { + m.setAccess(tokens.AccessToken, w) + } + + if tokens.RefreshToken.Payload != "" { + m.setRefresh(tokens.RefreshToken, w) + } +} + +func (m *manager) Clear(w http.ResponseWriter) { + if !m.enabled { + return + } + + for _, name := range []string{accessTokenCookieKey, refreshTokenCookieKey, expiresAtCookieKey} { + b := m.cookieFabric. + Builder(). + MaxAge(-1) + + switch name { + case expiresAtCookieKey: + b.ExpiresName() + case refreshTokenCookieKey: + b.RefreshName(). + HTTPOnly() + case accessTokenCookieKey: + b.AccessName(). + HTTPOnly() + } + http.SetCookie(w, b.Build()) + } +} + +func (m *manager) AccessToken(r *http.Request) *dto.AccessToken { + if !m.enabled { + return nil + } + + c, err := r.Cookie(accessTokenCookieKey) + if err != nil { + return nil + } + + return &dto.AccessToken{ + Payload: c.Value, + } +} + +func (m *manager) RefreshToken(r *http.Request) *dto.RefreshToken { + if !m.enabled { + return nil + } + + c, err := r.Cookie(refreshTokenCookieKey) + if err != nil { + return nil + } + + return &dto.RefreshToken{ + Payload: c.Value, + } +} + +func (m *manager) setAccess(token dto.AccessToken, w http.ResponseWriter) { + accessTTL := m.tokenTTL(token.Token()) + if accessTTL < 0 { + http.SetCookie(w, m.cookieFabric. + Builder(). + AccessName(). + HTTPOnly(). + MaxAge(-1). + Build(), + ) + http.SetCookie(w, m.cookieFabric. + Builder(). + ExpiresName(). + MaxAge(-1). + Build(), + ) + } else if tokenExpiresAt := m.tokenExpiry(token.Token()); !tokenExpiresAt.IsZero() { + http.SetCookie(w, m.cookieFabric. + Builder(). + AccessName(). + Value(token.Payload). + HTTPOnly(). + MaxAge(accessTTL). + Build(), + ) + http.SetCookie(w, m.cookieFabric. + Builder(). + ExpiresName(). + Value(strconv.FormatInt(tokenExpiresAt.Unix(), 10)). + MaxAge(accessTTL). + Build(), + ) + } +} + +func (m *manager) setRefresh(token dto.RefreshToken, w http.ResponseWriter) { + refreshTTL := m.tokenTTL(token.Token()) + if refreshTTL < 0 { + http.SetCookie(w, m.cookieFabric. + Builder(). + RefreshName(). + HTTPOnly(). + MaxAge(-1). + Build(), + ) + } else { + http.SetCookie(w, m.cookieFabric. + Builder(). + RefreshName(). + Value(token.Payload). + HTTPOnly(). + MaxAge(refreshTTL). + Build(), + ) + } +} + +func (m *manager) tokenTTL(token *dto.Token) int { + expiresAt := m.tokenExpiry(token) + + if expiresAt.IsZero() { + return -1 + } + + ttl := int(time.Until(expiresAt).Seconds()) + if ttl <= 0 { + return -1 + } + + return ttl +} + +func (m *manager) tokenExpiry(token *dto.Token) time.Time { + data, err := m.prv.ParseUnverified(token) + if err != nil { + return time.Time{} + } + return data.ExpiresAt +} diff --git a/service/internal/pkg/auth/jwtprovider/provider.go b/service/internal/pkg/auth/jwtprovider/provider.go new file mode 100644 index 0000000..a387854 --- /dev/null +++ b/service/internal/pkg/auth/jwtprovider/provider.go @@ -0,0 +1,187 @@ +package jwtprovider + +import ( + "crypto/rand" + "crypto/rsa" + "encoding/base64" + "fmt" + "log/slog" + "time" + + "github.com/golang-jwt/jwt/v5" + "github.com/google/uuid" + + "gitea.privcloud.tech/mtusi2026/internal/config" + "gitea.privcloud.tech/mtusi2026/internal/dto" + "gitea.privcloud.tech/mtusi2026/internal/pkg/types/environment" + "gitea.privcloud.tech/mtusi2026/internal/pkg/types/role" +) + +var ( + _ Provider = (*provider)(nil) +) + +type provider struct { + accessLifetime time.Duration + refreshLifetime time.Duration + publicKey *rsa.PublicKey + privateKey *rsa.PrivateKey + parseFunc claimsParseFunc +} + +func New(cfg *config.Config) Provider { + p := provider{ + accessLifetime: cfg.JWT.AccessTokenLifetime, + refreshLifetime: cfg.JWT.RefreshTokenLifetime, + } + p.parseFunc = p.parseClaimsFromProtectedJWT + + if cfg.Application.Environment == environment.ApplicationEnvironmentDev { + p.parseFunc = p.parseClaimsFromExposedJWT + } + + var err error + if p.publicKey, err = readPublicKeyFromFile(cfg.JWT.PublicKeyPath); err != nil { + slog.Error("failed to read public key", "error", err) + panic(fmt.Errorf("failed to read public key: %w", err)) + } + if p.privateKey, err = readPrivateKeyFromFile(cfg.JWT.PrivateKeyPath); err != nil { + slog.Error("failed to read private key", "error", err) + panic(fmt.Errorf("failed to read private key: %w", err)) + } + + return &p +} + +func (p *provider) Parse(token *dto.Token) (*dto.UserDataInToken, error) { + claims, err := p.parseFunc(token) + if err != nil { + return nil, err + } + return claims.userData() +} + +func (p *provider) ParseUnverified(t *dto.Token) (*dto.UserDataInToken, error) { + if t == nil { + return nil, ErrTokenIsMissing + } + + claims, err := p.parseClaimsFromExposedJWT(t) + if err != nil { + return nil, err + } + + return claims.userData() +} + +func (p *provider) ParseVerified(t *dto.Token) (*dto.UserDataInToken, error) { + if t == nil { + return nil, ErrTokenIsMissing + } + claims, err := p.parseClaimsFromProtectedJWT(t) + if err != nil { + return nil, fmt.Errorf("failed to parse token: %w", err) + } + + return claims.userData() +} + +func (p *provider) CreateAccessTokenForUser(userID uuid.UUID, role role.Role) (*dto.AccessToken, error) { + t, err := p.createTokenForUser(userID, true, role) + if err != nil { + return nil, err + } + return &dto.AccessToken{ + Payload: t.Payload, + }, nil +} + +func (p *provider) CreateRefreshTokenForUser(userID uuid.UUID, role role.Role) (*dto.RefreshToken, error) { + t, err := p.createTokenForUser(userID, false, role) + if err != nil { + return nil, err + } + return &dto.RefreshToken{ + Payload: t.Payload, + }, nil +} + +func (p *provider) CreateRefreshAndAccessTokensForUser(userID uuid.UUID, role role.Role) (*dto.Tokens, error) { + access, err := p.CreateAccessTokenForUser(userID, role) + if err != nil { + return nil, err + } + refresh, err := p.CreateRefreshTokenForUser(userID, role) + if err != nil { + return nil, err + } + return &dto.Tokens{ + AccessToken: *access, + RefreshToken: *refresh, + }, nil +} + +func (p *provider) createTokenForUser(userID uuid.UUID, isAccess bool, role role.Role) (*dto.Token, error) { + now := time.Now() + var tokenExpiresAt time.Time + if isAccess { + tokenExpiresAt = now.Add(p.accessLifetime) + } else { + tokenExpiresAt = now.Add(p.refreshLifetime) + } + var nonce [32]byte + if _, err := rand.Read(nonce[:]); err != nil { + return nil, fmt.Errorf("generate token nonce: %w", err) + } + claims := &jwtClaims{ + Nonce: base64.RawURLEncoding.EncodeToString(nonce[:]), + RegisteredClaims: jwt.RegisteredClaims{ + Subject: userID.String(), + ExpiresAt: jwt.NewNumericDate(tokenExpiresAt), + NotBefore: jwt.NewNumericDate(now), + IssuedAt: jwt.NewNumericDate(now), + ID: uuid.NewString(), + }, + IsAccess: isAccess, + Role: role, + } + t := jwt.NewWithClaims(jwt.SigningMethodRS256, claims) + + payload, err := t.SignedString(p.privateKey) + if err != nil { + slog.Error("failed to sign jwt", "error", err) + return nil, ErrFailedToSign + } + return &dto.Token{Payload: payload}, nil +} + +// TODO refactor maybe delete +func (p *provider) ValidateUserDataInToken(u *dto.UserDataInToken) error { + if u == nil { + return ErrInvalidUserDataInToken + } + + switch { + case u.TokenID == uuid.Nil: + return fmt.Errorf("%w: unexpected nil token id", ErrInvalidUserDataInToken) + case u.UserID == uuid.Nil: + return fmt.Errorf("%w: unexpected nil user id", ErrInvalidUserDataInToken) + case u.IssuedAt.IsZero(): + return fmt.Errorf("%w: unexpected zero issued at", ErrInvalidUserDataInToken) + case u.Email != nil && *u.Email == "": + return fmt.Errorf("%w: unexpected empty email", ErrInvalidUserDataInToken) + case u.PhoneNumber != nil && *u.PhoneNumber == "": + return fmt.Errorf("%w: unexpected empty phone number", ErrInvalidUserDataInToken) + case u.ExpiresAt.IsZero(): + return fmt.Errorf("%w: unexpected zero expires at", ErrInvalidUserDataInToken) + } + + return nil +} + +func (p *provider) readKeyFunc(token *jwt.Token) (interface{}, error) { + if token.Method != jwt.SigningMethodRS256 { + return nil, fmt.Errorf("unexpected signing method: %v", token.Header["alg"]) + } + return p.publicKey, nil +} diff --git a/service/internal/pkg/auth/jwtprovider/readers.go b/service/internal/pkg/auth/jwtprovider/readers.go new file mode 100644 index 0000000..f2c3ae5 --- /dev/null +++ b/service/internal/pkg/auth/jwtprovider/readers.go @@ -0,0 +1,35 @@ +package jwtprovider + +import ( + "crypto/rsa" + "fmt" + "os" + + "github.com/golang-jwt/jwt/v5" +) + +func readPrivateKeyFromFile(path string) (*rsa.PrivateKey, error) { + key, err := os.ReadFile(path) + if err != nil { + return nil, fmt.Errorf("failed to read public key: %w", err) + } + + privateKey, err := jwt.ParseRSAPrivateKeyFromPEM(key) + if err != nil { + return nil, fmt.Errorf("failed to parse private key: %w", err) + } + return privateKey, nil +} + +func readPublicKeyFromFile(path string) (*rsa.PublicKey, error) { + key, err := os.ReadFile(path) + if err != nil { + return nil, fmt.Errorf("failed to read public key: %w", err) + } + + privateKey, err := jwt.ParseRSAPublicKeyFromPEM(key) + if err != nil { + return nil, fmt.Errorf("failed to parse private key: %w", err) + } + return privateKey, nil +} diff --git a/service/internal/pkg/broker/rabbitmq.go b/service/internal/pkg/broker/rabbitmq.go new file mode 100644 index 0000000..b3ced1a --- /dev/null +++ b/service/internal/pkg/broker/rabbitmq.go @@ -0,0 +1,193 @@ +package broker + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "gitea.privcloud.tech/mtusi2026/internal/dto" + "gitea.privcloud.tech/mtusi2026/internal/pkg/fault" + "gitea.privcloud.tech/mtusi2026/internal/pkg/telemetry" + "gitea.privcloud.tech/mtusi2026/internal/pkg/transportsecurity" + "gitea.privcloud.tech/mtusi2026/internal/service/outbox" + "github.com/google/uuid" + amqp "github.com/rabbitmq/amqp091-go" + "go.opentelemetry.io/otel/trace" + "io" + "log/slog" + "net" + "regexp" + "time" +) + +const Queue = "attachments.v1" + +var tracePattern = regexp.MustCompile(`^00-[0-9a-f]{32}-[0-9a-f]{16}-[0-9a-f]{2}$`) + +type Rabbit struct{ URL string } + +func (r *Rabbit) connect(deadline time.Duration) (*amqp.Connection, *amqp.Channel, error) { + dial := amqp.DefaultDial(5 * time.Second) + if deadline > 0 { + dial = func(network, address string) (net.Conn, error) { + c, e := net.DialTimeout(network, address, 5*time.Second) + if e != nil { + return nil, e + } + if e = c.SetDeadline(time.Now().Add(deadline)); e != nil { + c.Close() + return nil, e + } + return c, nil + } + } + cfg := amqp.Config{Heartbeat: 10 * time.Second, Dial: dial} + if transportsecurity.Enabled() { + if len(r.URL) < 8 || r.URL[:8] != "amqps://" { + return nil, nil, errors.New("broker TLS required") + } + tc, e := transportsecurity.Client() + if e != nil { + return nil, nil, e + } + cfg.TLSClientConfig = tc + cfg.SASL = []amqp.Authentication{&amqp.ExternalAuth{}} + } + conn, e := amqp.DialConfig(r.URL, cfg) + if e != nil { + return nil, nil, errors.New("broker connection unavailable") + } + ch, e := conn.Channel() + if e != nil { + conn.Close() + return nil, nil, errors.New("broker channel unavailable") + } + + return conn, ch, nil +} +func (r *Rabbit) Publish(ctx context.Context, m outbox.DeliveryMessage) (err error) { + var job dto.JobMessage + if e := json.Unmarshal(m.Payload, &job); e != nil { + return e + } + ctx, sp := telemetry.Start(telemetry.Extract(ctx, job.Traceparent, job.Tracestate), "attachments publish", trace.SpanKindProducer) + defer func() { telemetry.End(sp, err) }() + job.Traceparent, job.Tracestate = telemetry.Inject(ctx) + body, marshalErr := json.Marshal(job) + if marshalErr != nil { + return marshalErr + } + conn, ch, e := r.connect(20 * time.Second) + if e != nil { + return e + } + defer conn.Close() + defer ch.Close() + if e = ch.Confirm(false); e != nil { + return e + } + confirms := ch.NotifyPublish(make(chan amqp.Confirmation, 1)) + returns := ch.NotifyReturn(make(chan amqp.Return, 1)) + fault.Hit(ctx, "before_publish", job.JobID) + if e = ch.PublishWithContext(ctx, "", Queue, true, false, amqp.Publishing{DeliveryMode: amqp.Persistent, ContentType: "application/json", MessageId: m.ID.String(), Body: body}); e != nil { + return errors.New("broker publish failed") + } + select { + case c, ok := <-confirms: + if !ok || !c.Ack { + return errors.New("broker did not confirm") + } + select { + case <-returns: + return errors.New("unroutable job") + default: + } + case <-time.After(15 * time.Second): + return errors.New("confirm timeout") + case <-ctx.Done(): + return ctx.Err() + } + fault.Hit(ctx, "after_publish", job.JobID) + return nil +} +func (r *Rabbit) Consume(ctx context.Context, concurrency, prefetch int, process func(context.Context, dto.JobMessage) error) error { + conn, ch, e := r.connect(0) + if e != nil { + return e + } + defer conn.Close() + defer ch.Close() + if e = ch.Qos(prefetch, 0, false); e != nil { + return e + } + deliveries, e := ch.Consume(Queue, "worker-"+uuid.NewString(), false, false, false, false, nil) + if e != nil { + return e + } + done := make(chan struct{}) + go func() { + select { + case <-ctx.Done(): + conn.Close() + case <-done: + } + }() + defer close(done) + tasks := make(chan amqp.Delivery) + finished := make(chan struct{}, concurrency) + for range concurrency { + go func() { + defer func() { finished <- struct{}{} }() + for d := range tasks { + var m dto.JobMessage + dec := json.NewDecoder(bytes.NewReader(d.Body)) + dec.DisallowUnknownFields() + e := dec.Decode(&m) + if e != nil || dec.Decode(new(any)) != io.EOF || m.SchemaVersion != 1 || m.Type != "attachment.process.v1" || m.JobID == uuid.Nil || m.AttachmentID == uuid.Nil || m.Attempt < 1 || m.PipelineVersion < 1 || m.CreatedAt.IsZero() || len(m.Tracestate) > 512 || (m.Traceparent != "" && !tracePattern.MatchString(m.Traceparent)) { + _ = d.Reject(false) + continue + } + jobctx, sp := telemetry.Start(telemetry.Extract(ctx, m.Traceparent, m.Tracestate), "attachments consume", trace.SpanKindConsumer) + slog.InfoContext(jobctx, "job_received", "job_id", m.JobID, "attempt", m.Attempt) + e = process(jobctx, m) + telemetry.End(sp, e) + if e != nil { + slog.WarnContext(jobctx, "job_not_acknowledged", "job_id", m.JobID, "reason", "processing_or_dependency_unavailable") + select { + case <-ctx.Done(): + case <-time.After(200 * time.Millisecond): + } + _ = d.Nack(false, true) + continue + } + fault.Hit(ctx, "before_ack", m.JobID) + _ = d.Ack(false) + } + }() + } + for d := range deliveries { + select { + case tasks <- d: + case <-ctx.Done(): + } + } + close(tasks) + for range concurrency { + <-finished + } + if ctx.Err() != nil { + return ctx.Err() + } + return errors.New("broker consumer disconnected") +} + +func (r *Rabbit) Depth(ctx context.Context) (int, error) { + conn, ch, e := r.connect(2 * time.Second) + if e != nil { + return 0, e + } + defer conn.Close() + defer ch.Close() + q, e := ch.QueueInspect(Queue) + return q.Messages, e +} diff --git a/service/internal/pkg/cache/redis.go b/service/internal/pkg/cache/redis.go new file mode 100644 index 0000000..91f90eb --- /dev/null +++ b/service/internal/pkg/cache/redis.go @@ -0,0 +1,43 @@ +package cache + +import ( + "context" + "time" + + "github.com/redis/go-redis/v9" + + "gitea.privcloud.tech/mtusi2026/internal/config" +) + +const defaultRedisPort = 6379 + +// OperationTimeout bounds each optional cache operation independently of the request. +const OperationTimeout = 300 * time.Millisecond + +type Client struct { + *redis.Client +} + +func New(cfg *config.Config) *Client { + if cfg.Redis.Port == 0 { + cfg.Redis.Port = defaultRedisPort + } + + return &Client{ + Client: redis.NewClient(cfg.Redis.BuildRedisOptions()), + } +} + +func (c *Client) Ping(ctx context.Context) error { + ctx, cancel := context.WithTimeout(ctx, OperationTimeout) + defer cancel() + return c.Client.Ping(ctx).Err() +} + +func (c *Client) Start(ctx context.Context) error { + return c.Ping(ctx) +} + +func (c *Client) Stop(_ context.Context) error { + return c.Client.Close() +} diff --git a/service/internal/pkg/encrypting/argon2id.go b/service/internal/pkg/encrypting/argon2id.go new file mode 100644 index 0000000..aab8e32 --- /dev/null +++ b/service/internal/pkg/encrypting/argon2id.go @@ -0,0 +1,120 @@ +package encrypting + +import ( + "context" + "crypto/rand" + "crypto/subtle" + "encoding/base64" + "fmt" + "golang.org/x/crypto/argon2" + "os" + "strconv" + "strings" +) + +type profile struct { + memory, iterations uint32 + parallelism uint8 +} + +var slots = make(chan struct{}, 2) + +func desired() (profile, error) { + p := profile{65536, 3, 1} + for name, dst := range map[string]*uint32{"ARGON2_MEMORY_KIB": &p.memory, "ARGON2_ITERATIONS": &p.iterations} { + if s := os.Getenv(name); s != "" { + n, e := strconv.ParseUint(s, 10, 32) + if e != nil { + return p, e + } + *dst = uint32(n) + } + } + if s := os.Getenv("ARGON2_PARALLELISM"); s != "" { + n, e := strconv.ParseUint(s, 10, 8) + if e != nil { + return p, e + } + p.parallelism = uint8(n) + } + return p, check(p) +} +func check(p profile) error { + if p.memory < 19456 || p.memory > 65536 || p.iterations < 2 || p.iterations > 4 || p.parallelism < 1 || p.parallelism > 4 { + return fmt.Errorf("unsupported Argon2id resource profile") + } + return nil +} +func prefix(p profile) string { + return fmt.Sprintf("$argon2id$v=19$m=%d,t=%d,p=%d$", p.memory, p.iterations, p.parallelism) +} +func derive(ctx context.Context, p profile, secret string, salt []byte) ([]byte, error) { + select { + case slots <- struct{}{}: + defer func() { <-slots }() + case <-ctx.Done(): + return nil, ctx.Err() + } + if err := ctx.Err(); err != nil { + return nil, err + } + key := argon2.IDKey([]byte(secret), salt, p.iterations, p.memory, p.parallelism, 32) + return key, ctx.Err() +} +func (e *encryptor) EncryptSecretUsingArgon2id(ctx context.Context, secret string) (string, error) { + p, err := desired() + if err != nil { + return "", err + } + salt := make([]byte, 16) + if _, err = rand.Read(salt); err != nil { + return "", ErrFailedToEncryptPassword + } + key, err := derive(ctx, p, secret, salt) + if err != nil { + return "", err + } + return prefix(p) + base64.RawStdEncoding.EncodeToString(salt) + "$" + base64.RawStdEncoding.EncodeToString(key), nil +} +func parse(hash string) (profile, []byte, []byte, error) { + var p profile + if len(hash) > 160 { + return p, nil, nil, ErrFailedToComparePasswords + } + parts := strings.Split(hash, "$") + if len(parts) != 6 || parts[0] != "" || parts[1] != "argon2id" || parts[2] != "v=19" { + return p, nil, nil, ErrFailedToComparePasswords + } + if _, err := fmt.Sscanf(parts[3], "m=%d,t=%d,p=%d", &p.memory, &p.iterations, &p.parallelism); err != nil || check(p) != nil || prefix(p) != "$argon2id$v=19$"+parts[3]+"$" { + return p, nil, nil, ErrFailedToComparePasswords + } + enc := base64.RawStdEncoding.Strict() + salt, e := enc.DecodeString(parts[4]) + if e != nil || len(salt) != 16 { + return p, nil, nil, ErrFailedToComparePasswords + } + key, e := enc.DecodeString(parts[5]) + if e != nil || len(key) != 32 { + return p, nil, nil, ErrFailedToComparePasswords + } + return p, salt, key, nil +} +func (e *encryptor) CompareHashAndSecretUsingArgon2id(ctx context.Context, hash, secret string) error { + p, salt, stored, err := parse(hash) + if err != nil { + return err + } + key, err := derive(ctx, p, secret, salt) + if err != nil { + return err + } + if subtle.ConstantTimeCompare(stored, key) != 1 { + return ErrFailedToComparePasswords + } + return nil +} +func NeedsRehash(hash string) bool { + p, e := desired() + return e == nil && !strings.HasPrefix(hash, prefix(p)) +} +func ValidateProfile() error { _, e := desired(); return e } diff --git a/service/internal/pkg/encrypting/encryptor.go b/service/internal/pkg/encrypting/encryptor.go new file mode 100644 index 0000000..1495a5b --- /dev/null +++ b/service/internal/pkg/encrypting/encryptor.go @@ -0,0 +1,72 @@ +package encrypting + +import ( + "context" + "crypto/sha256" + "crypto/subtle" + "encoding/hex" + "strings" + + "golang.org/x/crypto/bcrypt" +) + +type encryptor struct { + cost int + generateFromPasswordFunc func([]byte, int) ([]byte, error) + compareHashAndPasswordFunc func([]byte, []byte) error +} + +func New() Encryptor { + return &encryptor{ + cost: bcrypt.DefaultCost, + generateFromPasswordFunc: bcrypt.GenerateFromPassword, + compareHashAndPasswordFunc: bcrypt.CompareHashAndPassword, + } +} + +func (e *encryptor) EncryptSecret(_ context.Context, secret string) (string, error) { + res, err := e.generateFromPasswordFunc([]byte(secret), e.cost) + if err != nil { + return "", ErrFailedToEncryptPassword + } + return string(res), nil +} + +// EncryptSecretUsingSum256 hashes a high-entropy session token, not a password. +func (e *encryptor) EncryptSecretUsingSum256(ctx context.Context, secret string) (string, error) { + if err := ctx.Err(); err != nil { + return "", err + } + sum := sha256.Sum256([]byte(secret)) + return "sha256:" + hex.EncodeToString(sum[:]), nil +} + +func (e *encryptor) CompareHashAndSecret(_ context.Context, hash, password string) error { + err := e.compareHashAndPasswordFunc([]byte(hash), []byte(password)) + if err != nil { + return ErrFailedToComparePasswords + } + return nil +} + +func (e *encryptor) CompareHashAndSecretUsingSum256(ctx context.Context, hash, secret string) error { + if err := ctx.Err(); err != nil { + return err + } + sum := sha256.Sum256([]byte(secret)) + if payload, ok := strings.CutPrefix(hash, "sha256:"); ok { + stored, err := hex.DecodeString(payload) + if err != nil || len(stored) != sha256.Size || subtle.ConstantTimeCompare(stored, sum[:]) != 1 { + return ErrFailedToComparePasswords + } + return nil + } + // Preserve existing lab2/lab3 sessions across a rolling deployment. + // Passwords continue to use Argon2id; only old random-token fingerprints use this fallback. + if strings.HasPrefix(hash, "$2") { + if err := e.compareHashAndPasswordFunc([]byte(hash), []byte(hex.EncodeToString(sum[:]))); err == nil { + return nil + } + } + return ErrFailedToComparePasswords +} diff --git a/service/internal/pkg/encrypting/error.go b/service/internal/pkg/encrypting/error.go new file mode 100644 index 0000000..ffbc22f --- /dev/null +++ b/service/internal/pkg/encrypting/error.go @@ -0,0 +1,8 @@ +package encrypting + +import "errors" + +var ( + ErrFailedToEncryptPassword = errors.New("failed to encrypt password") + ErrFailedToComparePasswords = errors.New("failed to compare passwords") +) diff --git a/service/internal/pkg/encrypting/interface.go b/service/internal/pkg/encrypting/interface.go new file mode 100644 index 0000000..4f60f24 --- /dev/null +++ b/service/internal/pkg/encrypting/interface.go @@ -0,0 +1,12 @@ +package encrypting + +import "context" + +type Encryptor interface { + EncryptSecretUsingArgon2id(ctx context.Context, secret string) (string, error) + CompareHashAndSecretUsingArgon2id(ctx context.Context, hash, secret string) error + EncryptSecret(ctx context.Context, secret string) (string, error) + EncryptSecretUsingSum256(_ context.Context, secret string) (string, error) + CompareHashAndSecret(ctx context.Context, hash, secret string) error + CompareHashAndSecretUsingSum256(ctx context.Context, hash, secret string) error +} diff --git a/service/internal/pkg/encrypting/token_test.go b/service/internal/pkg/encrypting/token_test.go new file mode 100644 index 0000000..20934b1 --- /dev/null +++ b/service/internal/pkg/encrypting/token_test.go @@ -0,0 +1,43 @@ +package encrypting + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "golang.org/x/crypto/bcrypt" + "testing" +) + +func TestSessionFingerprints(t *testing.T) { + e := New() + ctx := context.Background() + token := "synthetic-high-entropy-session-value" + hash, err := e.EncryptSecretUsingSum256(ctx, token) + if err != nil { + t.Fatal(err) + } + if err = e.CompareHashAndSecretUsingSum256(ctx, hash, token); err != nil { + t.Fatal(err) + } + for _, bad := range []string{"sha256:00", "sha256:invalid", "unknown"} { + if e.CompareHashAndSecretUsingSum256(ctx, bad, token) == nil { + t.Fatal("accepted malformed fingerprint") + } + } + if e.CompareHashAndSecretUsingSum256(ctx, hash, token+"x") == nil { + t.Fatal("accepted wrong token") + } + sum := sha256.Sum256([]byte(token)) + legacy, err := bcrypt.GenerateFromPassword([]byte(hex.EncodeToString(sum[:])), bcrypt.MinCost) + if err != nil { + t.Fatal(err) + } + if err = e.CompareHashAndSecretUsingSum256(ctx, string(legacy), token); err != nil { + t.Fatal("legacy session rejected", err) + } + canceled, cancel := context.WithCancel(ctx) + cancel() + if e.CompareHashAndSecretUsingSum256(canceled, hash, token) == nil { + t.Fatal("ignored cancellation") + } +} diff --git a/service/internal/pkg/fault/fault.go b/service/internal/pkg/fault/fault.go new file mode 100644 index 0000000..0d5af15 --- /dev/null +++ b/service/internal/pkg/fault/fault.go @@ -0,0 +1,64 @@ +// Package fault implements explicit, file-controlled barriers for the isolated lab. +package fault + +import ( + "context" + "errors" + "github.com/google/uuid" + "os" + "path/filepath" + "strings" + "time" +) + +func Hit(ctx context.Context, point string, id uuid.UUID) { + if os.Getenv("LAB5_FAULTS") != "1" { + return + } + p := filepath.Join("/faults", point+"."+id.String()) + b, e := os.ReadFile(p) + if e != nil { + p = filepath.Join("/faults", point+".all") + b, e = os.ReadFile(p) + if e != nil { + return + } + } + hit, e := os.OpenFile(p+".hit", os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0666) + if e != nil { + return + } + _, _ = hit.WriteString(id.String() + "\n") + _ = hit.Close() + switch strings.TrimSpace(string(b)) { + case "crash": + _ = os.Remove(p) + os.Exit(86) + case "pause": + for { + if _, e = os.Stat(p); os.IsNotExist(e) { + return + } + select { + case <-ctx.Done(): + return + case <-time.After(50 * time.Millisecond): + } + } + } +} + +func Error(point string, id uuid.UUID) error { + if os.Getenv("LAB5_FAULTS") != "1" { + return nil + } + b, e := os.ReadFile(filepath.Join("/faults", point+"."+id.String())) + if e != nil { + return nil + } + v := strings.TrimSpace(string(b)) + if v == "temporary" || v == "permanent" { + return errors.New(v) + } + return nil +} diff --git a/service/internal/pkg/imageprocessor/image.go b/service/internal/pkg/imageprocessor/image.go new file mode 100644 index 0000000..b87678c --- /dev/null +++ b/service/internal/pkg/imageprocessor/image.go @@ -0,0 +1,83 @@ +package imageprocessor + +import ( + "context" + "fmt" + "image" + _ "image/jpeg" + _ "image/png" + "os" + "os/exec" + "path/filepath" + "time" +) + +func run(ctx context.Context, args ...string) error { + cmd := exec.CommandContext(ctx, "magick", args...) + cmd.Env = append(os.Environ(), "MAGICK_MEMORY_LIMIT=256MiB", "MAGICK_MAP_LIMIT=0", "MAGICK_DISK_LIMIT=256MiB", "MAGICK_THREAD_LIMIT=1", "MAGICK_TIME_LIMIT=60") + cmd.WaitDelay = time.Second + if err := cmd.Run(); err != nil { + if ctx.Err() != nil { + return ctx.Err() + } + return fmt.Errorf("image conversion failed: %w", err) + } + return nil +} +func Validate(ctx context.Context, path string) error { + return run(ctx, "-regard-warnings", path, "null:") +} + +type Result struct { + Name, Path, MIME string + Width, Height int +} + +func fit(w, h, limit int) (int, int) { + if w <= limit && h <= limit { + return w, h + } + if w >= h { + return limit, max(1, h*limit/w) + } + return max(1, w*limit/h), limit +} +func Process(ctx context.Context, source, dir string, version int) ([]Result, error) { + normalized := filepath.Join(dir, "normalized.png") + if e := run(ctx, source, "-auto-orient", "-background", "white", "-alpha", "remove", "-alpha", "off", "-strip", "png:"+normalized); e != nil { + return nil, e + } + f, e := os.Open(normalized) + if e != nil { + return nil, e + } + cfg, _, e := image.DecodeConfig(f) + f.Close() + if e != nil { + return nil, e + } + tw, th := fit(cfg.Width, cfg.Height, 256) + ow, oh := fit(cfg.Width, cfg.Height, 1280) + out := []Result{{"thumbnail", filepath.Join(dir, "thumbnail.jpg"), "image/jpeg", tw, th}, {"crop", filepath.Join(dir, "crop.jpg"), "image/jpeg", 128, 128}, {"optimized", filepath.Join(dir, "optimized.webp"), "image/webp", ow, oh}} + for _, v := range out { + args := []string{normalized} + if v.Name == "crop" { + side := min(cfg.Width, cfg.Height) + args = append(args, "-gravity", "center", "-crop", fmt.Sprintf("%dx%d+0+0", side, side), "+repage") + } + args = append(args, "-resize", fmt.Sprintf("%dx%d!", v.Width, v.Height), "-strip", "-quality") + quality := "85" + if v.Name == "optimized" && version == 1 { + quality = "80" + } + coder := "jpeg:" + if v.Name == "optimized" { + coder = "webp:" + } + args = append(args, quality, coder+v.Path) + if e = run(ctx, args...); e != nil { + return nil, e + } + } + return out, nil +} diff --git a/service/internal/pkg/migrator/tern.go b/service/internal/pkg/migrator/tern.go new file mode 100644 index 0000000..493fce6 --- /dev/null +++ b/service/internal/pkg/migrator/tern.go @@ -0,0 +1,164 @@ +package migrator + +import ( + "context" + "io/fs" + "log/slog" + + "github.com/jackc/pgx/v5" + "github.com/jackc/tern/v2/migrate" + + "gitea.privcloud.tech/mtusi2026/internal/config" + "gitea.privcloud.tech/mtusi2026/internal/pkg/apperr" + "gitea.privcloud.tech/mtusi2026/migrations" +) + +type Migrator interface { + MigrateUp(ctx context.Context) error + MigrateDown(ctx context.Context) error + MigrateTo(ctx context.Context, version int32) error + GetVersion(ctx context.Context) (int32, error) + Configure(ctx context.Context) error + Close(ctx context.Context) error + Start(ctx context.Context) error + Stop(ctx context.Context) error +} + +type migrator struct { + cfg *config.Config + conn *pgx.Conn + migrator *migrate.Migrator + migrations fs.FS + configured bool + closed bool +} + +func New(cfg *config.Config) Migrator { + return &migrator{ + cfg: cfg, + conn: nil, + migrator: nil, + migrations: migrations.Migrations, + configured: false, + closed: false, + } +} + +func (m *migrator) MigrateUp(ctx context.Context) error { + if !m.configured { + return apperr.New(apperr.CodeInternal, "migrator was not configured") + } + return m.migrator.Migrate(ctx) +} + +func (m *migrator) MigrateDown(ctx context.Context) error { + if !m.configured { + return apperr.New(apperr.CodeInternal, "migrator was not configured") + } + return m.migrator.MigrateTo(ctx, 0) +} + +func (m *migrator) MigrateTo(ctx context.Context, version int32) error { + if !m.configured { + return apperr.New(apperr.CodeInternal, "migrator was not configured") + } + return m.migrator.MigrateTo(ctx, version) +} + +func (m *migrator) GetVersion(ctx context.Context) (int32, error) { + if !m.configured { + return -1, apperr.New(apperr.CodeInternal, "migrator was not configured") + } + return m.migrator.GetCurrentVersion(ctx) +} + +func (m *migrator) Configure(ctx context.Context) error { + if err := m.initConn(ctx); err != nil { + return err + } + if err := m.initMigrator(ctx); err != nil { + return err + } + if err := m.loadMigrations(); err != nil { + return err + } + m.configured = true + return nil +} + +func (m *migrator) Close(ctx context.Context) error { + if m.conn == nil || m.closed { + return nil + } + if err := m.closeConn(ctx); err != nil { + return err + } + m.closed = true + return nil +} + +func (m *migrator) Start(ctx context.Context) error { + if !m.cfg.Postgres.GetRunMigrations() { + slog.Info("flag run migrations is disabled: skipping migrating") + return nil + } + + err := m.Configure(ctx) + if err != nil { + slog.Error("failed to configure migrator", "error", err) + return err + } + err = m.MigrateUp(ctx) + if err != nil { + slog.Error("failed to migrate database up", "error", err) + return err + } + + if err = m.Close(ctx); err != nil { + slog.Error("failed close connection to database", "error", err) + return err + } + + m.closed = true + + return nil +} + +func (m *migrator) Stop(ctx context.Context) error { + return m.Close(ctx) +} + +func (m *migrator) loadMigrations() (err error) { + err = m.migrator.LoadMigrations(m.migrations) + if err != nil { + slog.Error("failed to load migrations", "error", err) + return apperr.New(apperr.CodeInternal, "failed to load migrations") + } + slog.Info("successfully loaded migrations", "error", err) + return nil +} + +func (m *migrator) initConn(ctx context.Context) (err error) { + m.closed = false + m.conn, err = pgx.Connect(ctx, m.cfg.Postgres.GetURI()) + if err != nil { + slog.Error("failed to create connection to database", "error", err) + return apperr.New(apperr.CodeInternal, "failed to create connection to database") + } + slog.Info("connected to database", "db", m.cfg.Postgres.Database) + return nil +} + +func (m *migrator) initMigrator(ctx context.Context) (err error) { + m.migrator, err = migrate.NewMigrator(ctx, m.conn, m.cfg.Postgres.GetVersionTableName()) + if err != nil { + slog.Error("failed to initialize migrator", "error", err) + return apperr.New(apperr.CodeInternal, "failed to initialize migrator") + } + slog.Info("successfully initialized migrator") + return nil +} + +func (m *migrator) closeConn(ctx context.Context) error { + return m.conn.Close(ctx) +} diff --git a/service/internal/pkg/objectstore/s3.go b/service/internal/pkg/objectstore/s3.go new file mode 100644 index 0000000..8ef898a --- /dev/null +++ b/service/internal/pkg/objectstore/s3.go @@ -0,0 +1,111 @@ +package objectstore + +import ( + "context" + "errors" + "gitea.privcloud.tech/mtusi2026/internal/pkg/telemetry" + "gitea.privcloud.tech/mtusi2026/internal/pkg/transportsecurity" + "go.opentelemetry.io/otel/trace" + "io" + "net/http" + "strings" + "time" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/credentials" + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/aws/smithy-go" + + "gitea.privcloud.tech/mtusi2026/internal/config" +) + +type Store struct { + Client *s3.Client + Bucket string +} + +func New(c *config.Attachment) *Store { + client := &http.Client{Timeout: 30 * time.Second} + if transportsecurity.Enabled() { + c, err := transportsecurity.Client() + if err != nil { + panic("invalid S3 TLS configuration") + } + client.Transport = &http.Transport{TLSClientConfig: c} + } + return &Store{ + Bucket: c.Bucket, + Client: s3.New( + s3.Options{ + Region: c.Region, + BaseEndpoint: aws.String(c.Endpoint), + UsePathStyle: true, + Credentials: credentials.NewStaticCredentialsProvider( + c.AccessKey, + c.SecretKey, + "", + ), + RetryMaxAttempts: 1, + HTTPClient: client, + }, + ), + } +} + +func (s *Store) Put(ctx context.Context, key, mime string, r io.Reader, size int64) (err error) { + ctx, sp := telemetry.Start(ctx, "s3.put", trace.SpanKindClient) + defer func() { telemetry.End(sp, err) }() + + _, err = s.Client.PutObject(ctx, &s3.PutObjectInput{Bucket: &s.Bucket, Key: &key, Body: r, ContentLength: &size, ContentType: &mime}) + return err +} + +func (s *Store) Get(ctx context.Context, key string) (out *s3.GetObjectOutput, err error) { + ctx, sp := telemetry.Start(ctx, "s3.get", trace.SpanKindClient) + defer func() { telemetry.End(sp, err) }() + + return s.Client.GetObject(ctx, &s3.GetObjectInput{Bucket: &s.Bucket, Key: &key}) +} + +func (s *Store) Head(ctx context.Context, key string) (out *s3.HeadObjectOutput, err error) { + ctx, sp := telemetry.Start(ctx, "s3.head", trace.SpanKindClient) + defer func() { telemetry.End(sp, err) }() + + return s.Client.HeadObject(ctx, &s3.HeadObjectInput{Bucket: &s.Bucket, Key: &key}) +} + +func (s *Store) Delete(ctx context.Context, key string) (err error) { + ctx, sp := telemetry.Start(ctx, "s3.delete", trace.SpanKindClient) + defer func() { telemetry.End(sp, err) }() + + _, err = s.Client.DeleteObject(ctx, &s3.DeleteObjectInput{Bucket: &s.Bucket, Key: &key}) + return err +} + +func (s *Store) Ping(ctx context.Context) (err error) { + ctx, sp := telemetry.Start(ctx, "s3.ping", trace.SpanKindClient) + defer func() { telemetry.End(sp, err) }() + + // Readiness verifies the S3 write path as well as transport/authentication. + key := "staging/health-probe" + if err = s.Put(ctx, key, "application/octet-stream", strings.NewReader("ready"), 5); err != nil { + return err + } + return s.Delete(ctx, key) +} + +func Missing(err error) bool { + var api smithy.APIError + return errors.As(err, &api) && (api.ErrorCode() == "NoSuchKey" || api.ErrorCode() == "NotFound" || api.ErrorCode() == "404") +} + +func (s *Store) List(ctx context.Context, token string) (out *s3.ListObjectsV2Output, err error) { + ctx, sp := telemetry.Start(ctx, "s3.list", trace.SpanKindClient) + defer func() { telemetry.End(sp, err) }() + + q := &s3.ListObjectsV2Input{Bucket: &s.Bucket, MaxKeys: aws.Int32(100)} + if token != "" { + q.ContinuationToken = &token + } + return s.Client.ListObjectsV2(ctx, q) +} diff --git a/service/internal/pkg/openbao/client.go b/service/internal/pkg/openbao/client.go new file mode 100644 index 0000000..d87d759 --- /dev/null +++ b/service/internal/pkg/openbao/client.go @@ -0,0 +1,211 @@ +// Package openbao obtains process-scoped credentials using a short-lived client certificate. +// Secrets are materialized only in the container tmpfs. Renewal never falls back to local credentials. +package openbao + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "gitea.privcloud.tech/mtusi2026/internal/pkg/transportsecurity" + "net/http" + "os" + "path/filepath" + "strings" + "time" +) + +type response struct { + Auth struct { + ClientToken string `json:"client_token"` + LeaseDuration int `json:"lease_duration"` + } `json:"auth"` + Data map[string]json.RawMessage `json:"data"` + LeaseID string `json:"lease_id"` + LeaseDuration int `json:"lease_duration"` +} +type client struct { + http *http.Client + addr, token string +} + +func (c *client) call(ctx context.Context, method, path string, body any) (response, error) { + var out response + b, err := json.Marshal(body) + if err != nil { + return out, err + } + req, err := http.NewRequestWithContext(ctx, method, c.addr+"/v1/"+path, bytes.NewReader(b)) + if err != nil { + return out, errors.New("invalid OpenBao endpoint") + } + req.Header.Set("Content-Type", "application/json") + if c.token != "" { + req.Header.Set("X-Vault-Token", c.token) + } + r, err := c.http.Do(req) + if err != nil { + return out, errors.New("OpenBao transport unavailable") + } + defer r.Body.Close() + if r.StatusCode < 200 || r.StatusCode >= 300 { + return out, errors.New("OpenBao operation denied or unavailable") + } + err = json.NewDecoder(r.Body).Decode(&out) + return out, err +} +func str(m map[string]json.RawMessage, k string) string { + var s string + _ = json.Unmarshal(m[k], &s) + return s +} + +// Start is used by short-lived operator probes. +func Start(ctx context.Context) (func(), error) { _, close, err := Connect(ctx); return close, err } + +// Connect returns the lifetime shared by every request and background operation. +func Connect(ctx context.Context) (context.Context, func(), error) { + addr := os.Getenv("BAO_ADDR") + if addr == "" { + return ctx, func() {}, nil + } + if !strings.HasPrefix(addr, "https://") { + return nil, nil, errors.New("OpenBao HTTPS required") + } + tlsConfig, err := transportsecurity.Client() + if err != nil { + return nil, nil, err + } + c := &client{http: &http.Client{Timeout: 3 * time.Second, Transport: &http.Transport{TLSClientConfig: tlsConfig}}, addr: addr} + role := os.Getenv("SERVICE_ROLE") + if role != "api" && role != "worker" && role != "dispatcher" { + return nil, nil, errors.New("invalid service role") + } + login := func() (time.Time, error) { + issuedAt := time.Now() + r, e := c.call(ctx, "POST", "auth/cert/login", map[string]string{"name": role}) + if e != nil { + return time.Time{}, e + } + if r.Auth.ClientToken == "" || r.Auth.LeaseDuration < 1 { + return time.Time{}, errors.New("invalid scoped token") + } + c.token = r.Auth.ClientToken + return issuedAt.Add(time.Duration(r.Auth.LeaseDuration) * time.Second), nil + } + tokenExpiry, err := login() + if err != nil { + return nil, nil, err + } + kv, err := c.call(ctx, "GET", "secret/data/"+role, nil) + if err != nil { + return nil, nil, err + } + var secrets map[string]string + if err = json.Unmarshal(kv.Data["data"], &secrets); err != nil { + return nil, nil, err + } + dbIssuedAt := time.Now() + db, err := c.call(ctx, "GET", "database/creds/"+role, nil) + if err != nil { + return nil, nil, err + } + if db.LeaseDuration < 1 || db.LeaseID == "" { + return nil, nil, errors.New("invalid DB lease") + } + dbExpiry := dbIssuedAt.Add(time.Duration(db.LeaseDuration) * time.Second) + directory := "/run/credentials" + if err = os.MkdirAll(directory, 0700); err != nil { + return nil, nil, err + } + for key, value := range secrets { + env := map[string]string{"s3_access": "S3_ACCESS_KEY_ID_FILE", "s3_secret": "S3_SECRET_ACCESS_KEY_FILE", "redis_password": "REDIS_PASSWORD_FILE"}[key] + if env == "" { + continue + } + p := filepath.Join(directory, key) + if err = os.WriteFile(p, []byte(value), 0600); err != nil { + return nil, nil, err + } + os.Setenv(env, p) + } + os.Setenv("POSTGRES_USER", str(db.Data, "username")) + os.Setenv("POSTGRES_PASSWORD", str(db.Data, "password")) + os.Unsetenv("POSTGRES_PASSWORD_FILE") + ctx, cancel := context.WithCancel(ctx) + certExpiry := tlsConfig.Certificates[0].Leaf.NotAfter + expiryUpdates := make(chan time.Time) + deadline := func() time.Time { return minTime(tokenExpiry, dbExpiry, certExpiry).Add(-3 * time.Second) } + initialDeadline := deadline() + // This independent watchdog still fires while a renewal HTTP request is blocked. + go func() { + timer := time.NewTimer(max(time.Until(initialDeadline), 0)) + defer timer.Stop() + for { + select { + case <-ctx.Done(): + return + case <-timer.C: + cancel() + return + case at := <-expiryUpdates: + timer.Reset(max(time.Until(at), 0)) + } + } + }() + go func() { + tick := time.NewTicker(15 * time.Second) + defer tick.Stop() + for { + select { + case <-ctx.Done(): + return + case <-tick.C: + renewedAt := time.Now() + renewed, e := c.call(ctx, "POST", "auth/token/renew-self", map[string]int{"increment": 300}) + if e == nil && renewed.Auth.LeaseDuration > 0 { + tokenExpiry = renewedAt.Add(time.Duration(renewed.Auth.LeaseDuration) * time.Second) + } + leaseAt := time.Now() + lease, e := c.call(ctx, "PUT", "sys/leases/renew", map[string]string{"lease_id": db.LeaseID, "increment": "600"}) + if e == nil && lease.LeaseDuration > 0 { + dbExpiry = leaseAt.Add(time.Duration(lease.LeaseDuration) * time.Second) + } + current, e := c.call(ctx, "GET", "secret/data/"+role, nil) + if e == nil { + var next map[string]string + if json.Unmarshal(current.Data["data"], &next) == nil { + a, _ := json.Marshal(secrets) + b, _ := json.Marshal(next) + if !bytes.Equal(a, b) { + cancel() + return + } + } + } + // Re-authenticate before the renewable token reaches its max TTL. + if time.Until(tokenExpiry) < 45*time.Second { + if expiry, e := login(); e == nil { + tokenExpiry = expiry + } + } + select { + case expiryUpdates <- deadline(): + case <-ctx.Done(): + return + } + } + } + }() + return ctx, cancel, nil +} + +func minTime(values ...time.Time) time.Time { + result := values[0] + for _, value := range values[1:] { + if value.Before(result) { + result = value + } + } + return result +} diff --git a/service/internal/pkg/postgres/after-connect.go b/service/internal/pkg/postgres/after-connect.go new file mode 100644 index 0000000..eafcc63 --- /dev/null +++ b/service/internal/pkg/postgres/after-connect.go @@ -0,0 +1,25 @@ +package postgres + +import ( + "context" + + "github.com/jackc/pgx/v5" + pgxUUID "github.com/vgarvardt/pgx-google-uuid/v5" +) + +type AfterConnect func(ctx context.Context, conn *pgx.Conn) error + +func AddUUIDSupport(_ context.Context, conn *pgx.Conn) error { + pgxUUID.Register(conn.TypeMap()) + return nil +} +func WithEnumTypeSupport(enumName string) AfterConnect { + return func(ctx context.Context, conn *pgx.Conn) error { + dt, err := conn.LoadType(ctx, enumName) + if err != nil { + return err + } + conn.TypeMap().RegisterType(dt) + return nil + } +} diff --git a/service/internal/pkg/postgres/config.go b/service/internal/pkg/postgres/config.go new file mode 100644 index 0000000..b0df40a --- /dev/null +++ b/service/internal/pkg/postgres/config.go @@ -0,0 +1,7 @@ +package postgres + +type MigrationConfig interface { + GetURI() string + GetVersionTableName() string + GetRunMigrations() bool +} diff --git a/service/internal/pkg/postgres/err.go b/service/internal/pkg/postgres/err.go new file mode 100644 index 0000000..4380b24 --- /dev/null +++ b/service/internal/pkg/postgres/err.go @@ -0,0 +1,10 @@ +package postgres + +import ( + "errors" +) + +var ( + errCreatingPool = errors.New("failed to create postgres pool") + errParseConfig = errors.New("failed to parse postgres config") +) diff --git a/service/internal/pkg/postgres/interfaces.go b/service/internal/pkg/postgres/interfaces.go new file mode 100644 index 0000000..6afe65f --- /dev/null +++ b/service/internal/pkg/postgres/interfaces.go @@ -0,0 +1,15 @@ +package postgres + +import ( + "context" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgconn" +) + +type Conn interface { + SendBatch(ctx context.Context, b *pgx.Batch) pgx.BatchResults + Exec(ctx context.Context, sql string, arguments ...any) (commandTag pgconn.CommandTag, err error) + Query(ctx context.Context, sql string, args ...any) (pgx.Rows, error) + QueryRow(ctx context.Context, sql string, args ...any) pgx.Row +} diff --git a/service/internal/pkg/postgres/postgres.go b/service/internal/pkg/postgres/postgres.go new file mode 100644 index 0000000..27fc125 --- /dev/null +++ b/service/internal/pkg/postgres/postgres.go @@ -0,0 +1,88 @@ +package postgres + +import ( + "context" + "log/slog" + "strconv" + "time" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" + "github.com/jackc/pgx/v5/tracelog" + + "gitea.privcloud.tech/mtusi2026/internal/pkg/telemetry" +) + +type ConfigWithLog interface { + GetLogLevel() string +} + +type ConfigWithTimeouts interface { + GetConnectTimeout() time.Duration + GetQueryTimeout() time.Duration +} + +type ConfigWithMaxConn interface { + GetMaxConnections() int32 +} + +func getLevelFromConfig(cfg MigrationConfig) tracelog.LogLevel { + if logCfg, ok := cfg.(ConfigWithLog); ok { + lvl, _ := tracelog.LogLevelFromString(logCfg.GetLogLevel()) + return lvl + } + return tracelog.LogLevelError +} + +func addMaxConnectionsToCfg(cfg any, poolCfg *pgxpool.Config) { + if connCfg, ok := cfg.(ConfigWithMaxConn); ok { + poolCfg.MaxConns = connCfg.GetMaxConnections() + } +} + +func New(cfg MigrationConfig, afterConnect ...AfterConnect) *pgxpool.Pool { + + if len(afterConnect) == 0 { + afterConnect = []AfterConnect{AddUUIDSupport} + } + + poolConfig, err := pgxpool.ParseConfig(cfg.GetURI()) + if err != nil { + slog.Error("failed to parse postgres config", "error", err) + panic(err) + } + + addMaxConnectionsToCfg(cfg, poolConfig) + if timeoutCfg, ok := cfg.(ConfigWithTimeouts); ok { + poolConfig.ConnConfig.ConnectTimeout = timeoutCfg.GetConnectTimeout() + poolConfig.ConnConfig.RuntimeParams["statement_timeout"] = strconv.FormatInt(timeoutCfg.GetQueryTimeout().Milliseconds(), 10) + } + poolConfig.ConnConfig.RuntimeParams["application_name"] = "mtusi-api" + slog.Info("postgres_pool_config", "max_conns", poolConfig.MaxConns, + "connect_timeout", poolConfig.ConnConfig.ConnectTimeout, + "statement_timeout_ms", poolConfig.ConnConfig.RuntimeParams["statement_timeout"]) + + poolConfig.ConnConfig.Tracer = telemetry.DBTracer{} + + poolConfig.AfterConnect = load(afterConnect...) + + pool, err := pgxpool.NewWithConfig(context.Background(), poolConfig) + if err != nil { + slog.Error("failed to create postgres pool", "error", err) + panic(err) + } + + slog.Info("successfully connected to postgres pool") + return pool +} + +func load(afterConnects ...AfterConnect) func(ctx context.Context, conn *pgx.Conn) error { + return func(ctx context.Context, conn *pgx.Conn) error { + for _, afterConnect := range afterConnects { + if err := afterConnect(ctx, conn); err != nil { + return err + } + } + return nil + } +} diff --git a/service/internal/pkg/postgres/txmanager/tx.go b/service/internal/pkg/postgres/txmanager/tx.go new file mode 100644 index 0000000..c4aa2fe --- /dev/null +++ b/service/internal/pkg/postgres/txmanager/tx.go @@ -0,0 +1,59 @@ +package txmanager + +import ( + "context" + "errors" + "fmt" + "log/slog" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" +) + +type TransactionManager struct { + pool *pgxpool.Pool +} + +func New(pool *pgxpool.Pool) *TransactionManager { + return &TransactionManager{ + pool: pool, + } +} + +func (tm *TransactionManager) RunInTransaction(ctx context.Context, opts *pgx.TxOptions, fn func(tx pgx.Tx) error) (err error) { + var tx pgx.Tx + + if opts != nil { + tx, err = tm.pool.BeginTx(ctx, *opts) + } else { + tx, err = tm.pool.Begin(ctx) + } + if err != nil { + return fmt.Errorf("failed to begin transaction: %w", err) + } + + var committed bool + + defer func() { + if p := recover(); p != nil { + err = fmt.Errorf("panic in transaction: %v", p) + } + if !committed { + rollbackErr := tx.Rollback(ctx) + if rollbackErr != nil && !errors.Is(rollbackErr, pgx.ErrTxClosed) { + slog.Error("failed to rollback transaction", "error", rollbackErr) + } + } + }() + + if err = fn(tx); err != nil { + return fmt.Errorf("failed to execute transaction: %w", err) + } + + if err = tx.Commit(ctx); err != nil { + return fmt.Errorf("failed to commit transaction: %w", err) + } + + committed = true + return nil +} diff --git a/service/internal/pkg/ratelimit/redis.go b/service/internal/pkg/ratelimit/redis.go new file mode 100644 index 0000000..f8dd7b6 --- /dev/null +++ b/service/internal/pkg/ratelimit/redis.go @@ -0,0 +1,26 @@ +// Package ratelimit implements atomic, shared fixed-window counters, without storing raw credentials. +package ratelimit + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "github.com/redis/go-redis/v9" + "time" +) + +type Limiter struct{ Client *redis.Client } + +var script = redis.NewScript(`local n=redis.call('INCR',KEYS[1]); if n==1 then redis.call('EXPIRE',KEYS[1],60) end; return {n,redis.call('TTL',KEYS[1])}`) + +func (l *Limiter) Allow(ctx context.Context, scope, value string, limit int64) (bool, time.Duration, error) { + h := sha256.Sum256([]byte(value)) + key := "ratelimit:" + scope + ":" + hex.EncodeToString(h[:]) + ctx, cancel := context.WithTimeout(ctx, 300*time.Millisecond) + defer cancel() + v, err := script.Run(ctx, l.Client, []string{key}).Int64Slice() + if err != nil { + return false, 0, err + } + return v[0] <= limit, time.Duration(max(v[1], 1)) * time.Second, nil +} diff --git a/service/internal/pkg/rest/framework/framework.go b/service/internal/pkg/rest/framework/framework.go new file mode 100644 index 0000000..2b1d13e --- /dev/null +++ b/service/internal/pkg/rest/framework/framework.go @@ -0,0 +1,267 @@ +package framework + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "log/slog" + "mime" + "net/http" + "strconv" + "strings" + + "github.com/go-chi/chi/v5" + v10 "github.com/go-playground/validator/v10" + "github.com/google/uuid" + + "gitea.privcloud.tech/mtusi2026/internal/dto" + "gitea.privcloud.tech/mtusi2026/internal/gen/contracts" + "gitea.privcloud.tech/mtusi2026/internal/pkg/apperr" + "gitea.privcloud.tech/mtusi2026/internal/pkg/auth/jwtprovider" + "gitea.privcloud.tech/mtusi2026/internal/pkg/rest/headers" + "gitea.privcloud.tech/mtusi2026/internal/pkg/validator" +) + +func validate(ctx context.Context, v *validator.ValidatorImpl, value any) error { + err := v.ValidateCtx(ctx, value) + if err == nil { + return nil + } + var fieldErrors v10.ValidationErrors + if errors.As(err, &fieldErrors) { + return apperr.Wrap(err, apperr.CodeValidation, "invalid request fields") + } + return apperr.Wrap(err, apperr.CodeInternal, "validation failed") +} + +func Validate(w http.ResponseWriter, r *http.Request, v *validator.ValidatorImpl, value any) bool { + if err := validate(r.Context(), v, value); err != nil { + JSONError(r.Context(), w, err) + return false + } + return true +} + +func JSONResponse(w http.ResponseWriter, status int, body any) { + w.Header().Set(headers.HeaderContentType, "application/json") + w.WriteHeader(status) + + if status == http.StatusNoContent { + return + } + + if err := json.NewEncoder(w).Encode(body); err != nil { + wrapped := apperr.Wrap(err, apperr.CodeInternal, "failed json encode") + slog.Error("response", "error", wrapped) + } +} + +func JSONError(ctx context.Context, w http.ResponseWriter, err error) { + if err == nil { + return + } + + requestId, ok := ctx.Value(headers.HeaderRequestID).(uuid.UUID) + if !ok { + requestId = uuid.New() + } + w.Header().Set(headers.HeaderRequestID, requestId.String()) + + if errors.Is(err, context.DeadlineExceeded) { + err = apperr.Wrap(err, apperr.CodeServiceUnavailable, "request deadline exceeded") + } + appErr := apperr.As(err) + if appErr == nil { + appErr = apperr.As(apperr.New(apperr.CodeInternal, "internal server error")) + } + if appErr.Code().HTTPStatus() == http.StatusUnauthorized && w.Header().Get("WWW-Authenticate") == "" { + w.Header().Set("WWW-Authenticate", `Bearer realm="messenger"`) + } + + slog.ErrorContext(ctx, "http_error", "error_code", appErr.Code().String(), "status", appErr.Code().HTTPStatus(), "request_id", requestId.String()) + + var code contracts.ErrorErrorCode + switch appErr.Code() { + case apperr.CodeBadRequest, apperr.CodeValidation: + code = contracts.InvalidRequest + case apperr.CodeUnauthorized: + code = contracts.Unauthorized + case apperr.CodeNotFound: + code = contracts.NotFound + case apperr.CodeAlreadyExists, apperr.CodeConflict: + code = contracts.Conflict + case apperr.CodePayloadTooLarge: + code = contracts.PayloadTooLarge + case apperr.CodeTooManyRequests: + code = contracts.RateLimited + case apperr.CodeServiceUnavailable, apperr.CodeUnavailable: + code = contracts.Unavailable + case apperr.CodeUnsupportedMediaType: + code = contracts.UnsupportedMediaType + default: + code = contracts.InternalError + } + JSONResponse(w, appErr.Code().HTTPStatus(), contracts.Error{ + Error: struct { + Code contracts.ErrorErrorCode `json:"code"` + Message string `json:"message"` + RequestId string `json:"request_id"` + }{ + Code: code, + Message: appErr.Message(), + RequestId: requestId.String(), + }, + }) +} + +func Decode(w http.ResponseWriter, r *http.Request, object any) bool { + mediaType, params, err := mime.ParseMediaType(r.Header.Get(headers.HeaderContentType)) + if err != nil || mediaType != "application/json" || + (params["charset"] != "" && !strings.EqualFold(params["charset"], "utf-8")) { + JSONError(r.Context(), w, apperr.New(apperr.CodeUnsupportedMediaType, "expected application/json")) + return false + } + if r.ContentLength > 1<<20 { + JSONError(r.Context(), w, apperr.New(apperr.CodePayloadTooLarge, "JSON body too large")) + return false + } + r.Body = http.MaxBytesReader(w, r.Body, 1<<20) + decoder := json.NewDecoder(r.Body) + decoder.DisallowUnknownFields() + if err := decoder.Decode(object); err != nil { + var tooLarge *http.MaxBytesError + if errors.As(err, &tooLarge) { + JSONError(r.Context(), w, apperr.New(apperr.CodePayloadTooLarge, "JSON body too large")) + } else { + JSONError(r.Context(), w, apperr.New(apperr.CodeBadRequest, "invalid JSON")) + } + return false + } + var trailing any + if err := decoder.Decode(&trailing); err != io.EOF { + JSONError(r.Context(), w, apperr.New(apperr.CodeBadRequest, "expected one JSON object")) + return false + } + return true +} + +func PathParam(r *http.Request, key string) string { + return chi.URLParam(r, key) +} + +func QueryParam(r *http.Request, key string) string { + return r.URL.Query().Get(key) +} + +func HeaderParam(r *http.Request, key string) string { + return r.Header.Get(key) +} + +func ParseIDFromPathParam(ctx context.Context, w http.ResponseWriter, r *http.Request, key string) (uuid.UUID, error) { + rawID := PathParam(r, key) + id, err := uuid.Parse(rawID) + if err != nil { + JSONError(ctx, w, apperr.New(apperr.CodeValidation, fmt.Sprintf("provided invalid %s", key))) + return uuid.Nil, err + } + return id, nil +} + +func ParseIDFromQueryParam(ctx context.Context, w http.ResponseWriter, r *http.Request, key string) (uuid.UUID, error) { + rawID := QueryParam(r, key) + id, err := uuid.Parse(rawID) + if err != nil { + JSONError(ctx, w, apperr.New(apperr.CodeValidation, fmt.Sprintf("provided invalid %s", key))) + return uuid.Nil, err + } + return id, nil +} + +func ParseIDFromHeaderParam(ctx context.Context, w http.ResponseWriter, r *http.Request, key string) (uuid.UUID, error) { + rawID := HeaderParam(r, key) + id, err := uuid.Parse(rawID) + if err != nil { + JSONError(ctx, w, apperr.New(apperr.CodeUnauthorized, "missing or invalid user ID")) + return uuid.Nil, err + } + return id, nil +} + +func Header(r *http.Request, header string) (string, bool) { + return r.Header.Get(header), r.Header.Get(header) != "" +} + +func Token(r *http.Request) (string, bool) { + foundHeader := r.Header.Get(headers.HeaderAuthorization) + if foundHeader == "" { + return "", false + } + + token, found := strings.CutPrefix(foundHeader, "Bearer ") + if !found { + return "", false + } + + return token, true +} + +type sessionContextKey struct{} + +func WithVerifiedSession(r *http.Request, data *dto.UserDataInToken) *http.Request { + return r.WithContext(context.WithValue(r.Context(), sessionContextKey{}, *data)) +} + +func ParseToken(r *http.Request, provider jwtprovider.Provider) (*dto.UserDataInToken, error) { + if data, ok := r.Context().Value(sessionContextKey{}).(dto.UserDataInToken); ok { + return &data, nil + } + foundHeader := r.Header.Get(headers.HeaderAuthorization) + if foundHeader == "" { + return nil, apperr.New(apperr.CodeUnauthorized, "missing authorization header") + } + + token, found := strings.CutPrefix(foundHeader, "Bearer ") + if !found { + return nil, apperr.New(apperr.CodeUnauthorized, "invalid authorization header") + } + + userData, err := provider.ParseVerified(&dto.Token{Payload: token}) + if err != nil { + return nil, apperr.Wrap(err, apperr.CodeUnauthorized, "invalid token") + } + + return userData, nil +} + +func PageParams(w http.ResponseWriter, r *http.Request) (*int, *uuid.UUID, bool) { + query := r.URL.Query() + var limit *int + if values, exists := query["limit"]; exists { + if len(values) != 1 { + JSONError(r.Context(), w, apperr.New(apperr.CodeBadRequest, "invalid limit")) + return nil, nil, false + } + value, err := strconv.Atoi(values[0]) + if err != nil || value < 1 || value > 100 { + JSONError(r.Context(), w, apperr.New(apperr.CodeBadRequest, "invalid limit")) + return nil, nil, false + } + limit = &value + } + var afterID *uuid.UUID + if values, exists := query["cursor"]; exists { + if len(values) != 1 { + JSONError(r.Context(), w, apperr.New(apperr.CodeBadRequest, "invalid cursor")) + return nil, nil, false + } + id, err := uuid.Parse(values[0]) + if err != nil || id == uuid.Nil { + JSONError(r.Context(), w, apperr.New(apperr.CodeBadRequest, "invalid cursor")) + return nil, nil, false + } + afterID = &id + } + return limit, afterID, true +} diff --git a/service/internal/pkg/rest/headers/headers.go b/service/internal/pkg/rest/headers/headers.go new file mode 100644 index 0000000..c92a8de --- /dev/null +++ b/service/internal/pkg/rest/headers/headers.go @@ -0,0 +1,8 @@ +package headers + +const ( + HeaderContentType = "Content-Type" + HeaderRequestID = "X-Request-Id" + HeaderUserID = "X-User-Id" + HeaderAuthorization = "Authorization" +) diff --git a/service/internal/pkg/rest/middleware/auth.go b/service/internal/pkg/rest/middleware/auth.go new file mode 100644 index 0000000..a39c053 --- /dev/null +++ b/service/internal/pkg/rest/middleware/auth.go @@ -0,0 +1,75 @@ +package middleware + +import ( + "context" + "net/http" + + "gitea.privcloud.tech/mtusi2026/internal/dto" + "gitea.privcloud.tech/mtusi2026/internal/pkg/apperr" + "gitea.privcloud.tech/mtusi2026/internal/pkg/rest/framework" + "gitea.privcloud.tech/mtusi2026/internal/pkg/types/role" +) + +var _ AuthMiddleware = (*authMiddleware)(nil) + +type AuthMiddleware interface { + WithPrivateAccess(next http.Handler) http.Handler + WithAdminAccess(next http.Handler) http.Handler +} + +type SessionVerifier interface { + VerifySession(ctx context.Context, token dto.Token) (*dto.UserDataInToken, error) +} + +type authMiddleware struct { + verifier SessionVerifier +} + +func NewAuthMiddleware(verifier SessionVerifier) AuthMiddleware { + return &authMiddleware{verifier: verifier} +} + +func (a *authMiddleware) WithPrivateAccess(next http.Handler) http.Handler { + return a.privateAccessCheckCallback(next) +} + +func (a *authMiddleware) WithAdminAccess(next http.Handler) http.Handler { + return a.operatorAccessCheckCallback(role.Admin, next) +} + +func (a *authMiddleware) privateAccessCheckCallback(next http.Handler) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + data, err := a.verifySession(r) + if err != nil { + framework.JSONError(r.Context(), w, err) + return + } + + next.ServeHTTP(w, framework.WithVerifiedSession(r, data)) + } +} + +func (a *authMiddleware) operatorAccessCheckCallback(expectedRole role.Role, next http.Handler) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + u, err := a.verifySession(r) + if err != nil { + framework.JSONError(r.Context(), w, err) + return + } + + if u.Role != role.Admin && u.Role != expectedRole { + framework.JSONError(r.Context(), w, apperr.New(apperr.CodeForbidden, "insufficient permissions")) + return + } + + next.ServeHTTP(w, framework.WithVerifiedSession(r, u)) + } +} + +func (a *authMiddleware) verifySession(r *http.Request) (*dto.UserDataInToken, error) { + token, found := framework.Token(r) + if !found || token == "" { + return nil, apperr.New(apperr.CodeUnauthorized, "missing or invalid bearer token") + } + return a.verifier.VerifySession(r.Context(), dto.Token{Payload: token}) +} diff --git a/service/internal/pkg/rest/middleware/auth_test.go b/service/internal/pkg/rest/middleware/auth_test.go new file mode 100644 index 0000000..b62e5d8 --- /dev/null +++ b/service/internal/pkg/rest/middleware/auth_test.go @@ -0,0 +1,73 @@ +package middleware + +import ( + "context" + "errors" + "net/http" + "net/http/httptest" + "testing" + + "gitea.privcloud.tech/mtusi2026/internal/dto" + "gitea.privcloud.tech/mtusi2026/internal/pkg/apperr" + "gitea.privcloud.tech/mtusi2026/internal/pkg/rest/framework" + "github.com/google/uuid" +) + +type sessionVerifier struct { + calls int + data dto.UserDataInToken + err error +} + +func (v *sessionVerifier) VerifySession(context.Context, dto.Token) (*dto.UserDataInToken, error) { + v.calls++ + return &v.data, v.err +} + +func TestVerifiedSessionIsRequestScoped(t *testing.T) { + verifier := &sessionVerifier{data: dto.UserDataInToken{UserID: uuid.New()}} + middleware := NewAuthMiddleware(verifier) + handled := 0 + handler := middleware.WithPrivateAccess(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // A nil provider is intentional: the handler must use the verified request identity. + data, err := framework.ParseToken(r, nil) + if err != nil { + t.Fatal(err) + } + if data.UserID != verifier.data.UserID { + t.Fatal("verified identity changed") + } + handled++ + w.WriteHeader(http.StatusNoContent) + })) + for range 2 { + request := httptest.NewRequest("GET", "/private", nil) + request.Header.Set("Authorization", "Bearer synthetic-token") + recorder := httptest.NewRecorder() + handler.ServeHTTP(recorder, request) + if recorder.Code != 204 { + t.Fatal(recorder.Code) + } + } + if verifier.calls != 2 || handled != 2 { + t.Fatal("session verification must run for every request") + } + verifier.err = apperr.New(apperr.CodeUnauthorized, "revoked session") + request := httptest.NewRequest("GET", "/private", nil) + request.Header.Set("Authorization", "Bearer synthetic-token") + recorder := httptest.NewRecorder() + handler.ServeHTTP(recorder, request) + if recorder.Code != 401 || handled != 2 { + t.Fatal("revocation was bypassed by cached request identity") + } +} + +func TestMissingTokenDoesNotCallSessionStore(t *testing.T) { + verifier := &sessionVerifier{err: errors.New("must not be called")} + handler := NewAuthMiddleware(verifier).WithPrivateAccess(http.HandlerFunc(func(http.ResponseWriter, *http.Request) { t.Fatal("handler called without a token") })) + recorder := httptest.NewRecorder() + handler.ServeHTTP(recorder, httptest.NewRequest("GET", "/private", nil)) + if recorder.Code != 401 || verifier.calls != 0 { + t.Fatal("missing token was accepted") + } +} diff --git a/service/internal/pkg/rest/middleware/instance.go b/service/internal/pkg/rest/middleware/instance.go new file mode 100644 index 0000000..6447705 --- /dev/null +++ b/service/internal/pkg/rest/middleware/instance.go @@ -0,0 +1,12 @@ +package middleware + +import "net/http" + +func Instance(id string) Middleware { + return func(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("X-Instance-Id", id) + next.ServeHTTP(w, r) + }) + } +} diff --git a/service/internal/pkg/rest/middleware/logging.go b/service/internal/pkg/rest/middleware/logging.go new file mode 100644 index 0000000..8ffda7a --- /dev/null +++ b/service/internal/pkg/rest/middleware/logging.go @@ -0,0 +1,34 @@ +package middleware + +import ( + "log/slog" + "net/http" + "time" + + "github.com/go-chi/chi/v5/middleware" + + "gitea.privcloud.tech/mtusi2026/internal/pkg/rest/headers" +) + +func LoggingMiddleware() Middleware { + return func(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + started := time.Now() + wrapped := middleware.NewWrapResponseWriter(w, r.ProtoMajor) + next.ServeHTTP(wrapped, r) + + status := wrapped.Status() + if status == 0 { + status = http.StatusOK + } + slog.InfoContext(r.Context(), "http request", + "method", r.Method, + "path", r.URL.Path, + "status", status, + "duration", time.Since(started), + "bytes", wrapped.BytesWritten(), + "request_id", w.Header().Get(headers.HeaderRequestID), + ) + }) + } +} diff --git a/service/internal/pkg/rest/middleware/middleware.go b/service/internal/pkg/rest/middleware/middleware.go new file mode 100644 index 0000000..f54cea9 --- /dev/null +++ b/service/internal/pkg/rest/middleware/middleware.go @@ -0,0 +1,5 @@ +package middleware + +import "net/http" + +type Middleware = func(http.Handler) http.Handler diff --git a/service/internal/pkg/rest/middleware/request-id.go b/service/internal/pkg/rest/middleware/request-id.go new file mode 100644 index 0000000..eed3549 --- /dev/null +++ b/service/internal/pkg/rest/middleware/request-id.go @@ -0,0 +1,22 @@ +package middleware + +import ( + "context" + "net/http" + + "github.com/google/uuid" + + "gitea.privcloud.tech/mtusi2026/internal/pkg/rest/headers" +) + +func RequestIDMiddleware() Middleware { + return func(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + requestID := uuid.New() + w.Header().Set(headers.HeaderRequestID, requestID.String()) + ctx = context.WithValue(ctx, headers.HeaderRequestID, requestID) + next.ServeHTTP(w, r.WithContext(ctx)) + }) + } +} diff --git a/service/internal/pkg/rest/middleware/timeout.go b/service/internal/pkg/rest/middleware/timeout.go new file mode 100644 index 0000000..7542147 --- /dev/null +++ b/service/internal/pkg/rest/middleware/timeout.go @@ -0,0 +1,23 @@ +package middleware + +import ( + "context" + "net/http" + "strings" + "time" +) + +// RequestTimeout bounds database work and waiting for a pooled connection. +func RequestTimeout(timeout time.Duration, attachmentTimeout ...time.Duration) Middleware { + return func(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestTimeout := timeout + if len(attachmentTimeout) > 0 && strings.Contains(r.URL.Path, "/attachments") { + requestTimeout = attachmentTimeout[0] + } + ctx, cancel := context.WithTimeout(r.Context(), requestTimeout) + defer cancel() + next.ServeHTTP(w, r.WithContext(ctx)) + }) + } +} diff --git a/service/internal/pkg/rest/router/router.go b/service/internal/pkg/rest/router/router.go new file mode 100644 index 0000000..6c67300 --- /dev/null +++ b/service/internal/pkg/rest/router/router.go @@ -0,0 +1,135 @@ +package router + +import ( + "gitea.privcloud.tech/mtusi2026/internal/pkg/telemetry" + "net/http" + + "github.com/go-chi/chi/v5" + + "gitea.privcloud.tech/mtusi2026/internal/pkg/rest/middleware" +) + +type Router interface { + Private(method, pattern string, handler http.HandlerFunc, mw ...func(http.Handler) http.Handler) + Public(method, pattern string, handler http.HandlerFunc, mw ...func(http.Handler) http.Handler) + Admin(method, pattern string, handler http.HandlerFunc, mw ...func(http.Handler) http.Handler) + ServeHTTP(w http.ResponseWriter, r *http.Request) + With(mw ...func(http.Handler) http.Handler) Router + Use(mw ...func(http.Handler) http.Handler) Router + Handle(pattern string, handler http.Handler) + Get(pattern string, handler http.HandlerFunc) + Head(pattern string, handler http.HandlerFunc) + Options(pattern string, handler http.HandlerFunc) + Post(pattern string, handler http.HandlerFunc) + Put(pattern string, handler http.HandlerFunc) + Patch(pattern string, handler http.HandlerFunc) + Delete(pattern string, handler http.HandlerFunc) + Connect(pattern string, handler http.HandlerFunc) + Trace(pattern string, handler http.HandlerFunc) +} + +type router struct { + r chi.Router + jwtMw middleware.AuthMiddleware +} + +func New(jwtMw middleware.AuthMiddleware, mw ...middleware.Middleware) Router { + r := chi.NewRouter() + r.Use(mw...) + return &router{ + r: r, + jwtMw: jwtMw, + } +} + +func (c *router) Group(pattern string) Router { + sub := &router{ + r: chi.NewRouter(), + jwtMw: c.jwtMw, + } + c.r.Mount(pattern, sub.r) + return sub +} + +func (c *router) Public(method, pattern string, handler http.HandlerFunc, mw ...func(http.Handler) http.Handler) { + telemetry.RegisterRoute(method, pattern) + next := c.applyMiddlewares(handler, mw...) + c.r.MethodFunc(method, pattern, next.ServeHTTP) +} + +func (c *router) Private(method, pattern string, handler http.HandlerFunc, mw ...func(http.Handler) http.Handler) { + telemetry.RegisterRoute(method, pattern) + next := c.applyMiddlewares(handler, mw...) + c.r.MethodFunc(method, pattern, (c.jwtMw.WithPrivateAccess(next)).ServeHTTP) +} + +func (c *router) Admin(method, pattern string, handler http.HandlerFunc, mw ...func(http.Handler) http.Handler) { + telemetry.RegisterRoute(method, pattern) + next := c.applyMiddlewares(handler, mw...) + c.r.MethodFunc(method, pattern, (c.jwtMw.WithAdminAccess(next)).ServeHTTP) +} + +func (c *router) ServeHTTP(w http.ResponseWriter, r *http.Request) { + c.r.ServeHTTP(w, r) +} + +func (c *router) With(mw ...func(http.Handler) http.Handler) Router { + sub := &router{ + r: c.r.With(mw...), + jwtMw: c.jwtMw, + } + return sub +} + +func (c *router) Use(mw ...func(http.Handler) http.Handler) Router { + c.r.Use(mw...) + return c +} + +func (c *router) Handle(pattern string, handler http.Handler) { + c.r.Handle(pattern, handler) +} + +func (c *router) Get(pattern string, handler http.HandlerFunc) { + c.r.Get(pattern, handler) +} + +func (c *router) Head(pattern string, handler http.HandlerFunc) { + c.r.Head(pattern, handler) +} + +func (c *router) Options(pattern string, handler http.HandlerFunc) { + c.r.Options(pattern, handler) +} + +func (c *router) Post(pattern string, handler http.HandlerFunc) { + c.r.Post(pattern, handler) +} + +func (c *router) Put(pattern string, handler http.HandlerFunc) { + c.r.Put(pattern, handler) +} + +func (c *router) Patch(pattern string, handler http.HandlerFunc) { + c.r.Patch(pattern, handler) +} + +func (c *router) Delete(pattern string, handler http.HandlerFunc) { + c.r.Delete(pattern, handler) +} + +func (c *router) Connect(pattern string, handler http.HandlerFunc) { + c.r.Connect(pattern, handler) +} + +func (c *router) Trace(pattern string, handler http.HandlerFunc) { + c.r.Trace(pattern, handler) +} + +func (c *router) applyMiddlewares(handler http.HandlerFunc, mw ...func(http.Handler) http.Handler) http.Handler { + next := http.Handler(handler) + for _, mwFunc := range mw { + next = mwFunc(next) + } + return next +} diff --git a/service/internal/pkg/telemetry/http.go b/service/internal/pkg/telemetry/http.go new file mode 100644 index 0000000..d0399be --- /dev/null +++ b/service/internal/pkg/telemetry/http.go @@ -0,0 +1,56 @@ +package telemetry + +import ( + "context" + "github.com/go-chi/chi/v5" + mw "github.com/go-chi/chi/v5/middleware" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/codes" + "go.opentelemetry.io/otel/trace" + "log/slog" + "net/http" + "strconv" + "time" +) + +func HTTP(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + ctx, sp := Start(Extract(r.Context(), r.Header.Get("traceparent"), r.Header.Get("tracestate")), "HTTP", trace.SpanKindServer) + start := time.Now() + ww := mw.NewWrapResponseWriter(w, r.ProtoMajor) + defer func() { + route := chi.RouteContext(ctx).RoutePattern() + if route == "" { + route = "unmatched" + } + method := r.Method + switch method { + case "GET", "POST", "DELETE", "PUT", "PATCH", "HEAD", "OPTIONS": + default: + method = "OTHER" + } + status := ww.Status() + if status == 0 { + status = 200 + } + sp.SetName(method + " " + route) + sp.SetAttributes(attribute.String("http.route", route), attribute.String("http.request.method", method), attribute.Int("http.response.status_code", status)) + if status >= 500 { + sp.SetStatus(codes.Error, "server_error") + } + a := []attribute.KeyValue{attribute.String("method", method), attribute.String("route", route), attribute.String("status_class", strconv.Itoa(status/100)+"xx")} + Count(ctx, "messenger.http.requests", 1, a...) + Duration(ctx, "messenger.http.duration", time.Since(start), a[:2]...) + slog.InfoContext(ctx, "http_request", "method", method, "route", route, "status", status, "request_id", w.Header().Get("X-Request-Id")) + sp.End() + }() + next.ServeHTTP(ww, r.WithContext(ctx)) + }) +} + +// RegisterRoute publishes zero counters before any outcome, preserving the first spike. +func RegisterRoute(method, route string) { + for _, class := range []string{"1xx", "2xx", "3xx", "4xx", "5xx"} { + Count(context.Background(), "messenger.http.requests", 0, attribute.String("method", method), attribute.String("route", route), attribute.String("status_class", class)) + } +} diff --git a/service/internal/pkg/telemetry/observation.go b/service/internal/pkg/telemetry/observation.go new file mode 100644 index 0000000..22063f0 --- /dev/null +++ b/service/internal/pkg/telemetry/observation.go @@ -0,0 +1,57 @@ +package telemetry + +import ( + "context" + + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/metric" + "sync" + "sync/atomic" + "time" +) + +var QueueDepth atomic.Int64 +var observationMu sync.RWMutex + +type observationSnapshot struct { + Queued, Outbox, Good, Bad int64 + Locked int64 + Oldest float64 +} + +var snapshot observationSnapshot +var observed time.Time + +func PublishObservation(queued, outbox, good, bad int64, oldest float64, locked int64) { + s := observationSnapshot{Queued: queued, Outbox: outbox, Good: good, Bad: bad, Oldest: oldest, Locked: locked} + observationMu.Lock() + snapshot = s + observed = time.Now() + observationMu.Unlock() +} +func RegisterObservation() { + locks, _ := meter.Int64ObservableGauge("messenger.db.locks.waiting") + pending, _ := meter.Int64ObservableGauge("messenger.jobs.pending") + outbox, _ := meter.Int64ObservableGauge("messenger.outbox.pending") + queue, _ := meter.Int64ObservableGauge("messenger.queue.depth") + age, _ := meter.Float64ObservableGauge("messenger.queue.oldest.age", metric.WithUnit("s")) + fresh, _ := meter.Float64ObservableGauge("messenger.observation.timestamp") + slo, _ := meter.Int64ObservableCounter("messenger.ready.slo") + _, _ = meter.RegisterCallback(func(ctx context.Context, o metric.Observer) error { + observationMu.RLock() + s, t := snapshot, observed + observationMu.RUnlock() + if t.IsZero() { + return nil + } + o.ObserveInt64(locks, s.Locked) + o.ObserveInt64(pending, s.Queued) + o.ObserveInt64(outbox, s.Outbox) + o.ObserveInt64(queue, QueueDepth.Load(), metric.WithAttributes(attribute.String("queue", "attachments.v1"))) + o.ObserveFloat64(age, s.Oldest, metric.WithAttributes(attribute.String("queue", "attachments.v1"))) + o.ObserveFloat64(fresh, float64(t.Unix())) + o.ObserveInt64(slo, s.Good, metric.WithAttributes(attribute.String("outcome", "good"))) + o.ObserveInt64(slo, s.Bad, metric.WithAttributes(attribute.String("outcome", "bad"))) + return nil + }, locks, pending, outbox, queue, age, fresh, slo) +} diff --git a/service/internal/pkg/telemetry/process.go b/service/internal/pkg/telemetry/process.go new file mode 100644 index 0000000..8f8b495 --- /dev/null +++ b/service/internal/pkg/telemetry/process.go @@ -0,0 +1,76 @@ +package telemetry + +import ( + "context" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" + "github.com/shirou/gopsutil/v4/process" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/metric" + "go.opentelemetry.io/otel/trace" + "os" + "strings" + "sync/atomic" + "time" +) + +var waiting atomic.Int64 + +func ObservePool(pool *pgxpool.Pool, capacity int) { + names := []string{"messenger.db.pool.used", "messenger.db.pool.capacity", "messenger.db.pool.waiting", "messenger.worker.busy", "messenger.worker.capacity", "messenger.process.memory", "messenger.process.cpu", "messenger.telemetry.export_errors"} + instruments := make([]metric.Float64ObservableGauge, len(names)) + for i, n := range names { + instruments[i], _ = meter.Float64ObservableGauge(n) + } + p, _ := process.NewProcess(int32(os.Getpid())) + obs := make([]metric.Observable, len(instruments)) + for i, g := range instruments { + obs[i] = g + } + _, _ = meter.RegisterCallback(func(ctx context.Context, o metric.Observer) error { + st := pool.Stat() + v := []float64{float64(st.AcquiredConns()), float64(st.MaxConns()), float64(waiting.Load()), float64(Busy.Load()), float64(capacity), 0, 0, float64(ExportErrors.Load())} + if p != nil { + if m, e := p.MemoryInfoWithContext(ctx); e == nil { + v[5] = float64(m.RSS) + } + if t, e := p.TimesWithContext(ctx); e == nil { + v[6] = t.User + t.System + } + } + for i, g := range instruments { + o.ObserveFloat64(g, v[i]) + } + return nil + }, obs...) +} + +type DBTracer struct{} +type acquireKey struct{} + +func (DBTracer) TraceQueryStart(ctx context.Context, _ *pgx.Conn, d pgx.TraceQueryStartData) context.Context { + op := "OTHER" + f := strings.Fields(d.SQL) + if len(f) > 0 { + switch strings.ToUpper(f[0]) { + case "SELECT", "INSERT", "UPDATE", "DELETE", "BEGIN", "COMMIT", "ROLLBACK": + op = strings.ToUpper(f[0]) + } + } + ctx, sp := Start(ctx, "postgresql "+op, trace.SpanKindClient) + sp.SetAttributes(attribute.String("db.system.name", "postgresql"), attribute.String("db.operation.name", op)) + return ctx +} +func (DBTracer) TraceQueryEnd(ctx context.Context, _ *pgx.Conn, d pgx.TraceQueryEndData) { + End(trace.SpanFromContext(ctx), d.Err) +} +func (DBTracer) TraceAcquireStart(ctx context.Context, _ *pgxpool.Pool, _ pgxpool.TraceAcquireStartData) context.Context { + waiting.Add(1) + return context.WithValue(ctx, acquireKey{}, time.Now()) +} +func (DBTracer) TraceAcquireEnd(ctx context.Context, _ *pgxpool.Pool, d pgxpool.TraceAcquireEndData) { + waiting.Add(-1) + if t, ok := ctx.Value(acquireKey{}).(time.Time); ok { + Duration(ctx, "messenger.db.acquire.duration", time.Since(t)) + } +} diff --git a/service/internal/pkg/telemetry/telemetry.go b/service/internal/pkg/telemetry/telemetry.go new file mode 100644 index 0000000..4e6814a --- /dev/null +++ b/service/internal/pkg/telemetry/telemetry.go @@ -0,0 +1,150 @@ +// Package telemetry owns signal export and safe, low-cardinality instrumentation. +package telemetry + +import ( + "context" + "crypto/tls" + "fmt" + "gitea.privcloud.tech/mtusi2026/internal/pkg/transportsecurity" + "log/slog" + "os" + "strings" + "sync/atomic" + "time" + + "github.com/google/uuid" + "go.opentelemetry.io/contrib/bridges/otelslog" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/codes" + "go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp" + "go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp" + "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp" + "go.opentelemetry.io/otel/metric" + "go.opentelemetry.io/otel/propagation" + sdklog "go.opentelemetry.io/otel/sdk/log" + sdkmetric "go.opentelemetry.io/otel/sdk/metric" + "go.opentelemetry.io/otel/sdk/resource" + sdktrace "go.opentelemetry.io/otel/sdk/trace" + "go.opentelemetry.io/otel/trace" +) + +var InstanceID = uuid.NewString() +var Busy atomic.Int64 +var meter = otel.Meter("messenger") +var tracer = otel.Tracer("messenger") + +// A fanout retains local JSON and exports the same structured event over OTLP. +type handler struct{ local, remote slog.Handler } + +func (h handler) Enabled(c context.Context, l slog.Level) bool { return h.local.Enabled(c, l) } +func (h handler) Handle(c context.Context, r slog.Record) error { + // Errors can contain SQL values, credentials or invalid request content. + safe := slog.NewRecord(r.Time, r.Level, r.Message, r.PC) + r.Attrs(func(a slog.Attr) bool { + key := strings.ToLower(a.Key) + switch key { + case "authorization", "password", "token", "session_token", "body", "filename", "url": + return true + case "error": + a = slog.String("error.type", "operation_failed") + } + safe.AddAttrs(a) + return true + }) + r = safe + sc := trace.SpanContextFromContext(c) + r.AddAttrs(slog.String("event", r.Message)) + if sc.IsValid() { + r.AddAttrs(slog.String("trace_id", sc.TraceID().String()), slog.String("span_id", sc.SpanID().String())) + } + _ = h.remote.Handle(c, r) + return h.local.Handle(c, r) +} +func (h handler) WithAttrs(a []slog.Attr) slog.Handler { + return handler{h.local.WithAttrs(a), h.remote.WithAttrs(a)} +} +func (h handler) WithGroup(g string) slog.Handler { + return handler{h.local.WithGroup(g), h.remote.WithGroup(g)} +} + +func Init(ctx context.Context, name string) (func(), error) { + otel.SetTextMapPropagator(propagation.TraceContext{}) + if os.Getenv("OTEL_SDK_DISABLED") == "true" { + return func() {}, nil + } + if transportsecurity.Enabled() && !strings.HasPrefix(os.Getenv("OTEL_EXPORTER_OTLP_ENDPOINT"), "https://") { + return nil, fmt.Errorf("verified HTTPS OTLP endpoint required") + } + res := resource.NewWithAttributes("", attribute.String("service.name", name), attribute.String("service.version", "lab7"), attribute.String("service.instance.id", InstanceID), attribute.String("deployment.environment.name", "lab")) + var tlsConfig *tls.Config + if transportsecurity.Enabled() { + var err error + tlsConfig, err = transportsecurity.Client() + if err != nil { + return nil, err + } + } + te, e := otlptracehttp.New(ctx, otlptracehttp.WithTimeout(time.Second), otlptracehttp.WithTLSClientConfig(tlsConfig)) + if e != nil { + return nil, e + } + tp := sdktrace.NewTracerProvider(sdktrace.WithResource(res), sdktrace.WithSampler(sdktrace.ParentBased(sdktrace.AlwaysSample())), sdktrace.WithBatcher(te, sdktrace.WithMaxQueueSize(1024), sdktrace.WithMaxExportBatchSize(128), sdktrace.WithBatchTimeout(time.Second), sdktrace.WithExportTimeout(time.Second))) + me, e := otlpmetrichttp.New(ctx, otlpmetrichttp.WithTimeout(time.Second), otlpmetrichttp.WithTLSClientConfig(tlsConfig)) + if e != nil { + return nil, e + } + mp := sdkmetric.NewMeterProvider(sdkmetric.WithResource(res), sdkmetric.WithReader(sdkmetric.NewPeriodicReader(me, sdkmetric.WithInterval(5*time.Second), sdkmetric.WithTimeout(time.Second)))) + le, e := otlploghttp.New(ctx, otlploghttp.WithTimeout(time.Second), otlploghttp.WithTLSClientConfig(tlsConfig)) + if e != nil { + return nil, e + } + lp := sdklog.NewLoggerProvider(sdklog.WithResource(res), sdklog.WithProcessor(sdklog.NewBatchProcessor(le, sdklog.WithMaxQueueSize(1024), sdklog.WithExportMaxBatchSize(128), sdklog.WithExportInterval(time.Second), sdklog.WithExportTimeout(time.Second)))) + otel.SetTracerProvider(tp) + otel.SetMeterProvider(mp) + if name == "messenger-worker" { + for _, outcome := range []string{"ready", "retry", "failed", "duplicate"} { + Count(ctx, "messenger.jobs", 0, attribute.String("job_type", "attachment.process.v1"), attribute.String("outcome", outcome)) + } + } + local := slog.NewJSONHandler(os.Stdout, &slog.HandlerOptions{Level: slog.LevelInfo}).WithAttrs([]slog.Attr{slog.String("service.name", name), slog.String("service.instance.id", InstanceID)}) + slog.SetDefault(slog.New(handler{local, otelslog.NewHandler("messenger", otelslog.WithLoggerProvider(lp))})) + // Export failures are summarized safely and never include payloads or endpoints. + otel.SetErrorHandler(otel.ErrorHandlerFunc(func(error) { ExportErrors.Add(1) })) + return func() { + c, cancel := context.WithTimeout(context.Background(), 4*time.Second) + defer cancel() + _ = mp.Shutdown(c) + _ = lp.Shutdown(c) + _ = tp.Shutdown(c) + }, nil +} + +var ExportErrors atomic.Int64 + +func Start(ctx context.Context, name string, kind trace.SpanKind) (context.Context, trace.Span) { + return tracer.Start(ctx, name, trace.WithSpanKind(kind)) +} +func End(span trace.Span, err error) { + if err != nil { + span.SetStatus(codes.Error, "operation_failed") + span.SetAttributes(attribute.String("error.type", "dependency_or_processing")) + } + span.End() +} +func Inject(ctx context.Context) (string, string) { + c := propagation.MapCarrier{} + otel.GetTextMapPropagator().Inject(ctx, c) + return c.Get("traceparent"), c.Get("tracestate") +} +func Extract(ctx context.Context, p, s string) context.Context { + return otel.GetTextMapPropagator().Extract(ctx, propagation.MapCarrier{"traceparent": p, "tracestate": s}) +} +func Count(ctx context.Context, name string, n int64, a ...attribute.KeyValue) { + c, _ := meter.Int64Counter(name) + c.Add(ctx, n, metric.WithAttributes(a...)) +} +func Duration(ctx context.Context, name string, d time.Duration, a ...attribute.KeyValue) { + h, _ := meter.Float64Histogram(name, metric.WithUnit("s"), metric.WithExplicitBucketBoundaries(.005, .01, .025, .05, .1, .25, .5, 1, 2, 5, 10, 20, 30, 60, 120, 300)) + h.Record(ctx, d.Seconds(), metric.WithAttributes(a...)) +} diff --git a/service/internal/pkg/transportsecurity/tls.go b/service/internal/pkg/transportsecurity/tls.go new file mode 100644 index 0000000..3b5fd67 --- /dev/null +++ b/service/internal/pkg/transportsecurity/tls.go @@ -0,0 +1,87 @@ +// Package transportsecurity centralizes verified TLS. It never enables insecure verification. +package transportsecurity + +import ( + "crypto/tls" + "crypto/x509" + "encoding/pem" + "errors" + "os" + "path/filepath" + "strings" + "time" +) + +func Directory() string { + if p := os.Getenv("TLS_IDENTITY_DIR"); p != "" { + return p + } + return "/run/identity" +} +func roots() (*x509.CertPool, error) { + b, err := os.ReadFile(filepath.Join(Directory(), "ca.crt")) + if err != nil { + return nil, err + } + p := x509.NewCertPool() + if !p.AppendCertsFromPEM(b) { + return nil, errors.New("invalid TLS CA") + } + return p, nil +} +func Client() (*tls.Config, error) { + roots, err := roots() + if err != nil { + return nil, err + } + cert, err := tls.LoadX509KeyPair(filepath.Join(Directory(), "tls.crt"), filepath.Join(Directory(), "tls.key")) + if err != nil { + return nil, err + } + return &tls.Config{MinVersion: tls.VersionTLS12, RootCAs: roots, Certificates: []tls.Certificate{cert}}, nil +} +func Server(allowed ...string) (*tls.Config, error) { + c, err := Client() + if err != nil { + return nil, err + } + c.ClientCAs = c.RootCAs + c.ClientAuth = tls.RequireAndVerifyClientCert + c.SessionTicketsDisabled = true + c.VerifyConnection = func(s tls.ConnectionState) error { + if len(s.VerifiedChains) == 0 { + return errors.New("unverified client") + } + leaf := s.PeerCertificates[0] + b, e := os.ReadFile(filepath.Join(Directory(), "ca.crl")) + if e != nil { + return errors.New("revocation list unavailable") + } + block, _ := pem.Decode(b) + if block == nil { + return errors.New("invalid revocation list") + } + crl, e := x509.ParseRevocationList(block.Bytes) + if e != nil || len(s.VerifiedChains[0]) < 2 { + return errors.New("invalid revocation list") + } + if crl.CheckSignatureFrom(s.VerifiedChains[0][1]) != nil || time.Now().After(crl.NextUpdate) { + return errors.New("untrusted or stale revocation list") + } + for _, entry := range crl.RevokedCertificateEntries { + if entry.SerialNumber.Cmp(leaf.SerialNumber) == 0 { + return errors.New("revoked client") + } + } + for _, san := range leaf.DNSNames { + for _, a := range allowed { + if san == a { + return nil + } + } + } + return errors.New("client identity denied") + } + return c, nil +} +func Enabled() bool { return strings.EqualFold(os.Getenv("TLS_REQUIRED"), "true") } diff --git a/service/internal/pkg/types/authprovider/provider.go b/service/internal/pkg/types/authprovider/provider.go new file mode 100644 index 0000000..b05721c --- /dev/null +++ b/service/internal/pkg/types/authprovider/provider.go @@ -0,0 +1,82 @@ +package authprovider + +import ( + "encoding/json" + "strings" + + "gitea.privcloud.tech/mtusi2026/internal/pkg/types" +) + +type Type string + +const ( + TypeUndefined Type = "undefined" + TypePassword Type = "password" + TypeEmail Type = "email" + TypePhone Type = "phone" + TypeGoogle Type = "google" + TypeTelegram Type = "telegram" +) + +var allowedTypes = map[Type]struct{}{ + TypePassword: {}, + TypeEmail: {}, + TypePhone: {}, + TypeGoogle: {}, + TypeTelegram: {}, +} + +func ParseType(value string) Type { + t := Type(strings.ToLower(value)) + if !t.IsValid() { + return TypeUndefined + } + return t +} + +func ParseAndValidateType(value string) (Type, error) { + parsed := ParseType(value) + if _, ok := allowedTypes[parsed]; ok { + return parsed, nil + } + return TypeUndefined, types.ErrUnsupportedOTPStatus +} + +func (t Type) String() string { + return string(t) +} + +func (t Type) Validate() error { + if _, ok := allowedTypes[t]; ok { + return nil + } + return types.ErrUnsupportedAuthProviderType +} + +func (t Type) IsValid() bool { + _, ok := allowedTypes[t] + return ok +} + +func (t Type) MarshalJSON() ([]byte, error) { + if err := t.Validate(); err != nil { + return nil, err + } + return json.Marshal(t.String()) +} + +func (t *Type) UnmarshalJSON(data []byte) error { + if t == nil { + return types.ErrUnmarshalAuthProviderTypeIntoNilPointer + } + var value string + if err := json.Unmarshal(data, &value); err != nil { + return err + } + parsed := ParseType(value) + if err := parsed.Validate(); err != nil { + return err + } + *t = parsed + return nil +} diff --git a/service/internal/pkg/types/environment/environment.go b/service/internal/pkg/types/environment/environment.go new file mode 100644 index 0000000..c1bbd5f --- /dev/null +++ b/service/internal/pkg/types/environment/environment.go @@ -0,0 +1,77 @@ +package environment + +import ( + "encoding/json" + "strings" + + "gitea.privcloud.tech/mtusi2026/internal/pkg/types" +) + +type ApplicationEnvironment string + +const ( + ApplicationEnvironmentUndefined ApplicationEnvironment = "undefined" + ApplicationEnvironmentDev ApplicationEnvironment = "dev" + ApplicationEnvironmentProd ApplicationEnvironment = "prod" +) + +var allowedEnvironments = map[ApplicationEnvironment]struct{}{ + ApplicationEnvironmentUndefined: {}, + ApplicationEnvironmentDev: {}, + ApplicationEnvironmentProd: {}, +} + +func ParseEnvironment(value string) ApplicationEnvironment { + s := ApplicationEnvironment(strings.ToLower(value)) + if !s.IsValid() { + return ApplicationEnvironmentUndefined + } + return s +} + +func ParseAndValidateEnvironment(status string) (ApplicationEnvironment, error) { + parsed := ParseEnvironment(status) + if _, ok := allowedEnvironments[parsed]; ok { + return parsed, nil + } + return ApplicationEnvironmentUndefined, types.ErrUnsupportedApplicationEnvironment +} + +func (t ApplicationEnvironment) String() string { + return string(t) +} + +func (t ApplicationEnvironment) Validate() error { + if _, ok := allowedEnvironments[t]; ok { + return nil + } + return types.ErrUnsupportedApplicationEnvironment +} + +func (t ApplicationEnvironment) IsValid() bool { + _, ok := allowedEnvironments[t] + return ok +} + +func (t ApplicationEnvironment) MarshalJSON() ([]byte, error) { + if err := t.Validate(); err != nil { + return nil, err + } + return json.Marshal(t.String()) +} + +func (t *ApplicationEnvironment) UnmarshalJSON(data []byte) error { + if t == nil { + return types.ErrUnmarshalApplicationEnvironmentIntoNilPointer + } + var value string + if err := json.Unmarshal(data, &value); err != nil { + return err + } + parsed := ParseEnvironment(value) + if err := parsed.Validate(); err != nil { + return err + } + *t = parsed + return nil +} diff --git a/service/internal/pkg/types/error.go b/service/internal/pkg/types/error.go new file mode 100644 index 0000000..5f52357 --- /dev/null +++ b/service/internal/pkg/types/error.go @@ -0,0 +1,25 @@ +package types + +import "errors" + +var ( + ErrUnsupportedRole = errors.New("unsupported role") + ErrUnsupportedOTPType = errors.New("unsupported OTP type") + ErrUnsupportedAuthProviderType = errors.New("unsupported auth provider type") + ErrUnsupportedContactType = errors.New("unsupported contact type") + ErrUnsupportedOTPStatus = errors.New("unsupported OTP status") + ErrUnsupportedOTPResult = errors.New("unsupported OTP result") + ErrUnsupportedFormStatus = errors.New("unsupported form status") + ErrUnsupportedAuthIdentityStatus = errors.New("unsupported auth identity status") + ErrUnsupportedApplicationEnvironment = errors.New("unsupported application environment") + + ErrUnmarshalRoleIntoNilPointer = errors.New("unmarshal role into nil pointer") + ErrUnmarshalOTPTypeIntoNilPointer = errors.New("unmarshal OTP type into nil pointer") + ErrUnmarshalAuthProviderTypeIntoNilPointer = errors.New("unmarshal auth provider into nil pointer") + ErrUnmarshalContactTypeIntoNilPointer = errors.New("unmarshal contact into nil pointer") + ErrUnmarshalOTPStatusIntoNilPointer = errors.New("unmarshal OTP status into nil pointer") + ErrUnmarshalOTPResultIntoNilPointer = errors.New("unmarshal OTP result into nil pointer") + ErrUnmarshalFormStatusIntoNilPointer = errors.New("unmarshal form status into nil pointer") + ErrUnmarshalAuthIdentityStatusIntoNilPointer = errors.New("unmarshal auth identity status into nil pointer") + ErrUnmarshalApplicationEnvironmentIntoNilPointer = errors.New("unmarshal application environment into nil pointer") +) diff --git a/service/internal/pkg/types/role/role.go b/service/internal/pkg/types/role/role.go new file mode 100644 index 0000000..6f79d78 --- /dev/null +++ b/service/internal/pkg/types/role/role.go @@ -0,0 +1,105 @@ +package role + +import ( + "encoding/json" + "strings" + + "gitea.privcloud.tech/mtusi2026/internal/pkg/types" +) + +type Role string + +const ( + Undefined Role = "undefined" + User Role = "user" + Doctor Role = "doctor" + Manager Role = "manager" + Admin Role = "admin" +) + +var allowedRoles = map[Role]struct{}{ + User: {}, + Doctor: {}, + Manager: {}, + Admin: {}, +} + +func ParseRole(s string) Role { + r := Role(strings.ToLower(s)) + if !r.IsValid() { + return Undefined + } + return r +} + +func ParseAndValidateRole(s string) (Role, error) { + parsed := ParseRole(s) + if _, ok := allowedRoles[parsed]; ok { + return parsed, nil + } + return Undefined, types.ErrUnsupportedRole +} + +func (t Role) String() string { + return string(t) +} + +func (t Role) Validate() error { + if _, ok := allowedRoles[t]; ok { + return nil + } + return types.ErrUnsupportedRole +} + +func (t Role) IsValid() bool { + _, ok := allowedRoles[t] + return ok +} + +func (t Role) MarshalJSON() ([]byte, error) { + if err := t.Validate(); err != nil { + return nil, err + } + return json.Marshal(t.String()) +} + +func (t *Role) UnmarshalJSON(data []byte) error { + if t == nil { + return types.ErrUnmarshalRoleIntoNilPointer + } + var value string + if err := json.Unmarshal(data, &value); err != nil { + return err + } + parsed := ParseRole(value) + if err := parsed.Validate(); err != nil { + return err + } + *t = parsed + return nil +} + +func (t Role) IsAdmin() bool { + return t == Admin +} + +func (t Role) IsManager() bool { + return t == Manager +} + +func (t Role) IsDoctor() bool { + return t == Doctor +} + +func (t Role) IsUser() bool { + return t == User +} + +func (t Role) IsPrivileged() bool { + switch t { + case Doctor, Manager, Admin: + return true + default: + return false + } +} diff --git a/service/internal/pkg/validator/validator.go b/service/internal/pkg/validator/validator.go new file mode 100644 index 0000000..d4ab8aa --- /dev/null +++ b/service/internal/pkg/validator/validator.go @@ -0,0 +1,29 @@ +package validator + +import ( + "context" + + "github.com/go-playground/validator/v10" +) + +type TagKey string + +type ValidatorImpl struct { + validator *validator.Validate +} + +func NewValidatorImpl() *ValidatorImpl { + return &ValidatorImpl{validator: validator.New()} +} + +func (v *ValidatorImpl) Register(tag TagKey, fn validator.Func, callValidationEvenIfNull ...bool) error { + return v.validator.RegisterValidation(string(tag), fn, callValidationEvenIfNull...) +} + +func (v *ValidatorImpl) ValidateCtx(ctx context.Context, value any) error { + return v.validator.StructCtx(ctx, value) +} + +func (v *ValidatorImpl) Validate(value any) error { + return v.validator.Struct(value) +} diff --git a/service/internal/service/attachment/interface.go b/service/internal/service/attachment/interface.go new file mode 100644 index 0000000..0919786 --- /dev/null +++ b/service/internal/service/attachment/interface.go @@ -0,0 +1,55 @@ +package attachment + +import ( + "context" + "gitea.privcloud.tech/mtusi2026/internal/dto" + "gitea.privcloud.tech/mtusi2026/internal/service/outbox" + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/google/uuid" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" + "io" + "time" +) + +type TransactionManager interface { + RunInTransaction(context.Context, *pgx.TxOptions, func(pgx.Tx) error) error +} +type Outbox interface { + CreateTx(context.Context, pgx.Tx, outbox.CreateMessageReq) (outbox.CreateMessageRes, error) +} +type ObjectStore interface { + Put(context.Context, string, string, io.Reader, int64) error + Get(context.Context, string) (*s3.GetObjectOutput, error) + Head(context.Context, string) (*s3.HeadObjectOutput, error) + Delete(context.Context, string) error + Ping(context.Context) error + List(context.Context, string) (*s3.ListObjectsV2Output, error) +} +type Repository interface { + FailSLO(context.Context, pgx.Tx, uuid.UUID, time.Time) error + AcceptSLO(context.Context, pgx.Tx, uuid.UUID, time.Time, time.Time) error + ReadySLO(context.Context, pgx.Tx, uuid.UUID, time.Time) error + + SaveVariantTx(context.Context, pgx.Tx, uuid.UUID, int, string, string, string, int64) error + + Ping(context.Context) error + Get(context.Context, uuid.UUID) (*dto.AttachmentRecord, error) + GetTx(context.Context, pgx.Tx, uuid.UUID) (*dto.AttachmentRecord, error) + InsertTx(context.Context, pgx.Tx, *dto.AttachmentRecord) error + UpdateTx(context.Context, pgx.Tx, *dto.AttachmentRecord) error + JobTx(context.Context, pgx.Tx, uuid.UUID) (*dto.AttachmentJob, error) + InsertJobTx(context.Context, pgx.Tx, *dto.AttachmentJob) error + SaveJobTx(context.Context, pgx.Tx, *dto.AttachmentJob) error + ExpiredJobs(context.Context) ([]uuid.UUID, error) + PublicationDeadTx(context.Context, pgx.Tx, uuid.UUID, int) (bool, error) + FailedOutbox(context.Context) ([]uuid.UUID, error) + ListJobs(context.Context) ([]dto.AttachmentJob, error) + RetiredSources(context.Context, time.Time) ([]*dto.AttachmentRecord, error) + AuditTx(context.Context, pgx.Tx, uuid.UUID, string) error + Lock(context.Context, string) (*pgxpool.Conn, func(), error) + Exists(context.Context, *pgxpool.Conn, string) (bool, error) + Page(context.Context, uuid.UUID) ([]*dto.AttachmentRecord, error) + MarkMissing(context.Context, *pgxpool.Conn, uuid.UUID, bool) (bool, error) + CheckConn(context.Context, *pgxpool.Conn) error +} diff --git a/service/internal/service/attachment/pipeline.go b/service/internal/service/attachment/pipeline.go new file mode 100644 index 0000000..c3d5d24 --- /dev/null +++ b/service/internal/service/attachment/pipeline.go @@ -0,0 +1,555 @@ +package attachment + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/json" + "errors" + "fmt" + "gitea.privcloud.tech/mtusi2026/internal/dto" + "gitea.privcloud.tech/mtusi2026/internal/gen/contracts" + "gitea.privcloud.tech/mtusi2026/internal/pkg/apperr" + "gitea.privcloud.tech/mtusi2026/internal/pkg/fault" + "gitea.privcloud.tech/mtusi2026/internal/pkg/imageprocessor" + "gitea.privcloud.tech/mtusi2026/internal/pkg/objectstore" + "gitea.privcloud.tech/mtusi2026/internal/pkg/telemetry" + "gitea.privcloud.tech/mtusi2026/internal/service/outbox" + "github.com/google/uuid" + "github.com/jackc/pgx/v5" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/codes" + "go.opentelemetry.io/otel/trace" + "io" + "log/slog" + "os" + "path/filepath" + "strings" + "time" +) + +var ErrBusy = errors.New("job lease held") + +func validateImage(ctx context.Context, path string) error { return imageprocessor.Validate(ctx, path) } +func (s *Service) Fault(ctx context.Context, point string, id uuid.UUID) { fault.Hit(ctx, point, id) } +func (s *Service) newJob(ctx context.Context, tx pgx.Tx, a *dto.AttachmentRecord) error { + now := time.Now().UTC() + j := &dto.AttachmentJob{Message: dto.JobMessage{SchemaVersion: 1, JobID: uuid.New(), Type: "attachment.process.v1", AttachmentID: a.Public.Id, PipelineVersion: a.PipelineVersion, Attempt: 1, CreatedAt: now}, State: "queued", Token: uuid.New(), NextAttemptAt: now, AttemptLimit: s.Jobs.MaxAttempts, UpdatedAt: now} + j.Message.Traceparent, j.Message.Tracestate = telemetry.Inject(ctx) + if e := s.Repo.AcceptSLO(ctx, tx, j.Message.JobID, now, now.Add(30*time.Second)); e != nil { + return e + } + if e := s.Repo.InsertJobTx(ctx, tx, j); e != nil { + return e + } + return s.enqueue(ctx, tx, j) +} +func (s *Service) enqueue(ctx context.Context, tx pgx.Tx, j *dto.AttachmentJob) error { + j.Message.Traceparent, j.Message.Tracestate = telemetry.Inject(ctx) + b, e := json.Marshal(j.Message) + if e != nil { + return e + } + _, e = s.outbox.CreateTx(ctx, tx, outbox.CreateMessageReq{ID: uuid.New(), AggregateID: j.Message.JobID, Transport: outbox.TransportAMQP, Payload: b, CreatedAt: j.NextAttemptAt, MaxAttempts: s.Jobs.MaxAttempts}) + return e +} +func (s *Service) Content(ctx context.Context, user, id uuid.UUID, variant string) (*dto.AttachmentContent, error) { + a, e := s.Get(ctx, user, id) + if e != nil { + return nil, e + } + if a.Public.Status != "ready" { + return nil, apperr.New(apperr.CodeConflict, "attachment not ready") + } + key, mime, size := a.Key, string(a.Public.ContentType), int64(a.Public.SizeBytes) + if variant != "original" { + key = a.VariantKeys[variant] + if key == "" { + return nil, apperr.New(apperr.CodeNotFound, "variant not found") + } + for _, v := range a.Public.Variants { + if string(v.Name) == variant { + mime = string(v.ContentType) + size = int64(v.SizeBytes) + } + } + } + obj, e := s.Store.Get(ctx, key) + if e != nil { + return nil, Unavailable() + } + return &dto.AttachmentContent{Body: obj.Body, ContentType: mime, Size: size}, nil +} +func (s *Service) Process(ctx context.Context, m dto.JobMessage) (processErr error) { + ctx, sp := telemetry.Start(ctx, "attachment.process", trace.SpanKindInternal) + outcome := "unavailable" + started := time.Now() + telemetry.Busy.Add(1) + defer telemetry.Busy.Add(-1) + defer func() { + if outcome == "retry" || outcome == "failed" { + sp.SetStatus(codes.Error, outcome) + } + telemetry.End(sp, processErr) + telemetry.Duration(ctx, "messenger.job.duration", time.Since(started), attribute.String("job_type", m.Type), attribute.String("outcome", outcome)) + }() + + var j *dto.AttachmentJob + var a *dto.AttachmentRecord + claimed := false + err := s.tm.RunInTransaction(ctx, nil, func(tx pgx.Tx) error { + var e error + j, e = s.Repo.JobTx(ctx, tx, m.JobID) + if errors.Is(e, pgx.ErrNoRows) { + return nil + } + if e != nil { + return e + } + // Broker data is a notification. Trusted metadata and attempt live in PostgreSQL. + if j.Message.AttachmentID != m.AttachmentID || j.Message.PipelineVersion != m.PipelineVersion || j.Message.Attempt != m.Attempt { + return nil + } + a, e = s.Repo.GetTx(ctx, tx, m.AttachmentID) + if e != nil { + return e + } + if a.PipelineVersion != m.PipelineVersion || j.State == "ready" || j.State == "failed" { + return nil + } + if j.State == "processing" { + return ErrBusy + } + if time.Now().Before(j.NextAttemptAt) { + return ErrBusy + } + j.State = "processing" + j.Token = uuid.New() + lease := time.Now().Add(s.Jobs.Lease) + j.LeaseUntil = &lease + j.UpdatedAt = time.Now().UTC() + a.Public.Status = "processing" + a.Public.UpdatedAt = j.UpdatedAt + if e = s.Repo.SaveJobTx(ctx, tx, j); e != nil { + return e + } + if e = s.Repo.UpdateTx(ctx, tx, a); e != nil { + return e + } + claimed = true + return nil + }) + if err != nil || !claimed { + if err == nil { + outcome = "duplicate" + telemetry.Count(ctx, "messenger.jobs", 1, attribute.String("job_type", m.Type), attribute.String("outcome", "duplicate")) + } + return err + } + s.Fault(ctx, "after_claim", m.JobID) + task, cancel := context.WithTimeout(ctx, s.Jobs.Timeout) + defer cancel() + result, e := s.effects(task, j, a) + if e != nil { + if ctx.Err() != nil { + return ctx.Err() + } + finishErr := s.finish(ctx, j, nil, e) + if finishErr == nil { + outcome = "retry" + if j.State == "failed" { + outcome = "failed" + } + telemetry.Count(ctx, "messenger.jobs", 1, attribute.String("job_type", m.Type), attribute.String("outcome", outcome)) + slog.WarnContext(ctx, "job_"+outcome, "job_id", m.JobID) + } + return finishErr + } + s.Fault(ctx, "after_objects", m.JobID) + if err = s.finish(ctx, j, result, nil); err != nil { + return err + } + if !result.Committed { + outcome = "stale" + return nil + } + outcome = "ready" + s.Fault(ctx, "after_result_commit", m.JobID) + telemetry.Count(ctx, "messenger.jobs", 1, attribute.String("job_type", m.Type), attribute.String("outcome", "ready")) + telemetry.Duration(ctx, "messenger.job.ready.duration", time.Since(j.Message.CreatedAt), attribute.String("job_type", m.Type)) + slog.InfoContext(ctx, "job_ready", "job_id", m.JobID) + + return nil +} + +type processed struct { + Committed bool + Key string + Variants []contracts.Variant + Keys map[string]string +} +type permanentError struct{ code string } + +func (e permanentError) Error() string { return e.code } +func (s *Service) effects(ctx context.Context, j *dto.AttachmentJob, a *dto.AttachmentRecord) (*processed, error) { + dir, e := os.MkdirTemp("", "pipeline-*") + if e != nil { + return nil, e + } + defer os.RemoveAll(dir) + key := a.StagingKey + if key == "" { + key = a.Key + } + obj, e := s.Store.Get(ctx, key) + if e != nil { + if objectstore.Missing(e) { + return nil, permanentError{"source_missing"} + } + return nil, e + } + defer obj.Body.Close() + path := filepath.Join(dir, "source") + f, e := os.Create(path) + if e != nil { + return nil, e + } + hash := sha256.New() + n, e := io.Copy(io.MultiWriter(f, hash), io.LimitReader(obj.Body, s.Config.MaxBytes+1)) + f.Close() + if e != nil { + return nil, e + } + if n != int64(a.Public.SizeBytes) || fmt.Sprintf("%x", hash.Sum(nil)) != a.Public.Sha256 { + return nil, permanentError{"source_corrupt"} + } + f, e = os.Open(path) + if e != nil { + return nil, e + } + e = s.Validate(ctx, f, string(a.Public.ContentType)) + f.Close() + if e != nil { + if ctx.Err() != nil { + return nil, ctx.Err() + } + return nil, permanentError{"invalid_image"} + } + s.Fault(ctx, "before_objects", j.Message.JobID) + if e := fault.Error("before_objects", j.Message.JobID); e != nil { + if e.Error() == "permanent" { + return nil, permanentError{"invalid_image"} + } + return nil, e + } + imagectx, imageSpan := telemetry.Start(ctx, "image.transform", trace.SpanKindInternal) + converted, e := imageprocessor.Process(imagectx, path, dir, j.Message.PipelineVersion) + telemetry.End(imageSpan, e) + if e != nil { + return nil, e + } + prefix := fmt.Sprintf("results/%s/v%d/", a.Public.Id, j.Message.PipelineVersion) + result := &processed{Key: prefix + "original/" + a.Public.Sha256, Variants: []contracts.Variant{}, Keys: map[string]string{}} + f, e = os.Open(path) + if e != nil { + return nil, e + } + e = s.Store.Put(ctx, result.Key, string(a.Public.ContentType), f, n) + f.Close() + if e != nil { + return nil, e + } + s.Fault(ctx, "after_original_put", j.Message.JobID) + for _, v := range converted { + b, e := os.ReadFile(v.Path) + if e != nil { + return nil, e + } + sum := fmt.Sprintf("%x", sha256.Sum256(b)) + key := prefix + v.Name + "/" + sum + if e = s.Store.Put(ctx, key, v.MIME, bytes.NewReader(b), int64(len(b))); e != nil { + return nil, e + } + result.Keys[v.Name] = key + result.Variants = append(result.Variants, contracts.Variant{Name: contracts.VariantName(v.Name), ContentType: contracts.VariantContentType(v.MIME), Width: v.Width, Height: v.Height, SizeBytes: len(b), Sha256: sum}) + } + return result, nil +} +func (s *Service) finish(ctx context.Context, owned *dto.AttachmentJob, result *processed, cause error) error { + applied := false + err := s.tm.RunInTransaction(ctx, nil, func(tx pgx.Tx) error { + j, e := s.Repo.JobTx(ctx, tx, owned.Message.JobID) + if e != nil { + return e + } + a, e := s.Repo.GetTx(ctx, tx, j.Message.AttachmentID) + if e != nil { + return e + } + if j.Token != owned.Token || j.State != "processing" || j.LeaseUntil == nil || !j.LeaseUntil.After(time.Now()) || a.PipelineVersion != j.Message.PipelineVersion { + return nil + } + if cause != nil { + err := s.retry(ctx, tx, j, a, cause) + if err == nil { + owned.State = j.State + } + return err + } + + if e = s.Repo.SaveVariantTx(ctx, tx, a.Public.Id, j.Message.PipelineVersion, "original", result.Key, a.Public.Sha256, int64(a.Public.SizeBytes)); e != nil { + return e + } + for _, v := range result.Variants { + if e = s.Repo.SaveVariantTx(ctx, tx, a.Public.Id, j.Message.PipelineVersion, string(v.Name), result.Keys[string(v.Name)], v.Sha256, int64(v.SizeBytes)); e != nil { + return e + } + } + j.State = "ready" + j.ErrorCode = nil + j.LeaseUntil = nil + j.UpdatedAt = time.Now().UTC() + a.Missing = false + a.Key = result.Key + a.VariantKeys = result.Keys + a.Public.Variants = result.Variants + a.Public.Status = "ready" + a.Public.ErrorCode = nil + a.Public.UpdatedAt = j.UpdatedAt + if e = s.Repo.SaveJobTx(ctx, tx, j); e != nil { + return e + } + if e = s.Repo.UpdateTx(ctx, tx, a); e != nil { + return e + } + applied = true + return s.Repo.ReadySLO(ctx, tx, j.Message.JobID, j.UpdatedAt) + }) + if result != nil { + result.Committed = err == nil && applied + } + return err +} +func (s *Service) retry(ctx context.Context, tx pgx.Tx, j *dto.AttachmentJob, a *dto.AttachmentRecord, cause error) error { + j.Message.Traceparent, j.Message.Tracestate = telemetry.Inject(ctx) + + code := "processing_failed" + var permanent permanentError + terminal := errors.As(cause, &permanent) || j.Message.Attempt >= j.AttemptLimit + if permanent.code != "" { + code = permanent.code + } + if terminal && permanent.code == "" { + code = "retry_exhausted" + } + j.LeaseUntil = nil + j.Token = uuid.New() + j.UpdatedAt = time.Now().UTC() + j.ErrorCode = &code + if terminal { + if e := s.Repo.FailSLO(ctx, tx, j.Message.JobID, j.UpdatedAt); e != nil { + return e + } + j.State = "failed" + a.Public.Status = "failed" + a.Public.ErrorCode = &code + } else { + j.State = "queued" + j.Message.Attempt++ + j.NextAttemptAt = j.UpdatedAt.Add(outbox.Backoff(j.Message.Attempt - 1)) + a.Public.Status = "queued" + a.Public.ErrorCode = nil + if e := s.enqueue(ctx, tx, j); e != nil { + return e + } + } + a.Public.UpdatedAt = j.UpdatedAt + if e := s.Repo.SaveJobTx(ctx, tx, j); e != nil { + return e + } + return s.Repo.UpdateTx(ctx, tx, a) +} +func (s *Service) Recover(ctx context.Context) error { + ids, e := s.Repo.ExpiredJobs(ctx) + if e != nil { + return e + } + for _, id := range ids { + e = s.tm.RunInTransaction(ctx, nil, func(tx pgx.Tx) error { + j, e := s.Repo.JobTx(ctx, tx, id) + if e != nil { + return e + } + if j.State != "processing" || j.LeaseUntil == nil || j.LeaseUntil.After(time.Now()) { + return nil + } + a, e := s.Repo.GetTx(ctx, tx, j.Message.AttachmentID) + if e != nil { + return e + } + if a.PipelineVersion != j.Message.PipelineVersion { + return nil + } + return s.retry(telemetry.Extract(ctx, j.Message.Traceparent, j.Message.Tracestate), tx, j, a, errors.New("lease expired")) + }) + if e != nil { + return e + } + } + dead, e := s.Repo.FailedOutbox(ctx) + if e != nil { + return e + } + for _, id := range dead { + if e = s.PublishFailed(ctx, id); e != nil { + return e + } + } + return nil +} +func (s *Service) PublishFailed(ctx context.Context, id uuid.UUID) error { + return s.tm.RunInTransaction(ctx, nil, func(tx pgx.Tx) error { + j, e := s.Repo.JobTx(ctx, tx, id) + if e != nil { + return e + } + if j.State != "queued" { + return nil + } + dead, e := s.Repo.PublicationDeadTx(ctx, tx, id, j.Message.Attempt) + if e != nil { + return e + } + if !dead { + return nil + } + a, e := s.Repo.GetTx(ctx, tx, j.Message.AttachmentID) + if e != nil { + return e + } + if a.PipelineVersion != j.Message.PipelineVersion { + return nil + } + if e := s.Repo.FailSLO(ctx, tx, j.Message.JobID, time.Now().UTC()); e != nil { + return e + } + code := "publish_failed" + j.State = "failed" + j.ErrorCode = &code + j.UpdatedAt = time.Now().UTC() + a.Public.Status = "failed" + a.Public.ErrorCode = &code + a.Public.UpdatedAt = j.UpdatedAt + if e = s.Repo.SaveJobTx(ctx, tx, j); e != nil { + return e + } + return s.Repo.UpdateTx(ctx, tx, a) + }) +} +func (s *Service) Replay(ctx context.Context, id uuid.UUID) error { + return s.tm.RunInTransaction(ctx, nil, func(tx pgx.Tx) error { + j, e := s.Repo.JobTx(ctx, tx, id) + if e != nil { + return e + } + a, e := s.Repo.GetTx(ctx, tx, j.Message.AttachmentID) + if e != nil { + return e + } + if j.State != "failed" || a.PipelineVersion != j.Message.PipelineVersion { + return apperr.New(apperr.CodeConflict, "only current failed job can be replayed") + } + source := a.StagingKey + if source == "" { + return errors.New("source unavailable; restore original first") + } + if _, e = s.Store.Head(ctx, source); e != nil { + return errors.New("source unavailable") + } + j.State = "queued" + j.Message.Attempt++ + j.AttemptLimit = j.Message.Attempt + s.Jobs.MaxAttempts - 1 + j.NextAttemptAt = time.Now().UTC() + j.UpdatedAt = j.NextAttemptAt + j.Token = uuid.New() + j.ErrorCode = nil + j.LeaseUntil = nil + a.Public.Status = "queued" + a.Public.ErrorCode = nil + a.Public.UpdatedAt = j.UpdatedAt + if e = s.Repo.SaveJobTx(ctx, tx, j); e != nil { + return e + } + if e = s.Repo.UpdateTx(ctx, tx, a); e != nil { + return e + } + if e = s.Repo.AuditTx(ctx, tx, id, "replay"); e != nil { + return e + } + return s.enqueue(ctx, tx, j) + }) +} +func (s *Service) Reprocess(ctx context.Context, id uuid.UUID, version int) error { + if version < 1 || version > 2 { + return errors.New("supported pipeline versions: 1, 2") + } + return s.tm.RunInTransaction(ctx, nil, func(tx pgx.Tx) error { + a, e := s.Repo.GetTx(ctx, tx, id) + if e != nil { + return e + } + if version <= a.PipelineVersion || a.Public.Status != "ready" { + return errors.New("requires ready attachment and newer version") + } + a.PipelineVersion = version + a.StagingKey = a.Key + a.Public.Status = "queued" + a.Public.Variants = []contracts.Variant{} + a.VariantKeys = map[string]string{} + a.Public.UpdatedAt = time.Now().UTC() + if e = s.Repo.UpdateTx(ctx, tx, a); e != nil { + return e + } + return s.newJob(ctx, tx, a) + }) +} +func (s *Service) Cleanup(ctx context.Context) error { + records, e := s.Repo.RetiredSources(ctx, time.Now().Add(-s.Jobs.Retention)) + if e != nil { + return e + } + for _, record := range records { + // Serialize deletion with operator Restore and the initial upload handoff. + _, unlock, lockErr := s.Repo.Lock(ctx, record.StagingKey) + if lockErr != nil { + return lockErr + } + e = s.tm.RunInTransaction(ctx, nil, func(tx pgx.Tx) error { + a, e := s.Repo.GetTx(ctx, tx, record.Public.Id) + if e != nil { + return e + } + if a.StagingKey == "" || a.StagingKey == a.Key && a.Public.Status == "ready" { + return nil + } + if a.Public.Status != "ready" && (a.Public.Status != "failed" || time.Since(a.Public.UpdatedAt) < s.Jobs.Retention) { + return nil + } + if !strings.HasPrefix(a.StagingKey, "results/") { + if e = s.Store.Delete(ctx, a.StagingKey); e != nil { + return e + } + } + a.StagingKey = "" + return s.Repo.UpdateTx(ctx, tx, a) + }) + unlock() + if e != nil { + return e + } + } + return nil +} +func managedPipelineKey(key string) bool { + return strings.HasPrefix(key, "staging/") || strings.HasPrefix(key, "results/") +} diff --git a/service/internal/service/attachment/reconcile.go b/service/internal/service/attachment/reconcile.go new file mode 100644 index 0000000..0ad54ad --- /dev/null +++ b/service/internal/service/attachment/reconcile.go @@ -0,0 +1,228 @@ +package attachment + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "fmt" + "io" + "os" + "time" + + "github.com/aws/aws-sdk-go-v2/aws" + + "github.com/google/uuid" + "github.com/jackc/pgx/v5" + + "gitea.privcloud.tech/mtusi2026/internal/pkg/objectstore" +) + +type ScanResult struct { + Apply bool `json:"apply"` + Objects int `json:"objects"` + Metadata int `json:"metadata"` + Orphans int `json:"orphans"` + Fresh int `json:"fresh_orphans"` + Deleted int `json:"deleted"` + Missing int `json:"missing"` + Marked int `json:"marked_missing"` + Protected int `json:"referenced_objects"` + S3Requests int `json:"s3_requests"` + DBChecks int `json:"db_checks"` + LockAttempts int `json:"lock_attempts"` + DurationMS int64 `json:"duration_ms"` + GraceSeconds float64 `json:"grace_seconds"` +} + +func (s *Service) Reconcile(ctx context.Context, apply bool) (result ScanResult, err error) { + start := time.Now() + result.Apply = apply + result.GraceSeconds = s.Config.Grace.Seconds() + defer func() { result.DurationMS = time.Since(start).Milliseconds() }() + if err = s.Repo.Ping(ctx); err != nil { + return + } + result.S3Requests++ + if err = s.Store.Ping(ctx); err != nil { + return + } + token := "" + for { + result.S3Requests++ + page, e := s.Store.List(ctx, token) + if e != nil { + return result, e + } + for _, obj := range page.Contents { + result.Objects++ + key := aws.ToString(obj.Key) + if !ManagedKey(key) { + continue + } + result.LockAttempts++ + conn, unlock, e := s.Repo.Lock(ctx, key) + if e != nil { + return result, e + } + e = func() error { + defer unlock() + result.DBChecks++ + found, e := s.Repo.Exists(ctx, conn, key) + if e != nil { + return e + } + if found { + result.Protected++ + return nil + } + // Re-read both metadata and object state under the same lock as upload. + result.S3Requests++ + head, e := s.Store.Head(ctx, key) + if objectstore.Missing(e) { + return nil + } + if e != nil { + return e + } + result.Orphans++ + if head.LastModified == nil || time.Since(*head.LastModified) <= s.Config.Grace { + result.Fresh++ + return nil + } + if !apply { + return nil + } + result.DBChecks++ + if e = s.Repo.CheckConn(ctx, conn); e != nil { + return e + } + result.S3Requests++ + if e = s.Store.Delete(ctx, key); e != nil { + return e + } + result.Deleted++ + return nil + }() + if e != nil { + return result, e + } + } + if !aws.ToBool(page.IsTruncated) { + break + } + token = aws.ToString(page.NextContinuationToken) + } + + after := uuid.Nil + for { + result.DBChecks++ + page, e := s.Repo.Page(ctx, after) + if e != nil { + return result, e + } + if len(page) == 0 { + break + } + for _, a := range page { + result.Metadata++ + after = a.Public.Id + if a.Public.Status != "ready" { + continue + } + result.LockAttempts++ + conn, unlock, e := s.Repo.Lock(ctx, a.Key) + if e != nil { + return result, e + } + e = func() error { + defer unlock() + result.S3Requests++ + _, e := s.Store.Head(ctx, a.Key) + if e == nil { + return nil + } + if !objectstore.Missing(e) { + return e + } + result.Missing++ + if apply { + result.DBChecks++ + changed, err := s.Repo.MarkMissing(ctx, conn, a.Public.Id, true) + if err != nil { + return err + } + if changed { + result.Marked++ + } + } + return nil + }() + if e != nil { + return result, e + } + } + } + return result, nil +} +func (s *Service) Restore(ctx context.Context, id uuid.UUID, path string) error { + a, e := s.Repo.Get(ctx, id) + if e != nil { + return e + } + f, e := os.Open(path) + if e != nil { + return e + } + defer f.Close() + hash := sha256.New() + size, e := io.Copy(hash, io.LimitReader(f, s.Config.MaxBytes+1)) + if e != nil { + return e + } + if size != int64(a.Public.SizeBytes) || hex.EncodeToString(hash.Sum(nil)) != a.Public.Sha256 { + return fmt.Errorf("original checksum or size mismatch") + } + if e = s.Validate(ctx, f, string(a.Public.ContentType)); e != nil { + return e + } + conn, unlock, e := s.Repo.Lock(ctx, a.Key) + if e != nil { + return e + } + defer unlock() + if _, e = f.Seek(0, 0); e != nil { + return e + } + if e = s.Store.Put(ctx, a.Key, string(a.Public.ContentType), f, size); e != nil { + return e + } + obj, e := s.Store.Get(ctx, a.Key) + if e != nil { + return e + } + defer obj.Body.Close() + hash.Reset() + n, e := io.Copy(hash, io.LimitReader(obj.Body, s.Config.MaxBytes+1)) + if e != nil { + return e + } + if n != size || hex.EncodeToString(hash.Sum(nil)) != a.Public.Sha256 { + return fmt.Errorf("stored original verification failed") + } + _, e = s.Repo.MarkMissing(ctx, conn, id, false) + if e != nil { + return e + } + return s.tm.RunInTransaction(ctx, nil, func(tx pgx.Tx) error { + current, e := s.Repo.GetTx(ctx, tx, id) + if e != nil { + return e + } + if current.Public.Status == "failed" { + current.StagingKey = current.Key + current.Public.UpdatedAt = time.Now().UTC() + return s.Repo.UpdateTx(ctx, tx, current) + } + return nil + }) +} diff --git a/service/internal/service/attachment/service.go b/service/internal/service/attachment/service.go new file mode 100644 index 0000000..47749e4 --- /dev/null +++ b/service/internal/service/attachment/service.go @@ -0,0 +1,169 @@ +package attachment + +import ( + "context" + "crypto/sha256" + "fmt" + "gitea.privcloud.tech/mtusi2026/internal/pkg/telemetry" + "go.opentelemetry.io/otel/trace" + "image" + _ "image/jpeg" + _ "image/png" + "io" + + "os" + "strings" + "time" + "unicode" + "unicode/utf8" + + "github.com/google/uuid" + + "gitea.privcloud.tech/mtusi2026/internal/config" + "gitea.privcloud.tech/mtusi2026/internal/dto" + "gitea.privcloud.tech/mtusi2026/internal/gen/contracts" + "gitea.privcloud.tech/mtusi2026/internal/pkg/apperr" + "github.com/jackc/pgx/v5" +) + +const Prefix = "originals/" +const StagingPrefix = "staging/" + +type Membership interface { + GetForMember(context.Context, dto.ChatMember) (*dto.Chat, error) +} +type Service struct { + Repo Repository + Store ObjectStore + Config *config.Attachment + chats Membership + tm TransactionManager + outbox Outbox + Jobs *config.Pipeline + uploads, decoders chan struct{} +} + +func New(repo Repository, store ObjectStore, c *config.Attachment, chats Membership) *Service { + return &Service{Repo: repo, Store: store, Config: c, chats: chats, uploads: make(chan struct{}, c.Uploads), decoders: make(chan struct{}, c.Decoders)} +} +func ValidFilename(s string) bool { + if strings.TrimSpace(s) == "" || !utf8.ValidString(s) || utf8.RuneCountInString(s) < 1 || utf8.RuneCountInString(s) > 255 { + return false + } + for _, r := range s { + if r == '/' || r == '\\' || unicode.IsControl(r) { + return false + } + } + return true +} +func (s *Service) Configure(tm TransactionManager, outbox Outbox, jobs *config.Pipeline) { + s.tm = tm + s.outbox = outbox + s.Jobs = jobs +} +func (s *Service) Upload(ctx context.Context, req dto.UploadAttachment) (out *contracts.Attachment, err error) { + ctx, sp := telemetry.Start(ctx, "attachment.upload", trace.SpanKindInternal) + defer func() { telemetry.End(sp, err) }() + if _, e := s.chats.GetForMember(ctx, dto.ChatMember{UserID: req.UserID, ChatID: req.ChatID}); e != nil { + return nil, e + } + select { + case s.uploads <- struct{}{}: + defer func() { <-s.uploads }() + default: + return nil, apperr.New(apperr.CodeTooManyRequests, "upload capacity exceeded") + } + f, e := os.CreateTemp("", "attachment-*") + if e != nil { + return nil, e + } + defer os.Remove(f.Name()) + defer f.Close() + hash := sha256.New() + size, e := io.Copy(io.MultiWriter(f, hash), io.LimitReader(req.Body, s.Config.MaxBytes+1)) + if e != nil { + return nil, apperr.New(apperr.CodeBadRequest, "incomplete upload") + } + if size > s.Config.MaxBytes { + return nil, apperr.New(apperr.CodePayloadTooLarge, "upload too large") + } + if e = s.Validate(ctx, f, req.ContentType); e != nil { + return nil, e + } + now := time.Now().UTC().Truncate(time.Microsecond) + id := uuid.New() + key := StagingPrefix + id.String() + a := &dto.AttachmentRecord{Key: key, StagingKey: key, PipelineVersion: 1, VariantKeys: map[string]string{}, Public: contracts.Attachment{Id: id, ChatId: req.ChatID, OwnerId: req.UserID, Filename: req.Filename, ContentType: contracts.AttachmentContentType(req.ContentType), SizeBytes: int(size), Sha256: fmt.Sprintf("%x", hash.Sum(nil)), Status: "queued", Variants: []contracts.Variant{}, CreatedAt: now, UpdatedAt: now}} + _, unlock, e := s.Repo.Lock(ctx, key) + if e != nil { + return nil, e + } + defer unlock() + if _, e = f.Seek(0, 0); e != nil { + return nil, e + } + if e = s.Store.Put(ctx, key, req.ContentType, f, size); e != nil { + return nil, Unavailable() + } + s.Fault(ctx, "after_staging", id) + if e = s.tm.RunInTransaction(ctx, nil, func(tx pgx.Tx) error { + if e := s.Repo.InsertTx(ctx, tx, a); e != nil { + return e + } + return s.newJob(ctx, tx, a) + }); e != nil { + return nil, e + } + s.Fault(ctx, "after_upload_commit", id) + return &a.Public, nil +} +func (s *Service) Validate(ctx context.Context, f *os.File, mt string) error { + if _, e := f.Seek(0, 0); e != nil { + return e + } + cfg, format, e := image.DecodeConfig(f) + if e != nil || ("image/"+format) != mt { + return apperr.New(apperr.CodeUnsupportedMediaType, "invalid image or MIME mismatch") + } + if cfg.Width <= 0 || cfg.Height <= 0 || int64(cfg.Width) > s.Config.MaxPixels/int64(cfg.Height) { + return apperr.New(apperr.CodePayloadTooLarge, "image pixel limit exceeded") + } + select { + case s.decoders <- struct{}{}: + defer func() { <-s.decoders }() + case <-ctx.Done(): + return ctx.Err() + } + if _, e = f.Seek(0, 0); e != nil { + return e + } + if e = validateImage(ctx, f.Name()); e != nil { + return apperr.New(apperr.CodeUnsupportedMediaType, "corrupt image") + } + return ctx.Err() +} +func (s *Service) Get(ctx context.Context, user, id uuid.UUID) (*dto.AttachmentRecord, error) { + a, e := s.Repo.Get(ctx, id) + if e != nil { + return nil, e + } + if _, e = s.chats.GetForMember(ctx, dto.ChatMember{UserID: user, ChatID: a.Public.ChatId}); e != nil { + return nil, e + } + if a.Missing { + return nil, Unavailable() + } + return a, nil +} +func Unavailable() error { return apperr.New(apperr.CodeUnavailable, "attachment storage unavailable") } +func ManagedKey(key string) bool { + if managedPipelineKey(key) { + return true + } + if !strings.HasPrefix(key, Prefix) { + return false + } + id, e := uuid.Parse(strings.TrimPrefix(key, Prefix)) + return e == nil && key == Prefix+id.String() +} diff --git a/service/internal/service/auth/api.go b/service/internal/service/auth/api.go new file mode 100644 index 0000000..6c9ec6a --- /dev/null +++ b/service/internal/service/auth/api.go @@ -0,0 +1,15 @@ +package auth + +import ( + "context" + + "gitea.privcloud.tech/mtusi2026/internal/dto" +) + +type API interface { + Login(ctx context.Context, req dto.Login) (*dto.LoginResult, error) + VerifyAccessToken(ctx context.Context, req dto.Token) (*dto.UserDataInToken, error) + VerifySession(ctx context.Context, token dto.Token) (*dto.UserDataInToken, error) + RefreshTokens(ctx context.Context, req dto.RefreshToken) (*dto.Tokens, error) + Logout(ctx context.Context, req dto.RefreshToken) error +} diff --git a/service/internal/service/auth/interface.go b/service/internal/service/auth/interface.go new file mode 100644 index 0000000..8ef7ddc --- /dev/null +++ b/service/internal/service/auth/interface.go @@ -0,0 +1,22 @@ +package auth + +import ( + "context" + + "github.com/google/uuid" + "github.com/jackc/pgx/v5" + + "gitea.privcloud.tech/mtusi2026/internal/dto" +) + +type userRepo interface { + UpdatePasswordHash(context.Context, uuid.UUID, string, string) error + GetCredentialsByUsername(ctx context.Context, username string) (*dto.UserCredentials, error) +} + +type refreshTokensRepo interface { + GetByID(ctx context.Context, id uuid.UUID) (*dto.StoredRefreshToken, error) + GetByIDForUpdateTx(ctx context.Context, tx pgx.Tx, id uuid.UUID) (*dto.StoredRefreshToken, error) + CreateTx(ctx context.Context, tx pgx.Tx, req dto.StoredRefreshToken) error + RevokeTx(ctx context.Context, tx pgx.Tx, req dto.RevokeRefreshToken) error +} diff --git a/service/internal/service/auth/svc.go b/service/internal/service/auth/svc.go new file mode 100644 index 0000000..1d59358 --- /dev/null +++ b/service/internal/service/auth/svc.go @@ -0,0 +1,315 @@ +package auth + +import ( + "context" + "gitea.privcloud.tech/mtusi2026/internal/pkg/ratelimit" + "gitea.privcloud.tech/mtusi2026/internal/pkg/telemetry" + "go.opentelemetry.io/otel/trace" + "time" + + "github.com/google/uuid" + "github.com/jackc/pgx/v5" + + "gitea.privcloud.tech/mtusi2026/internal/dto" + "gitea.privcloud.tech/mtusi2026/internal/pkg/apperr" + "gitea.privcloud.tech/mtusi2026/internal/pkg/auth/jwtprovider" + "gitea.privcloud.tech/mtusi2026/internal/pkg/encrypting" + "gitea.privcloud.tech/mtusi2026/internal/pkg/postgres/txmanager" + "gitea.privcloud.tech/mtusi2026/internal/pkg/types/role" +) + +var _ API = (*Auth)(nil) + +type Auth struct { + limiter *ratelimit.Limiter + jwt jwtprovider.Provider + tm *txmanager.TransactionManager + encryptor encrypting.Encryptor + + usersRepo userRepo + refreshTokensRepo refreshTokensRepo +} + +func New( + jwt jwtprovider.Provider, + tm *txmanager.TransactionManager, + encryptor encrypting.Encryptor, + usersRepo userRepo, + refreshTokensRepo refreshTokensRepo, +) *Auth { + return &Auth{ + jwt: jwt, + tm: tm, + encryptor: encryptor, + usersRepo: usersRepo, + refreshTokensRepo: refreshTokensRepo, + } +} + +func (a *Auth) SetLimiter(l *ratelimit.Limiter) { a.limiter = l } + +func (a *Auth) Login(ctx context.Context, req dto.Login) (*dto.LoginResult, error) { + ctx, span := telemetry.Start(ctx, "auth.Login", trace.SpanKindInternal) + defer span.End() + if a.limiter != nil { + for _, rule := range []struct { + scope, value string + limit int64 + }{{"ip", req.ClientIP, 60}, {"username", req.Username, 10}} { + allowed, _, err := a.limiter.Allow(ctx, rule.scope, rule.value, rule.limit) + if err != nil { + return nil, apperr.New(apperr.CodeUnavailable, "login protection unavailable") + } + if !allowed { + return nil, apperr.New(apperr.CodeTooManyRequests, "login rate limited") + } + } + } + if req.Username == "" || req.Password == "" { + return nil, apperr.New(apperr.CodeUnauthorized, "invalid credentials") + } + + credentials, err := a.usersRepo.GetCredentialsByUsername(ctx, req.Username) + if err != nil { + if apperr.CodeFromError(err) == apperr.CodeNotFound { + // A missing user still incurs the bounded password verification cost. + a.encryptor.CompareHashAndSecretUsingArgon2id(ctx, "$argon2id$v=19$m=65536,t=3,p=1$AAAAAAAAAAAAAAAAAAAAAA$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", req.Password) + return nil, apperr.New(apperr.CodeUnauthorized, "invalid credentials") + } + return nil, err + } + if err = a.encryptor.CompareHashAndSecretUsingArgon2id(ctx, credentials.PasswordHash, req.Password); err != nil { + if ctx.Err() != nil { + return nil, ctx.Err() + } + return nil, apperr.New(apperr.CodeUnauthorized, "invalid credentials") + } + + if encrypting.NeedsRehash(credentials.PasswordHash) { + replacement, e := a.encryptor.EncryptSecretUsingArgon2id(ctx, req.Password) + if e != nil { + return nil, e + } + if e = a.usersRepo.UpdatePasswordHash(ctx, credentials.User.ID, credentials.PasswordHash, replacement); e != nil { + return nil, e + } + } + + var ( + tokens *dto.Tokens + data *dto.UserDataInToken + ) + err = a.tm.RunInTransaction(ctx, nil, func(tx pgx.Tx) error { + tokens, err = a.jwt.CreateRefreshAndAccessTokensForUser(credentials.User.ID, role.User) + if err != nil { + return apperr.Wrap(err, apperr.CodeInternal, "failed to generate tokens") + } + data, err = a.verifyToken(tokens.RefreshToken.Token(), false) + if err != nil { + return apperr.Wrap(err, apperr.CodeInternal, "failed to validate generated refresh token") + } + var hash string + hash, err = a.encryptor.EncryptSecretUsingSum256(ctx, tokens.RefreshToken.Payload) + if err != nil { + return apperr.Wrap(err, apperr.CodeInternal, "failed to hash refresh token") + } + stored := dto.StoredRefreshToken{ + ID: data.TokenID, + UserID: data.UserID, + TokenHash: hash, + Role: data.Role, + ExpiresAt: data.ExpiresAt, + CreatedAt: data.IssuedAt, + } + if err = a.refreshTokensRepo.CreateTx(ctx, tx, stored); err != nil { + return err + } + return nil + }) + if err != nil { + return nil, err + } + + return &dto.LoginResult{ + ExpiresAt: data.ExpiresAt, + TokenType: "Bearer", + User: credentials.User, + Tokens: *tokens, + }, nil +} + +func (a *Auth) VerifyAccessToken(ctx context.Context, req dto.Token) (*dto.UserDataInToken, error) { + ctx, span := telemetry.Start(ctx, "auth.VerifyAccessToken", trace.SpanKindInternal) + defer span.End() + if err := ctx.Err(); err != nil { + return nil, err + } + return a.verifyToken(&req, true) +} + +func (a *Auth) VerifySession(ctx context.Context, token dto.Token) (*dto.UserDataInToken, error) { + ctx, span := telemetry.Start(ctx, "auth.VerifySession", trace.SpanKindInternal) + defer span.End() + if err := ctx.Err(); err != nil { + return nil, err + } + data, err := a.verifyTokenClaims(&token) + if err != nil { + return nil, err + } + if data.IsAccess { + return nil, apperr.New(apperr.CodeUnauthorized, "invalid session token") + } + + stored, err := a.refreshTokensRepo.GetByID(ctx, data.TokenID) + if err != nil { + if apperr.CodeFromError(err) == apperr.CodeNotFound { + return nil, apperr.New(apperr.CodeUnauthorized, "invalid session") + } + return nil, err + } + if err = a.validateStoredSession(ctx, token, data, stored); err != nil { + return nil, err + } + return data, nil +} + +func (a *Auth) Logout(ctx context.Context, req dto.RefreshToken) error { + ctx, span := telemetry.Start(ctx, "auth.Logout", trace.SpanKindInternal) + defer span.End() + data, err := a.verifyToken(req.Token(), false) + if err != nil { + return err + } + + return a.tm.RunInTransaction(ctx, nil, func(tx pgx.Tx) error { + stored, err := a.activeRefreshTokenTx(ctx, tx, req, data) + if err != nil { + return err + } + return a.refreshTokensRepo.RevokeTx(ctx, tx, dto.RevokeRefreshToken{ + ID: stored.ID, + RevokedAt: time.Now(), + }) + }) +} + +func (a *Auth) RefreshTokens(ctx context.Context, req dto.RefreshToken) (*dto.Tokens, error) { + ctx, span := telemetry.Start(ctx, "auth.RefreshTokens", trace.SpanKindInternal) + defer span.End() + data, err := a.verifyToken(req.Token(), false) + if err != nil { + return nil, err + } + + var tokens *dto.Tokens + err = a.tm.RunInTransaction(ctx, nil, func(tx pgx.Tx) error { + stored, err := a.activeRefreshTokenTx(ctx, tx, req, data) + if err != nil { + return err + } + if err = a.refreshTokensRepo.RevokeTx(ctx, tx, dto.RevokeRefreshToken{ + ID: stored.ID, + RevokedAt: time.Now(), + }); err != nil { + return err + } + tokens, err = a.createTokensTx(ctx, tx, stored.UserID, stored) + return err + }) + if err != nil { + return nil, err + } + return tokens, nil +} + +func (a *Auth) verifyToken(token *dto.Token, isAccess bool) (*dto.UserDataInToken, error) { + data, err := a.verifyTokenClaims(token) + if err != nil { + return nil, err + } + if data.IsAccess != isAccess || data.Role != role.User { + return nil, apperr.New(apperr.CodeUnauthorized, "invalid token") + } + return data, nil +} + +func (a *Auth) verifyTokenClaims(token *dto.Token) (*dto.UserDataInToken, error) { + data, err := a.jwt.ParseVerified(token) + if err != nil { + return nil, apperr.Wrap(err, apperr.CodeUnauthorized, "invalid token") + } + if err = data.Validate(); err != nil { + return nil, apperr.Wrap(err, apperr.CodeUnauthorized, "invalid token claims") + } + now := time.Now() + if data.IssuedAt.After(now) || !data.ExpiresAt.After(now) { + return nil, apperr.New(apperr.CodeUnauthorized, "invalid token") + } + return data, nil +} + +func (a *Auth) activeRefreshTokenTx(ctx context.Context, tx pgx.Tx, token dto.RefreshToken, data *dto.UserDataInToken) (*dto.StoredRefreshToken, error) { + ctx, span := telemetry.Start(ctx, "auth.activeRefreshTokenTx", trace.SpanKindInternal) + defer span.End() + stored, err := a.refreshTokensRepo.GetByIDForUpdateTx(ctx, tx, data.TokenID) + if err != nil { + if apperr.CodeFromError(err) == apperr.CodeNotFound { + return nil, apperr.New(apperr.CodeUnauthorized, "invalid refresh token") + } + return nil, err + } + + // Recheck expiry after acquiring the row lock: a concurrent rotation may have held it. + if err = a.validateStoredSession(ctx, dto.Token(token), data, stored); err != nil { + return nil, err + } + return stored, nil +} + +func (a *Auth) validateStoredSession(ctx context.Context, token dto.Token, data *dto.UserDataInToken, stored *dto.StoredRefreshToken) error { + ctx, span := telemetry.Start(ctx, "auth.validateStoredSession", trace.SpanKindInternal) + defer span.End() + now := time.Now() + if stored.RevokedAt != nil || !stored.ExpiresAt.After(now) || !data.ExpiresAt.After(now) || + stored.UserID != data.UserID || stored.Role != data.Role { + return apperr.New(apperr.CodeUnauthorized, "invalid session") + } + if err := a.encryptor.CompareHashAndSecretUsingSum256(ctx, stored.TokenHash, token.Payload); err != nil { + return apperr.New(apperr.CodeUnauthorized, "invalid session") + } + return nil +} + +func (a *Auth) createTokensTx(ctx context.Context, tx pgx.Tx, userID uuid.UUID, previous *dto.StoredRefreshToken) (*dto.Tokens, error) { + ctx, span := telemetry.Start(ctx, "auth.createTokensTx", trace.SpanKindInternal) + defer span.End() + tokens, err := a.jwt.CreateRefreshAndAccessTokensForUser(userID, role.User) + if err != nil { + return nil, apperr.Wrap(err, apperr.CodeInternal, "failed to generate tokens") + } + data, err := a.verifyToken(tokens.RefreshToken.Token(), false) + if err != nil { + return nil, apperr.Wrap(err, apperr.CodeInternal, "failed to validate generated refresh token") + } + hash, err := a.encryptor.EncryptSecretUsingSum256(ctx, tokens.RefreshToken.Payload) + if err != nil { + return nil, apperr.Wrap(err, apperr.CodeInternal, "failed to hash refresh token") + } + stored := dto.StoredRefreshToken{ + ID: data.TokenID, + UserID: data.UserID, + TokenHash: hash, + Role: data.Role, + ExpiresAt: data.ExpiresAt, + CreatedAt: data.IssuedAt, + } + if previous != nil { + stored.UserAgent = previous.UserAgent + stored.IPAddress = previous.IPAddress + } + if err = a.refreshTokensRepo.CreateTx(ctx, tx, stored); err != nil { + return nil, err + } + return tokens, nil +} diff --git a/service/internal/service/chat/api.go b/service/internal/service/chat/api.go new file mode 100644 index 0000000..cae5a2d --- /dev/null +++ b/service/internal/service/chat/api.go @@ -0,0 +1,15 @@ +package chat + +import ( + "context" + + "gitea.privcloud.tech/mtusi2026/internal/dto" +) + +type API interface { + CreateChat(ctx context.Context, req dto.CreateChat) (*dto.Chat, error) + GetForMember(ctx context.Context, req dto.ChatMember) (*dto.Chat, error) + ListForMember(ctx context.Context, req dto.ChatFilters) (*dto.Page[dto.Chat], error) + CreateMessage(ctx context.Context, req dto.CreateMessage) (*dto.Message, error) + ListForChat(ctx context.Context, req dto.MessageFilters) (*dto.Page[dto.Message], error) +} diff --git a/service/internal/service/chat/idempotency.go b/service/internal/service/chat/idempotency.go new file mode 100644 index 0000000..13fa962 --- /dev/null +++ b/service/internal/service/chat/idempotency.go @@ -0,0 +1,54 @@ +package chat + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/json" + "gitea.privcloud.tech/mtusi2026/internal/dto" + "gitea.privcloud.tech/mtusi2026/internal/pkg/apperr" + "github.com/google/uuid" + "github.com/jackc/pgx/v5" + "time" +) + +// CreateIdempotentTx performs the message use case inside an existing transaction. +func CreateIdempotentTx(ctx context.Context, tx pgx.Tx, repo messagesRepo, req dto.CreateMessage) (*dto.Message, error) { + scope := req.SenderID.String() + ":POST:" + req.ChatID.String() + ":" + req.IdempotencyKey + if err := repo.LockKeyTx(ctx, tx, scope); err != nil { + return nil, err + } + canonical := struct { + Text string `json:"text"` + AttachmentIDs []uuid.UUID `json:"attachment_ids"` + }{req.Text, append([]uuid.UUID{}, req.AttachmentIDs...)} + encoded, err := json.Marshal(canonical) + if err != nil { + return nil, err + } + hash := sha256.Sum256(encoded) + replay, err := repo.ReplayTx(ctx, tx, req) + if err != nil { + return nil, err + } + if replay != nil { + if replay.HashVersion == 1 && len(req.AttachmentIDs) == 0 { + hash = sha256.Sum256([]byte(req.Text)) + } else if replay.HashVersion != 2 { + return nil, apperr.New(apperr.CodeConflict, "idempotency key used with different content") + } + if !bytes.Equal(hash[:], replay.Hash) { + return nil, apperr.New(apperr.CodeConflict, "idempotency key used with different content") + } + return repo.MessageTx(ctx, tx, replay.MessageID) + } + now := time.Now().UTC().Truncate(time.Microsecond) + message := dto.Message{ID: uuid.New(), ChatID: req.ChatID, SenderID: req.SenderID, Text: req.Text, AttachmentIDs: append([]uuid.UUID{}, req.AttachmentIDs...), CreatedAt: now} + if err = repo.InsertMessageTx(ctx, tx, message); err != nil { + return nil, err + } + if err = repo.SaveKeyTx(ctx, tx, req, hash[:], message.ID, now.Add(24*time.Hour)); err != nil { + return nil, err + } + return &message, nil +} diff --git a/service/internal/service/chat/interface.go b/service/internal/service/chat/interface.go new file mode 100644 index 0000000..17c7742 --- /dev/null +++ b/service/internal/service/chat/interface.go @@ -0,0 +1,33 @@ +package chat + +import ( + "context" + "time" + + "github.com/google/uuid" + "github.com/jackc/pgx/v5" + + "gitea.privcloud.tech/mtusi2026/internal/dto" +) + +type chatsRepo interface { + CreateTx(ctx context.Context, tx pgx.Tx, req dto.Chat) error + GetForMember(ctx context.Context, req dto.ChatMember) (*dto.Chat, error) + ListForMember(ctx context.Context, userID uuid.UUID, page dto.PageQuery) ([]dto.Chat, error) +} + +type messagesRepo interface { + AttachmentsForMessageTx(ctx context.Context, tx pgx.Tx, ids []uuid.UUID) ([]dto.MessageAttachment, error) + LockKeyTx(context.Context, pgx.Tx, string) error + ReplayTx(context.Context, pgx.Tx, dto.CreateMessage) (*dto.MessageReplay, error) + MessageTx(context.Context, pgx.Tx, uuid.UUID) (*dto.Message, error) + InsertMessageTx(context.Context, pgx.Tx, dto.Message) error + SaveKeyTx(context.Context, pgx.Tx, dto.CreateMessage, []byte, uuid.UUID, time.Time) error + GetByIDForChat(ctx context.Context, chatID, messageID uuid.UUID) (*dto.Message, error) + ListForChat(ctx context.Context, chatID uuid.UUID, page dto.PageQuery) ([]dto.Message, error) +} + +type chatsCache interface { + Create(ctx context.Context, req dto.Chat) error + Get(ctx context.Context, chatID uuid.UUID) (*dto.Chat, error) +} diff --git a/service/internal/service/chat/svc.go b/service/internal/service/chat/svc.go new file mode 100644 index 0000000..ebe847a --- /dev/null +++ b/service/internal/service/chat/svc.go @@ -0,0 +1,303 @@ +package chat + +import ( + "bytes" + "context" + "errors" + "gitea.privcloud.tech/mtusi2026/internal/pkg/telemetry" + "go.opentelemetry.io/otel/trace" + "log/slog" + "slices" + "strings" + "time" + "unicode/utf8" + + "github.com/google/uuid" + "github.com/jackc/pgx/v5" + "github.com/redis/go-redis/v9" + "golang.org/x/sync/singleflight" + + "gitea.privcloud.tech/mtusi2026/internal/dto" + "gitea.privcloud.tech/mtusi2026/internal/pkg/apperr" + "gitea.privcloud.tech/mtusi2026/internal/pkg/postgres/txmanager" +) + +var _ API = (*Chat)(nil) + +const chatLoadTimeout = 2 * time.Second + +type Chat struct { + tm *txmanager.TransactionManager + chatsRepo chatsRepo + messagesRepo messagesRepo + chatsCache chatsCache + loads singleflight.Group +} + +func New(tm *txmanager.TransactionManager, chatsRepo chatsRepo, messagesRepo messagesRepo, chatsCache chatsCache) *Chat { + return &Chat{ + tm: tm, + chatsRepo: chatsRepo, + messagesRepo: messagesRepo, + chatsCache: chatsCache, + } +} + +func (c *Chat) CreateChat(ctx context.Context, req dto.CreateChat) (*dto.Chat, error) { + ctx, span := telemetry.Start(ctx, "chat.CreateChat", trace.SpanKindInternal) + defer span.End() + seen := make(map[uuid.UUID]struct{}, len(req.MemberIDs)) + for _, id := range req.MemberIDs { + if id == uuid.Nil { + return nil, apperr.New(apperr.CodeBadRequest, "invalid chat member") + } + if _, exists := seen[id]; exists { + return nil, apperr.New(apperr.CodeBadRequest, "duplicate chat member") + } + seen[id] = struct{}{} + } + + if _, included := seen[req.CreatorID]; !included { + return nil, apperr.New(apperr.CodeBadRequest, "creator must be a chat member") + } + + members := slices.Clone(req.MemberIDs) + slices.SortFunc(members, func(a, b uuid.UUID) int { + return bytes.Compare(a[:], b[:]) + }) + chat := dto.Chat{ + ID: uuid.New(), + Title: req.Title, + MemberIDs: members, + CreatedAt: time.Now().Truncate(time.Microsecond), + } + + if err := c.tm.RunInTransaction(ctx, nil, func(tx pgx.Tx) error { + return c.chatsRepo.CreateTx(ctx, tx, chat) + }); err != nil { + return nil, err + } + + return &chat, nil +} + +func (c *Chat) GetForMember(ctx context.Context, req dto.ChatMember) (*dto.Chat, error) { + ctx, span := telemetry.Start(ctx, "chat.GetForMember", trace.SpanKindInternal) + defer span.End() + if err := ctx.Err(); err != nil { + return nil, err + } + if chat, err := c.cachedChat(ctx, req.ChatID); err == nil { + return chatForMember(chat, req.UserID) + } + + // Repository results depend on membership: never share a user's 404 with + // another user requesting the same chat. + key := req.ChatID.String() + ":" + req.UserID.String() + result := c.loads.DoChan(key, func() (any, error) { + // One client disconnecting must not cancel the other clients' load. + loadCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), chatLoadTimeout) + defer cancel() + + // A previous load may have populated Redis after our initial miss. + if chat, err := c.cachedChat(loadCtx, req.ChatID); err == nil { + return chatForMember(chat, req.UserID) + } + + slog.InfoContext(loadCtx, "chat_db_lookup", "chat_id", req.ChatID) + chat, err := c.chatsRepo.GetForMember(loadCtx, req) + if err != nil { + return nil, err + } + if err = c.chatsCache.Create(loadCtx, *chat); err != nil { + slog.WarnContext(loadCtx, "chat_cache_write_failed", "chat_id", req.ChatID, "error", err) + } + return chat, nil + }) + + select { + case <-ctx.Done(): + return nil, ctx.Err() + case loaded := <-result: + if err := ctx.Err(); err != nil { + return nil, err + } + if loaded.Err != nil { + return nil, loaded.Err + } + return chatForMember(loaded.Val.(*dto.Chat), req.UserID) + } +} + +func (c *Chat) cachedChat(ctx context.Context, chatID uuid.UUID) (*dto.Chat, error) { + ctx, span := telemetry.Start(ctx, "chat.cachedChat", trace.SpanKindInternal) + defer span.End() + chat, err := c.chatsCache.Get(ctx, chatID) + switch { + case err == nil: + slog.DebugContext(ctx, "chat_cache_hit", "chat_id", chatID) + case errors.Is(err, redis.Nil): + slog.DebugContext(ctx, "chat_cache_miss", "chat_id", chatID) + default: + slog.WarnContext(ctx, "chat_cache_read_error", "chat_id", chatID, "error", err) + } + return chat, err +} + +func chatForMember(chat *dto.Chat, userID uuid.UUID) (*dto.Chat, error) { + if !slices.Contains(chat.MemberIDs, userID) { + return nil, apperr.New(apperr.CodeNotFound, "chat not found") + } + return chat, nil +} + +func (c *Chat) ListForMember(ctx context.Context, req dto.ChatFilters) (*dto.Page[dto.Chat], error) { + ctx, span := telemetry.Start(ctx, "chat.ListForMember", trace.SpanKindInternal) + defer span.End() + limit, err := pageLimit(req.Limit) + if err != nil { + return nil, err + } + + query := dto.PageQuery{Limit: limit + 1} + if req.AfterID != nil { + cursor, err := c.GetForMember(ctx, dto.ChatMember{UserID: req.UserID, ChatID: *req.AfterID}) + if err != nil { + return nil, cursorError(err) + } + query.After = &dto.PagePosition{CreatedAt: cursor.CreatedAt, ID: cursor.ID} + } + + items, err := c.chatsRepo.ListForMember(ctx, req.UserID, query) + if err != nil { + return nil, err + } + + return makePage(items, limit, func(chat dto.Chat) uuid.UUID { return chat.ID }), nil +} + +func (c *Chat) CreateMessage(ctx context.Context, req dto.CreateMessage) (*dto.Message, error) { + ctx, span := telemetry.Start(ctx, "chat.CreateMessage", trace.SpanKindInternal) + defer span.End() + if _, err := c.GetForMember(ctx, dto.ChatMember{UserID: req.SenderID, ChatID: req.ChatID}); err != nil { + return nil, err + } + + if err := validateMessage(req); err != nil { + return nil, err + } + req.AttachmentIDs = append([]uuid.UUID{}, req.AttachmentIDs...) + + var message *dto.Message + err := c.tm.RunInTransaction(ctx, nil, func(tx pgx.Tx) error { + if len(req.AttachmentIDs) > 0 { + attachments, err := c.messagesRepo.AttachmentsForMessageTx(ctx, tx, req.AttachmentIDs) + if err != nil { + return err + } + if len(attachments) != len(req.AttachmentIDs) { + return apperr.New(apperr.CodeNotFound, "attachment not found") + } + for _, attachment := range attachments { + if attachment.ChatID != req.ChatID { + return apperr.New(apperr.CodeNotFound, "attachment not found") + } + if attachment.Status == "failed" { + return apperr.New(apperr.CodeConflict, "failed attachment") + } + if attachment.Missing { + return apperr.New(apperr.CodeUnavailable, "attachment storage unavailable") + } + } + } + + var err error + message, err = CreateIdempotentTx(ctx, tx, c.messagesRepo, req) + return err + }) + return message, err +} + +func (c *Chat) ListForChat(ctx context.Context, req dto.MessageFilters) (*dto.Page[dto.Message], error) { + ctx, span := telemetry.Start(ctx, "chat.ListForChat", trace.SpanKindInternal) + defer span.End() + if _, err := c.GetForMember(ctx, dto.ChatMember{UserID: req.UserID, ChatID: req.ChatID}); err != nil { + return nil, err + } + + limit, err := pageLimit(req.Limit) + if err != nil { + return nil, err + } + + query := dto.PageQuery{Limit: limit + 1} + if req.AfterID != nil { + cursor, err := c.messagesRepo.GetByIDForChat(ctx, req.ChatID, *req.AfterID) + if err != nil { + return nil, cursorError(err) + } + query.After = &dto.PagePosition{CreatedAt: cursor.CreatedAt, ID: cursor.ID} + } + + items, err := c.messagesRepo.ListForChat(ctx, req.ChatID, query) + if err != nil { + return nil, err + } + + return makePage(items, limit, func(message dto.Message) uuid.UUID { return message.ID }), nil +} + +func pageLimit(limit *int) (int, error) { + size := 50 + if limit != nil { + size = *limit + } + if size < 1 || size > 100 { + return 0, apperr.New(apperr.CodeBadRequest, "invalid limit") + } + return size, nil +} + +func cursorError(err error) error { + if apperr.CodeFromError(err) == apperr.CodeNotFound { + return apperr.New(apperr.CodeBadRequest, "invalid cursor") + } + return err +} + +func makePage[T any](items []T, limit int, id func(T) uuid.UUID) *dto.Page[T] { + if items == nil { + items = make([]T, 0) + } + page := dto.Page[T]{Items: items} + if len(items) > limit { + page.Items = items[:limit] + cursor := id(page.Items[limit-1]) + page.NextCursorID = &cursor + } + return &page +} + +// Content rules belong to the service so non-HTTP callers cannot bypass them. +func validateMessage(req dto.CreateMessage) error { + if !utf8.ValidString(req.Text) || strings.ContainsRune(req.Text, '\x00') || utf8.RuneCountInString(req.Text) > 4000 { + return apperr.New(apperr.CodeBadRequest, "invalid message text") + } + if len(req.AttachmentIDs) > 10 { + return apperr.New(apperr.CodeBadRequest, "too many attachments") + } + seen := make(map[uuid.UUID]struct{}, len(req.AttachmentIDs)) + for _, id := range req.AttachmentIDs { + if id == uuid.Nil { + return apperr.New(apperr.CodeBadRequest, "invalid attachment id") + } + if _, ok := seen[id]; ok { + return apperr.New(apperr.CodeBadRequest, "duplicate attachment id") + } + seen[id] = struct{}{} + } + if strings.TrimSpace(req.Text) == "" && len(req.AttachmentIDs) == 0 { + return apperr.New(apperr.CodeBadRequest, "message requires text or an attachment") + } + return nil +} diff --git a/service/internal/service/observation/service.go b/service/internal/service/observation/service.go new file mode 100644 index 0000000..98790b8 --- /dev/null +++ b/service/internal/service/observation/service.go @@ -0,0 +1,64 @@ +// Package observation evaluates the lab SLO; storage only persists facts and decisions. +package observation + +import ( + "context" + "github.com/google/uuid" + "log/slog" + "time" +) + +type Sample struct { + ID uuid.UUID + Accepted, Deadline time.Time + Ready *time.Time + Failed *time.Time +} +type Snapshot struct { + Queued, Outbox int64 + Locked int64 + Oldest float64 + Good, Bad int64 +} +type Repository interface { + Pending(context.Context) ([]Sample, error) + Decide(context.Context, uuid.UUID, string, time.Time) error + Snapshot(context.Context) (Snapshot, error) +} +type Service struct { + Repo Repository + Publish func(Snapshot) +} + +func (s *Service) Observe(ctx context.Context) error { + rows, e := s.Repo.Pending(ctx) + if e != nil { + return e + } + now := time.Now().UTC() + for _, r := range rows { + outcome := "" + if r.Failed != nil { + outcome = "bad" + } else if r.Ready != nil { + outcome = "bad" + if !r.Ready.After(r.Deadline) { + outcome = "good" + } + } else if !now.Before(r.Deadline) { + outcome = "bad" + } + if outcome != "" { + if e = s.Repo.Decide(ctx, r.ID, outcome, now); e != nil { + return e + } + } + } + state, e := s.Repo.Snapshot(ctx) + if e != nil { + return e + } + s.Publish(state) + slog.DebugContext(ctx, "observation_updated") + return nil +} diff --git a/service/internal/service/outbox/api.go b/service/internal/service/outbox/api.go new file mode 100644 index 0000000..aaac1a7 --- /dev/null +++ b/service/internal/service/outbox/api.go @@ -0,0 +1,12 @@ +package outbox + +import ( + "context" + + "github.com/jackc/pgx/v5" +) + +type API interface { + CreateTx(context.Context, pgx.Tx, CreateMessageReq) (CreateMessageRes, error) + Create(context.Context, CreateMessageReq) (CreateMessageRes, error) +} diff --git a/service/internal/service/outbox/models.go b/service/internal/service/outbox/models.go new file mode 100644 index 0000000..fc90e0e --- /dev/null +++ b/service/internal/service/outbox/models.go @@ -0,0 +1,115 @@ +package outbox + +import ( + "time" + + "github.com/google/uuid" +) + +type CreateMessageReq struct { + ID uuid.UUID + CreatedAt time.Time + Transport Transport + AggregateType string + AggregateID uuid.UUID + MessageType string + IdempotencyKey *string + SchemaVersion int + CausationID *uuid.UUID + Metadata map[string]string + Payload []byte + MaxAttempts int + Status Status +} + +type CreateMessageRes struct { + ID uuid.UUID + CreatedAt time.Time +} + +type MarkPublishedMessageReq struct { + ID uuid.UUID + UpdatedAt time.Time + PublishedAt time.Time +} + +type MarkFailedMessageReq struct { + ID uuid.UUID + UpdatedAt time.Time + NextAttemptAt time.Time + LastError string +} + +type MarkDeadLetterMessageReq struct { + ID uuid.UUID + UpdatedAt time.Time + DeadLetterAt time.Time + LastError string +} + +type AcquireMessagesForProcessingReq struct { + Limit uint + LockDuration time.Duration +} + +type AcquiredMessage struct { + Token uuid.UUID + ID uuid.UUID + Transport Transport + AggregateType string + AggregateID uuid.UUID + MessageType string + SchemaVersion int + CorrelationID uuid.UUID + CausationID *uuid.UUID + Status Status + Metadata map[string]string + Payload []byte + Attempts int + MaxAttempts int + LastError *string + LockedAt *time.Time + LockedUntil *time.Time + NextAttemptAt *time.Time + PublishedAt *time.Time + DeadLetterAt *time.Time + CreatedAt time.Time + UpdatedAt *time.Time +} + +type RecoverStuckMessagesAsFailedReq struct { + UpdatedAt time.Time + NextAttemptAt time.Time + LastError string + Limit uint +} + +type RecoverStuckMessagesAsDeadLetterReq struct { + UpdatedAt time.Time + DeadLetterAt time.Time + LastError string + Limit uint +} + +type DeliveryMessage struct { + ID uuid.UUID + Transport Transport + AggregateType string + AggregateID uuid.UUID + MessageType string + SchemaVersion int + CorrelationID uuid.UUID + CausationID *uuid.UUID + Payload []byte + Metadata map[string]string +} + +type ProcessOnceResult struct { + RecoveredAsFailed int64 + RecoveredAsDeadLetter int64 + + Acquired int + Published int + Failed int + DeadLettered int +} diff --git a/service/internal/service/outbox/status.go b/service/internal/service/outbox/status.go new file mode 100644 index 0000000..c7a1d3a --- /dev/null +++ b/service/internal/service/outbox/status.go @@ -0,0 +1,16 @@ +package outbox + +const ( + StatusCreated Status = "created" + StatusPending Status = "pending" + StatusProcessing Status = "processing" + StatusPublished Status = "published" + StatusFailed Status = "failed" + StatusDeadLetter Status = "dead_letter" +) + +type Status string + +func (s Status) String() string { + return string(s) +} diff --git a/service/internal/service/outbox/svc.go b/service/internal/service/outbox/svc.go new file mode 100644 index 0000000..7dd7faa --- /dev/null +++ b/service/internal/service/outbox/svc.go @@ -0,0 +1,114 @@ +// Adapted from mamont-optics/application/internal/service/outbox. +// Retains transactional creation, batch acquisition, bounded retries and lease recovery. +package outbox + +import ( + "context" + "encoding/json" + "errors" + "github.com/google/uuid" + "github.com/jackc/pgx/v5" + "math/rand/v2" + "time" +) + +type Repository interface { + CreateTx(context.Context, pgx.Tx, CreateMessageReq) (CreateMessageRes, error) + AcquireManyForProcessing(context.Context, AcquireMessagesForProcessingReq) ([]AcquiredMessage, error) + Expired(context.Context) ([]AcquiredMessage, error) + Update(context.Context, AcquiredMessage, Status, time.Time, string) error +} +type TransportPublisher interface { + Publish(context.Context, DeliveryMessage) error +} +type TransactionManager interface { + RunInTransaction(context.Context, *pgx.TxOptions, func(pgx.Tx) error) error +} +type Svc struct { + repo Repository + tm TransactionManager + publisher TransportPublisher + OnDead func(context.Context, uuid.UUID) error +} + +func New(repo Repository, tm TransactionManager, publisher TransportPublisher) *Svc { + return &Svc{repo: repo, tm: tm, publisher: publisher} +} +func (s *Svc) CreateTx(ctx context.Context, tx pgx.Tx, r CreateMessageReq) (CreateMessageRes, error) { + if r.ID == uuid.Nil { + r.ID = uuid.New() + } + if r.CreatedAt.IsZero() { + r.CreatedAt = time.Now().UTC() + } + if r.Status == "" { + r.Status = StatusPending + } + if r.MaxAttempts == 0 { + r.MaxAttempts = 5 + } + return s.repo.CreateTx(ctx, tx, r) +} +func (s *Svc) Create(ctx context.Context, r CreateMessageReq) (res CreateMessageRes, err error) { + err = s.tm.RunInTransaction(ctx, nil, func(tx pgx.Tx) error { var e error; res, e = s.CreateTx(ctx, tx, r); return e }) + return +} +func Backoff(attempt int) time.Duration { + n := min(max(attempt-1, 0), 5) + cap := min(time.Second*time.Duration(1<= m.MaxAttempts { + status = StatusDeadLetter + } + if err := s.repo.Update(ctx, m, status, time.Now().UTC().Add(Backoff(m.Attempts)), code); err != nil { + return err + } + if status == StatusDeadLetter && s.OnDead != nil { + var j struct { + JobID uuid.UUID `json:"job_id"` + } + if err := json.Unmarshal(m.Payload, &j); err != nil { + return err + } + return s.OnDead(ctx, j.JobID) + } + return nil +} diff --git a/service/internal/service/outbox/transport.go b/service/internal/service/outbox/transport.go new file mode 100644 index 0000000..a92608c --- /dev/null +++ b/service/internal/service/outbox/transport.go @@ -0,0 +1,7 @@ +package outbox + +type Transport string + +const TransportAMQP Transport = "amqp" + +func (t Transport) String() string { return string(t) } diff --git a/service/internal/service/user/api.go b/service/internal/service/user/api.go new file mode 100644 index 0000000..65329cf --- /dev/null +++ b/service/internal/service/user/api.go @@ -0,0 +1,15 @@ +package user + +import ( + "context" + + "github.com/google/uuid" + + "gitea.privcloud.tech/mtusi2026/internal/dto" +) + +type API interface { + Create(ctx context.Context, req dto.CreateUser) (*dto.User, error) + GetByID(ctx context.Context, id uuid.UUID) (*dto.User, error) + GetByUsername(ctx context.Context, username string) (*dto.User, error) +} diff --git a/service/internal/service/user/interface.go b/service/internal/service/user/interface.go new file mode 100644 index 0000000..4eb78b5 --- /dev/null +++ b/service/internal/service/user/interface.go @@ -0,0 +1,15 @@ +package user + +import ( + "context" + + "github.com/google/uuid" + + "gitea.privcloud.tech/mtusi2026/internal/dto" +) + +type userRepo interface { + Create(ctx context.Context, req dto.CreateStoredUser) error + GetByID(ctx context.Context, id uuid.UUID) (*dto.User, error) + GetByUsername(ctx context.Context, username string) (*dto.User, error) +} diff --git a/service/internal/service/user/svc.go b/service/internal/service/user/svc.go new file mode 100644 index 0000000..966329f --- /dev/null +++ b/service/internal/service/user/svc.go @@ -0,0 +1,75 @@ +package user + +import ( + "context" + "gitea.privcloud.tech/mtusi2026/internal/pkg/telemetry" + "go.opentelemetry.io/otel/trace" + "time" + + "github.com/google/uuid" + + "gitea.privcloud.tech/mtusi2026/internal/dto" + "gitea.privcloud.tech/mtusi2026/internal/pkg/apperr" + "gitea.privcloud.tech/mtusi2026/internal/pkg/encrypting" +) + +var ( + _ API = (*User)(nil) +) + +type User struct { + encryptor encrypting.Encryptor + usersRepo userRepo +} + +func New(encryptor encrypting.Encryptor, usersRepo userRepo) *User { + return &User{encryptor: encryptor, usersRepo: usersRepo} +} + +func (u *User) Create(ctx context.Context, req dto.CreateUser) (*dto.User, error) { + ctx, span := telemetry.Start(ctx, "user.Create", trace.SpanKindInternal) + defer span.End() + hash, err := u.encryptor.EncryptSecretUsingArgon2id(ctx, req.Password) + if err != nil { + return nil, apperr.Wrap(err, apperr.CodeInternal, "failed to hash password") + } + + user := dto.User{ + ID: uuid.New(), + Username: req.Username, + DisplayName: req.DisplayName, + CreatedAt: time.Now().Truncate(time.Microsecond), + } + if err = u.usersRepo.Create(ctx, dto.CreateStoredUser{ + ID: user.ID, + Username: user.Username, + DisplayName: user.DisplayName, + PasswordHash: hash, + CreatedAt: user.CreatedAt, + UpdatedAt: user.CreatedAt, + }); err != nil { + return nil, err + } + + return &user, nil +} + +func (u *User) GetByID(ctx context.Context, id uuid.UUID) (*dto.User, error) { + ctx, span := telemetry.Start(ctx, "user.GetByID", trace.SpanKindInternal) + defer span.End() + user, err := u.usersRepo.GetByID(ctx, id) + if err != nil { + return nil, err + } + return user, nil +} + +func (u *User) GetByUsername(ctx context.Context, username string) (*dto.User, error) { + ctx, span := telemetry.Start(ctx, "user.GetByUsername", trace.SpanKindInternal) + defer span.End() + user, err := u.usersRepo.GetByUsername(ctx, username) + if err != nil { + return nil, err + } + return user, nil +} diff --git a/service/migrations/001_create_users_table.sql b/service/migrations/001_create_users_table.sql new file mode 100644 index 0000000..60f730a --- /dev/null +++ b/service/migrations/001_create_users_table.sql @@ -0,0 +1,18 @@ +CREATE TABLE IF NOT EXISTS users +( + id UUID PRIMARY KEY, + name VARCHAR(100) NOT NULL, + username VARCHAR(32) NOT NULL, + password_hash TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + + CONSTRAINT users_username_unique UNIQUE (username), + CONSTRAINT users_username_format CHECK (username ~ '^[a-z0-9_]{3,32}$'), + CONSTRAINT users_name_not_blank CHECK (name !~ '^[[:space:]]*$'), + CONSTRAINT users_password_hash_not_empty CHECK (password_hash <> '') +); + +---- create above / drop below ---- + +DROP TABLE IF EXISTS users; diff --git a/service/migrations/002_create_refresh_tokens_table.sql b/service/migrations/002_create_refresh_tokens_table.sql new file mode 100644 index 0000000..cbc7308 --- /dev/null +++ b/service/migrations/002_create_refresh_tokens_table.sql @@ -0,0 +1,19 @@ +CREATE TABLE IF NOT EXISTS refresh_tokens +( + id UUID PRIMARY KEY, + user_id UUID NOT NULL REFERENCES users (id), + token_hash TEXT NOT NULL, + user_agent TEXT, + ip_address INET, + role VARCHAR(16) NOT NULL, + expires_at TIMESTAMPTZ NOT NULL, + revoked_at TIMESTAMPTZ, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW() +); + +CREATE INDEX IF NOT EXISTS idx_refresh_user_id ON refresh_tokens (user_id); +CREATE INDEX IF NOT EXISTS idx_refresh_expires ON refresh_tokens (expires_at); + +---- create above / drop below ---- + +DROP TABLE IF EXISTS refresh_tokens; \ No newline at end of file diff --git a/service/migrations/003_create_chats_table.sql b/service/migrations/003_create_chats_table.sql new file mode 100644 index 0000000..b45df91 --- /dev/null +++ b/service/migrations/003_create_chats_table.sql @@ -0,0 +1,14 @@ +CREATE TABLE IF NOT EXISTS chats +( + id UUID PRIMARY KEY, + title VARCHAR(100) NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + + CONSTRAINT chats_title_not_blank CHECK (title !~ '^[[:space:]]*$') +); + +CREATE INDEX IF NOT EXISTS idx_chats_created_at_id ON chats (created_at, id); + +---- create above / drop below ---- + +DROP TABLE IF EXISTS chats; diff --git a/service/migrations/004_create_chat_members_table.sql b/service/migrations/004_create_chat_members_table.sql new file mode 100644 index 0000000..6413198 --- /dev/null +++ b/service/migrations/004_create_chat_members_table.sql @@ -0,0 +1,15 @@ +CREATE TABLE IF NOT EXISTS chat_members +( + chat_id UUID NOT NULL, + user_id UUID NOT NULL, + + PRIMARY KEY (chat_id, user_id), + CONSTRAINT chat_members_chat_fk FOREIGN KEY (chat_id) REFERENCES chats (id) ON DELETE NO ACTION, + CONSTRAINT chat_members_user_fk FOREIGN KEY (user_id) REFERENCES users (id) ON DELETE NO ACTION +); + +CREATE INDEX IF NOT EXISTS idx_chat_members_user_id_chat_id ON chat_members (user_id, chat_id); + +---- create above / drop below ---- + +DROP TABLE IF EXISTS chat_members; diff --git a/service/migrations/005_create_messages_table.sql b/service/migrations/005_create_messages_table.sql new file mode 100644 index 0000000..1ab0d7d --- /dev/null +++ b/service/migrations/005_create_messages_table.sql @@ -0,0 +1,18 @@ +CREATE TABLE IF NOT EXISTS messages +( + id UUID PRIMARY KEY, + chat_id UUID NOT NULL, + sender_id UUID NOT NULL, + text VARCHAR(4000) NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + + CONSTRAINT messages_text_not_blank CHECK (text !~ '^[[:space:]]*$'), + CONSTRAINT messages_sender_member_fk FOREIGN KEY (chat_id, sender_id) + REFERENCES chat_members (chat_id, user_id) ON DELETE NO ACTION +); + +CREATE INDEX IF NOT EXISTS idx_messages_chat_id_created_at_id ON messages (chat_id, created_at, id); + +---- create above / drop below ---- + +DROP TABLE IF EXISTS messages; diff --git a/service/migrations/006_message_idempotency.sql b/service/migrations/006_message_idempotency.sql new file mode 100644 index 0000000..8e48e9b --- /dev/null +++ b/service/migrations/006_message_idempotency.sql @@ -0,0 +1,14 @@ +CREATE TABLE message_idempotency ( + user_id UUID NOT NULL REFERENCES users(id), + method TEXT NOT NULL CHECK (method = 'POST'), + chat_id UUID NOT NULL REFERENCES chats(id), + key TEXT NOT NULL CHECK (key ~ '^[A-Za-z0-9._:-]{1,128}$'), + request_hash BYTEA NOT NULL, + message_id UUID NOT NULL REFERENCES messages(id), + expires_at TIMESTAMPTZ NOT NULL, + PRIMARY KEY (user_id, method, chat_id, key) +); +CREATE INDEX message_idempotency_expiry ON message_idempotency(expires_at); + +---- create above / drop below ---- +DROP TABLE message_idempotency; diff --git a/service/migrations/007_attachments.sql b/service/migrations/007_attachments.sql new file mode 100644 index 0000000..050c686 --- /dev/null +++ b/service/migrations/007_attachments.sql @@ -0,0 +1,44 @@ +CREATE TABLE attachments +( + id UUID PRIMARY KEY, + chat_id UUID NOT NULL REFERENCES chats (id), + owner_id UUID NOT NULL, + filename TEXT NOT NULL, + object_key TEXT NOT NULL UNIQUE, + content_type TEXT NOT NULL CHECK (content_type IN ('image/png', 'image/jpeg')), + size_bytes BIGINT NOT NULL CHECK (size_bytes BETWEEN 1 AND 10485760), + sha256 TEXT NOT NULL CHECK (sha256 ~ '^[a-f0-9]{64}$'), + integrity_missing BOOLEAN NOT NULL DEFAULT false, + created_at TIMESTAMPTZ NOT NULL, + updated_at TIMESTAMPTZ NOT NULL, + UNIQUE (id, chat_id), + FOREIGN KEY (chat_id, owner_id) REFERENCES chat_members (chat_id, user_id) +); +CREATE INDEX attachments_scan ON attachments (id); +ALTER TABLE messages + DROP CONSTRAINT messages_text_not_blank; +ALTER TABLE messages + ADD CONSTRAINT messages_id_chat UNIQUE (id, chat_id); +CREATE TABLE message_attachments +( + message_id UUID NOT NULL, + chat_id UUID NOT NULL, + attachment_id UUID NOT NULL, + position SMALLINT NOT NULL CHECK (position BETWEEN 0 AND 9), + PRIMARY KEY (message_id, attachment_id), + UNIQUE (message_id, position), + FOREIGN KEY (message_id, chat_id) REFERENCES messages (id, chat_id), + FOREIGN KEY (attachment_id, chat_id) REFERENCES attachments (id, chat_id) +); +ALTER TABLE message_idempotency + ADD COLUMN hash_version SMALLINT NOT NULL DEFAULT 1; + +---- create above / drop below ---- +ALTER TABLE message_idempotency + DROP COLUMN hash_version; +DROP TABLE message_attachments; +ALTER TABLE messages + DROP CONSTRAINT messages_id_chat; +ALTER TABLE messages + ADD CONSTRAINT messages_text_not_blank CHECK (text !~ '^[[:space:]]*$'); +DROP TABLE attachments; diff --git a/service/migrations/008_async_pipeline.sql b/service/migrations/008_async_pipeline.sql new file mode 100644 index 0000000..4607e20 --- /dev/null +++ b/service/migrations/008_async_pipeline.sql @@ -0,0 +1,41 @@ +ALTER TABLE attachments + ADD COLUMN status text NOT NULL DEFAULT 'queued' CHECK(status IN ('queued','processing','ready','failed')), + ADD COLUMN error_code text, + ADD COLUMN pipeline_version integer NOT NULL DEFAULT 1, + ADD COLUMN staging_key text NOT NULL DEFAULT '', + ADD COLUMN variants jsonb NOT NULL DEFAULT '[]', + ADD COLUMN variant_keys jsonb NOT NULL DEFAULT '{}'; +UPDATE attachments SET staging_key=object_key; +CREATE TABLE attachment_jobs ( + id uuid PRIMARY KEY, attachment_id uuid NOT NULL REFERENCES attachments(id), + pipeline_version integer NOT NULL, attempt integer NOT NULL CHECK(attempt>0), + attempt_limit integer NOT NULL, state text NOT NULL, + token uuid NOT NULL, lease_until timestamptz, next_attempt_at timestamptz NOT NULL, + error_code text, created_at timestamptz NOT NULL, updated_at timestamptz NOT NULL, + UNIQUE(attachment_id,pipeline_version) +); +CREATE INDEX jobs_recovery ON attachment_jobs(state,lease_until); +CREATE TABLE outbox ( + id uuid PRIMARY KEY, job_id uuid NOT NULL REFERENCES attachment_jobs(id), + payload jsonb NOT NULL, status text NOT NULL, + attempts integer NOT NULL DEFAULT 0, max_attempts integer NOT NULL, + token uuid NOT NULL, locked_until timestamptz, next_attempt_at timestamptz NOT NULL, + last_error text, created_at timestamptz NOT NULL, updated_at timestamptz NOT NULL +); +CREATE INDEX outbox_delivery ON outbox(status,next_attempt_at); +CREATE TABLE job_audit ( + id bigserial PRIMARY KEY, job_id uuid NOT NULL REFERENCES attachment_jobs(id), + action text NOT NULL, created_at timestamptz NOT NULL DEFAULT clock_timestamp() +); +-- Existing lab4 originals become durable sources for the first asynchronous run. +INSERT INTO attachment_jobs + SELECT gen_random_uuid(),id,1,1,5,'queued',gen_random_uuid(),NULL,clock_timestamp(),NULL,created_at,clock_timestamp() FROM attachments; +INSERT INTO outbox + SELECT gen_random_uuid(),id,jsonb_build_object('schema_version',1,'job_id',id,'type','attachment.process.v1','attachment_id',attachment_id,'pipeline_version',1,'attempt',1,'created_at',created_at), + 'pending',0,5,gen_random_uuid(),NULL,clock_timestamp(),NULL,clock_timestamp(),clock_timestamp() FROM attachment_jobs; + +---- create above / drop below ---- +DROP TABLE job_audit; +DROP TABLE outbox; +DROP TABLE attachment_jobs; +ALTER TABLE attachments DROP COLUMN status,DROP COLUMN error_code,DROP COLUMN pipeline_version,DROP COLUMN staging_key,DROP COLUMN variants,DROP COLUMN variant_keys; diff --git a/service/migrations/009_variant_manifest.sql b/service/migrations/009_variant_manifest.sql new file mode 100644 index 0000000..84413c8 --- /dev/null +++ b/service/migrations/009_variant_manifest.sql @@ -0,0 +1,8 @@ +CREATE TABLE attachment_variants ( + attachment_id uuid NOT NULL REFERENCES attachments(id), pipeline_version integer NOT NULL, + variant text NOT NULL CHECK(variant IN ('original','thumbnail','crop','optimized')), + object_key text NOT NULL UNIQUE, sha256 text NOT NULL, size_bytes bigint NOT NULL, + PRIMARY KEY(attachment_id,pipeline_version,variant) +); +---- create above / drop below ---- +DROP TABLE attachment_variants; diff --git a/service/migrations/010_observability.sql b/service/migrations/010_observability.sql new file mode 100644 index 0000000..f6ac457 --- /dev/null +++ b/service/migrations/010_observability.sql @@ -0,0 +1,10 @@ +ALTER TABLE attachment_jobs ADD COLUMN traceparent text NOT NULL DEFAULT '', ADD COLUMN tracestate text NOT NULL DEFAULT ''; +CREATE TABLE job_slo ( + job_id uuid PRIMARY KEY REFERENCES attachment_jobs(id) DEFERRABLE INITIALLY DEFERRED, + accepted_at timestamptz NOT NULL, deadline_at timestamptz NOT NULL, + ready_at timestamptz, outcome text CHECK(outcome IN ('good','bad')), evaluated_at timestamptz +); +CREATE INDEX job_slo_pending ON job_slo(deadline_at) WHERE outcome IS NULL; +---- create above / drop below ---- +DROP TABLE job_slo; +ALTER TABLE attachment_jobs DROP COLUMN traceparent, DROP COLUMN tracestate; diff --git a/service/migrations/011_terminal_slo.sql b/service/migrations/011_terminal_slo.sql new file mode 100644 index 0000000..d7f25d8 --- /dev/null +++ b/service/migrations/011_terminal_slo.sql @@ -0,0 +1,3 @@ +ALTER TABLE job_slo ADD COLUMN failed_at timestamptz; +---- create above / drop below ---- +ALTER TABLE job_slo DROP COLUMN failed_at; diff --git a/service/migrations/embed.go b/service/migrations/embed.go new file mode 100644 index 0000000..eb05421 --- /dev/null +++ b/service/migrations/embed.go @@ -0,0 +1,8 @@ +package migrations + +import ( + "embed" +) + +//go:embed *.sql +var Migrations embed.FS diff --git a/service/tests/api_test.go b/service/tests/api_test.go new file mode 100644 index 0000000..720cea8 --- /dev/null +++ b/service/tests/api_test.go @@ -0,0 +1,285 @@ +package tests + +import ( + "encoding/json" + "fmt" + "github.com/google/uuid" + "io" + "net/http" + "net/url" + "os" + "strings" + "testing" + "time" +) + +type apiClient struct { + baseURL string + prefix string + tokens map[string]string +} + +type apiResponse struct { + status int + requestID string + instanceID string + body []byte +} + +func newAPI(t *testing.T) *apiClient { + t.Helper() + base := os.Getenv("TEST_BASE_URL") + if base == "" { + base = "http://localhost:18086" + } + return &apiClient{baseURL: strings.TrimRight(base, "/"), prefix: uuid.NewString()[:8] + "_", tokens: map[string]string{}} +} +func (api *apiClient) request(t *testing.T, method, path, userID, body string) apiResponse { + return api.requestKey(t, method, path, userID, body, uuid.NewString()) +} +func (api *apiClient) requestKey(t *testing.T, method, path, userID, body, key string) apiResponse { + t.Helper() + response, err := api.send(method, path, userID, body, key) + if err != nil { + t.Fatal(err) + } + return response +} +func (api *apiClient) send(method, path, userID, body, key string) (apiResponse, error) { + if path == "/api/v1/users" { + var fields map[string]any + if json.Unmarshal([]byte(body), &fields) == nil { + if name, ok := fields["username"].(string); ok { + fields["username"] = api.prefix + name + data, err := json.Marshal(fields) + if err != nil { + return apiResponse{}, err + } + body = string(data) + } + } + } + req, err := http.NewRequest(method, api.baseURL+path, strings.NewReader(body)) + if err != nil { + return apiResponse{}, err + } + if body != "" { + req.Header.Set("Content-Type", "application/json") + } + if userID != "" { + req.Header.Set("Authorization", "Bearer "+api.tokens[userID]) + } + if key != "" { + req.Header.Set("Idempotency-Key", key) + } + response, err := (&http.Client{Timeout: 5 * time.Second}).Do(req) + if err != nil { + return apiResponse{}, fmt.Errorf("running lab5 stand required: %w", err) + } + defer response.Body.Close() + data, err := io.ReadAll(response.Body) + if err != nil { + return apiResponse{}, err + } + requestID, instanceID := response.Header.Get("X-Request-Id"), response.Header.Get("X-Instance-Id") + if requestID == "" || instanceID == "" { + return apiResponse{}, fmt.Errorf("%s %s: HTTP %d missing response identity", method, path, response.StatusCode) + } + return apiResponse{status: response.StatusCode, requestID: requestID, instanceID: instanceID, body: data}, nil +} + +func jsonString(t *testing.T, value any) string { + t.Helper() + data, err := json.Marshal(value) + if err != nil { + t.Fatalf("encode request: %v", err) + } + return string(data) +} + +func decodeOK(t *testing.T, resp apiResponse, wantStatus int, target any) { + t.Helper() + if resp.status != wantStatus { + t.Fatalf("status = %d, want %d; body: %s", resp.status, wantStatus, resp.body) + } + if err := json.Unmarshal(resp.body, target); err != nil { + t.Fatalf("decode response %s: %v", resp.body, err) + } +} + +func assertError(t *testing.T, resp apiResponse, wantStatus int, wantCode string) { + t.Helper() + if resp.status != wantStatus { + t.Fatalf("status = %d, want %d; body: %s", resp.status, wantStatus, resp.body) + } + var body struct { + Error struct { + Code string `json:"code"` + Message string `json:"message"` + RequestID string `json:"request_id"` + } `json:"error"` + } + if err := json.Unmarshal(resp.body, &body); err != nil { + t.Fatalf("decode error response %s: %v", resp.body, err) + } + if body.Error.Code != wantCode || body.Error.Message == "" || body.Error.RequestID != resp.requestID { + t.Fatalf("invalid error response: header request ID %q, body %s", resp.requestID, resp.body) + } +} + +func (api *apiClient) createUser(t *testing.T, username string) string { + t.Helper() + resp := api.request(t, http.MethodPost, "/api/v1/users", "", jsonString(t, map[string]string{ + "username": username, "display_name": username, "password": "Synthetic-password-123", + })) + var user struct { + ID string `json:"id"` + } + decodeOK(t, resp, http.StatusCreated, &user) + if user.ID == "" { + t.Fatal("created user has no ID") + } + req, _ := http.NewRequest(http.MethodPost, api.baseURL+"/api/v1/auth/sessions", nil) + req.SetBasicAuth(api.prefix+username, "Synthetic-password-123") + sessionResp, err := (&http.Client{Timeout: 5 * time.Second}).Do(req) + if err != nil { + t.Fatal(err) + } + defer sessionResp.Body.Close() + var session struct { + Token string `json:"token"` + } + if err = json.NewDecoder(sessionResp.Body).Decode(&session); err != nil || sessionResp.StatusCode != 201 || session.Token == "" { + t.Fatalf("login failed: status %d, error %v", sessionResp.StatusCode, err) + } + api.tokens[user.ID] = session.Token + return user.ID +} + +func (api *apiClient) createChat(t *testing.T, creatorID string, memberIDs ...string) string { + t.Helper() + resp := api.request(t, http.MethodPost, "/api/v1/chats", creatorID, jsonString(t, map[string]any{ + "title": "Test chat", "member_ids": memberIDs, + })) + var chat struct { + ID string `json:"id"` + } + decodeOK(t, resp, http.StatusCreated, &chat) + if chat.ID == "" { + t.Fatal("created chat has no ID") + } + return chat.ID +} + +func TestInvalidJSON(t *testing.T) { + api := newAPI(t) + for _, tc := range []struct { + name string + body string + }{ + {name: "malformed", body: `{"username":`}, + {name: "wrong type", body: `{"username":42,"display_name":"Alice"}`}, + {name: "unknown field", body: `{"username":"alice","display_name":"Alice","admin":true}`}, + } { + t.Run(tc.name, func(t *testing.T) { + resp := api.request(t, http.MethodPost, "/api/v1/users", "", tc.body) + assertError(t, resp, http.StatusBadRequest, "invalid_request") + }) + } +} + +func TestForeignChatIsHidden(t *testing.T) { + api := newAPI(t) + alice := api.createUser(t, "alice") + bob := api.createUser(t, "bob") + eve := api.createUser(t, "eve") + chatID := api.createChat(t, alice, alice, bob) + + for _, tc := range []struct { + method string + path string + body string + }{ + {method: http.MethodGet, path: "/api/v1/chats/" + chatID}, + {method: http.MethodGet, path: "/api/v1/chats/" + chatID + "/messages"}, + {method: http.MethodPost, path: "/api/v1/chats/" + chatID + "/messages", body: `{"text":"secret"}`}, + } { + resp := api.request(t, tc.method, tc.path, eve, tc.body) + assertError(t, resp, http.StatusNotFound, "not_found") + } +} + +func TestEmptyMessage(t *testing.T) { + api := newAPI(t) + alice := api.createUser(t, "alice") + chatID := api.createChat(t, alice, alice) + path := "/api/v1/chats/" + chatID + "/messages" + for _, body := range []string{`{"text":""}`, `{"text":" \t\n "}`} { + resp := api.request(t, http.MethodPost, path, alice, body) + assertError(t, resp, http.StatusBadRequest, "invalid_request") + } +} + +func TestDuplicateUsername(t *testing.T) { + api := newAPI(t) + api.createUser(t, "alice") + resp := api.request(t, http.MethodPost, "/api/v1/users", "", `{"username":"alice","display_name":"Another Alice","password":"Synthetic-password-123"}`) + assertError(t, resp, http.StatusConflict, "conflict") +} + +func TestMessagePagination(t *testing.T) { + api := newAPI(t) + alice := api.createUser(t, "alice") + chatID := api.createChat(t, alice, alice) + wantIDs := make(map[string]bool) + for i := range 5 { + resp := api.request(t, http.MethodPost, "/api/v1/chats/"+chatID+"/messages", alice, + jsonString(t, map[string]string{"text": fmt.Sprintf("message %d", i)})) + var message struct { + ID string `json:"id"` + } + decodeOK(t, resp, http.StatusCreated, &message) + if message.ID == "" || wantIDs[message.ID] { + t.Fatalf("invalid or duplicate created message ID %q", message.ID) + } + wantIDs[message.ID] = true + } + + seen := make(map[string]bool) + cursor := "" + pages := 0 + for { + path := "/api/v1/chats/" + chatID + "/messages?limit=2" + if cursor != "" { + path += "&cursor=" + url.QueryEscape(cursor) + } + resp := api.request(t, http.MethodGet, path, alice, "") + var page struct { + Items []struct { + ID string `json:"id"` + } `json:"items"` + NextCursor *string `json:"next_cursor"` + } + decodeOK(t, resp, http.StatusOK, &page) + pages++ + if pages > 3 || len(page.Items) == 0 || len(page.Items) > 2 { + t.Fatalf("unexpected page %d: %s", pages, resp.body) + } + for _, message := range page.Items { + if !wantIDs[message.ID] || seen[message.ID] { + t.Fatalf("unexpected or repeated message ID %q on page %d", message.ID, pages) + } + seen[message.ID] = true + } + if page.NextCursor == nil { + break + } + if *page.NextCursor == "" || *page.NextCursor == cursor { + t.Fatalf("invalid next cursor on page %d: %s", pages, resp.body) + } + cursor = *page.NextCursor + } + if pages != 3 || len(seen) != len(wantIDs) { + t.Fatalf("got %d pages and %d unique messages, want 3 pages and %d messages", pages, len(seen), len(wantIDs)) + } +} diff --git a/service/tests/attachments_test.go b/service/tests/attachments_test.go new file mode 100644 index 0000000..1a3e6bf --- /dev/null +++ b/service/tests/attachments_test.go @@ -0,0 +1,226 @@ +package tests + +import ( + "bytes" + "crypto/sha256" + "encoding/binary" + "encoding/hex" + "encoding/json" + "hash/crc32" + "image" + "image/jpeg" + "io" + "net/http" + "net/url" + "os" + "strings" + "sync" + "testing" + "time" + + "github.com/google/uuid" +) + +func (api *apiClient) upload(t *testing.T, chat, user, name, mime string, data []byte, chunked bool) apiResponse { + t.Helper() + req, e := http.NewRequest("POST", api.baseURL+"/api/v1/chats/"+chat+"/attachments?filename="+url.QueryEscape(name), bytes.NewReader(data)) + if e != nil { + t.Fatal(e) + } + if chunked { + req.ContentLength = -1 + } + req.Header.Set("Content-Type", mime) + if user != "" { + req.Header.Set("Authorization", "Bearer "+api.tokens[user]) + } + resp, e := (&http.Client{Timeout: 20 * time.Second}).Do(req) + if e != nil { + t.Fatal(e) + } + defer resp.Body.Close() + body, e := io.ReadAll(resp.Body) + if e != nil { + t.Fatal(e) + } + if resp.Header.Get("X-Instance-Id") == "" { + t.Fatal("missing instance id") + } + return apiResponse{status: resp.StatusCode, body: body, requestID: resp.Header.Get("X-Request-Id"), instanceID: resp.Header.Get("X-Instance-Id")} +} +func fixturePNG(t *testing.T) []byte { + t.Helper() + b, e := os.ReadFile("../../tests/fixtures/sample.png") + if e != nil { + t.Fatal(e) + } + return b +} +func TestAttachmentContractAndErrors(t *testing.T) { + api := newAPI(t) + alice := api.createUser(t, "alice") + bob := api.createUser(t, "bob") + eve := api.createUser(t, "eve") + chat := api.createChat(t, alice, alice, bob) + other := api.createChat(t, alice, alice) + data := fixturePNG(t) + var att struct { + ID string `json:"id"` + Sha string `json:"sha256"` + Size int `json:"size_bytes"` + } + decodeOK(t, api.upload(t, chat, alice, "Файл.png", "image/png", data, false), 202, &att) + hash := sha256.Sum256(data) + if att.Sha != hex.EncodeToString(hash[:]) || att.Size != len(data) { + t.Fatal("original metadata mismatch") + } + waitAttachmentReady(t, api, alice, att.ID) + content := "/api/v1/attachments/" + att.ID + "/content" + got := api.request(t, "GET", content, bob, "") + if got.status != 200 || !bytes.Equal(got.body, data) { + t.Fatal("download changed bytes") + } + for _, suffix := range []string{"", "/content"} { + assertError(t, api.request(t, "GET", "/api/v1/attachments/"+att.ID+suffix, eve, ""), 404, "not_found") + } + assertError(t, api.upload(t, chat, eve, "file.png", "image/png", data, false), 404, "not_found") + assertError(t, api.upload(t, chat, "", "file.png", "image/png", data, false), 401, "unauthorized") + for _, name := range []string{"", " ", "../a.png", "a/b.png", "a\\b.png", "bad\x00.png", "bad\n.png", strings.Repeat("я", 256)} { + assertError(t, api.upload(t, chat, alice, name, "image/png", data, false), 400, "invalid_request") + } + for _, tc := range []struct { + name, mime string + data []byte + }{ + {"fake", "image/png", []byte("not an image")}, + {"mime mismatch", "image/jpeg", data}, + {"wrong type", "application/octet-stream", data}, + {"truncated", "image/png", data[:80]}, + } { + t.Run(tc.name, func(t *testing.T) { + assertError(t, api.upload(t, chat, alice, "test.png", tc.mime, tc.data, false), 415, "unsupported_media_type") + }) + } + bomb := bytes.Clone(data) + binary.BigEndian.PutUint32(bomb[16:20], 40000001) + binary.BigEndian.PutUint32(bomb[20:24], 1) + binary.BigEndian.PutUint32(bomb[29:33], crc32.ChecksumIEEE(bomb[12:29])) + assertError(t, api.upload(t, chat, alice, "bomb.png", "image/png", bomb, false), 413, "payload_too_large") + // A syntactically valid ancillary PNG chunk reaches the exact byte boundary + // without allocating an enormous decoded image. + padding := make([]byte, 10485760-len(data)-12) + chunk := append([]byte("paDd"), padding...) + boundary := append([]byte{}, data[:len(data)-12]...) + boundary = binary.BigEndian.AppendUint32(boundary, uint32(len(padding))) + boundary = append(boundary, chunk...) + boundary = binary.BigEndian.AppendUint32(boundary, crc32.ChecksumIEEE(chunk)) + boundary = append(boundary, data[len(data)-12:]...) + var boundaryAtt map[string]any + decodeOK(t, api.upload(t, chat, alice, "boundary.png", "image/png", boundary, false), 202, &boundaryAtt) + + oversized := make([]byte, 10485761) + copy(oversized, data) + for _, chunked := range []bool{false, true} { + assertError(t, api.upload(t, chat, alice, "large.png", "image/png", oversized, chunked), 413, "payload_too_large") + } + var jpg bytes.Buffer + if e := jpeg.Encode(&jpg, image.NewRGBA(image.Rect(0, 0, 32, 24)), nil); e != nil { + t.Fatal(e) + } + var jpegAtt map[string]any + decodeOK(t, api.upload(t, chat, bob, "file.jpg", "image/jpeg", jpg.Bytes(), false), 202, &jpegAtt) + imageOnly := `{"text":"","attachment_ids":["` + att.ID + `"]}` + var message map[string]any + key := uuid.NewString() + decodeOK(t, api.requestKey(t, "POST", "/api/v1/chats/"+chat+"/messages", bob, imageOnly, key), 201, &message) + var replay map[string]any + decodeOK(t, api.requestKey(t, "POST", "/api/v1/chats/"+chat+"/messages", bob, imageOnly, key), 201, &replay) + if jsonString(t, message) != jsonString(t, replay) { + t.Fatal("attachment retry changed message") + } + assertError(t, api.requestKey(t, "POST", "/api/v1/chats/"+chat+"/messages", bob, `{"text":"changed","attachment_ids":["`+att.ID+`"]}`, key), 409, "conflict") + assertError(t, api.request(t, "POST", "/api/v1/chats/"+other+"/messages", alice, imageOnly), 404, "not_found") + for _, body := range []string{`{"text":"","attachment_ids":["` + uuid.NewString() + `"]}`} { + assertError(t, api.request(t, "POST", "/api/v1/chats/"+chat+"/messages", alice, body), 404, "not_found") + } + for _, body := range []string{`{"text":""}`, `{"text":" "}`, `{"attachment_ids":["` + att.ID + `"]}`, `{"text":"ok","attachment_ids":null}`, `{"text":"ok","attachment_ids":"bad"}`, `{"text":"ok","attachment_ids":["` + att.ID + `","` + att.ID + `"]}`} { + assertError(t, api.request(t, "POST", "/api/v1/chats/"+chat+"/messages", alice, body), 400, "invalid_request") + } + if got := api.request(t, "GET", content+"?variant=thumbnail", bob, ""); got.status != 200 { + t.Fatal("thumbnail unavailable") + } +} +func TestConcurrentAttachmentMessage(t *testing.T) { + api := newAPI(t) + user := api.createUser(t, "alice") + chat := api.createChat(t, user, user) + var att struct { + ID string `json:"id"` + } + decodeOK(t, api.upload(t, chat, user, "sample.png", "image/png", fixturePNG(t), false), 202, &att) + key := uuid.NewString() + body := `{"text":"","attachment_ids":["` + att.ID + `"]}` + type result struct { + resp apiResponse + err error + } + ch := make(chan result, 16) + var wg sync.WaitGroup + for range 16 { + wg.Go(func() { + resp, e := api.send("POST", "/api/v1/chats/"+chat+"/messages", user, body, key) + ch <- result{resp, e} + }) + } + wg.Wait() + close(ch) + seen := map[string]bool{} + var id string + for item := range ch { + if item.err != nil { + t.Fatal(item.err) + } + var msg struct { + ID string `json:"id"` + IDs []string `json:"attachment_ids"` + } + decodeOK(t, item.resp, 201, &msg) + if id != "" && id != msg.ID { + t.Fatal("duplicate attachment message") + } + id = msg.ID + seen[item.resp.instanceID] = true + if len(msg.IDs) != 1 || msg.IDs[0] != att.ID { + t.Fatal("lost attachment") + } + } + if len(seen) < 2 { + t.Fatal("did not exercise two replicas") + } + var page struct { + Items []json.RawMessage `json:"items"` + } + decodeOK(t, api.request(t, "GET", "/api/v1/chats/"+chat+"/messages", user, ""), 200, &page) + if len(page.Items) != 1 { + t.Fatal("duplicate history") + } +} + +func waitAttachmentReady(t *testing.T, api *apiClient, user, id string) { + t.Helper() + deadline := time.Now().Add(60 * time.Second) + for time.Now().Before(deadline) { + var att struct { + Status string `json:"status"` + } + decodeOK(t, api.request(t, "GET", "/api/v1/attachments/"+id, user, ""), 200, &att) + if att.Status == "ready" { + return + } + if att.Status == "failed" { + t.Fatal("attachment failed") + } + time.Sleep(100 * time.Millisecond) + } + t.Fatal("attachment readiness deadline") +} diff --git a/service/tests/concurrency_test.go b/service/tests/concurrency_test.go new file mode 100644 index 0000000..c48ec8c --- /dev/null +++ b/service/tests/concurrency_test.go @@ -0,0 +1,161 @@ +package tests + +import ( + "encoding/json" + "fmt" + "github.com/google/uuid" + "io" + "net/http" + "net/url" + "strings" + "sync" + "testing" + "time" +) + +type createdMessage struct { + ID string `json:"id"` + SenderID string `json:"sender_id"` + Text string `json:"text"` +} + +func postMessage(client *http.Client, endpoint, userID, text string) (createdMessage, error) { + body, err := json.Marshal(struct { + Text string `json:"text"` + }{Text: text}) + if err != nil { + return createdMessage{}, err + } + req, err := http.NewRequest(http.MethodPost, endpoint, strings.NewReader(string(body))) + if err != nil { + return createdMessage{}, err + } + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Authorization", "Bearer "+userID) + req.Header.Set("Idempotency-Key", uuid.NewString()) + resp, err := client.Do(req) + if err != nil { + return createdMessage{}, err + } + defer resp.Body.Close() + data, err := io.ReadAll(resp.Body) + if err != nil { + return createdMessage{}, err + } + if resp.StatusCode != http.StatusCreated { + return createdMessage{}, fmt.Errorf("POST status %d: %s", resp.StatusCode, data) + } + if resp.Header.Get("X-Request-Id") == "" { + return createdMessage{}, fmt.Errorf("POST response has no X-Request-Id") + } + var message createdMessage + if err := json.Unmarshal(data, &message); err != nil { + return createdMessage{}, fmt.Errorf("decode POST response: %w", err) + } + return message, nil +} + +func TestConcurrentMessages(t *testing.T) { + const clients = 10 + const messagesPerClient = 10 + const total = clients * messagesPerClient + + api := newAPI(t) + users := make([]string, clients) + for i := range users { + users[i] = api.createUser(t, fmt.Sprintf("user%02d", i)) + } + chatID := api.createChat(t, users[0], users...) + endpoint := api.baseURL + "/api/v1/chats/" + chatID + "/messages" + + type result struct { + message createdMessage + wantSenderID string + wantText string + err error + } + results := make(chan result, total) + start := make(chan struct{}) + var ready sync.WaitGroup + var workers sync.WaitGroup + ready.Add(clients) + workers.Add(clients) + for i, userID := range users { + go func(workerID int, userID string) { + defer workers.Done() + client := &http.Client{Timeout: 5 * time.Second} + defer client.CloseIdleConnections() + ready.Done() + <-start + for j := range messagesPerClient { + text := fmt.Sprintf("client-%02d-message-%02d", workerID, j) + message, err := postMessage(client, endpoint, api.tokens[userID], text) + results <- result{message: message, wantSenderID: userID, wantText: text, err: err} + } + }(i, userID) + } + ready.Wait() + close(start) + workers.Wait() + close(results) + + created := make(map[string]createdMessage, total) + count := 0 + for item := range results { + count++ + if item.err != nil { + t.Errorf("send %q: %v", item.wantText, item.err) + continue + } + if item.message.ID == "" || item.message.SenderID != item.wantSenderID || item.message.Text != item.wantText { + t.Errorf("unexpected response for %q: %+v", item.wantText, item.message) + continue + } + if _, duplicate := created[item.message.ID]; duplicate { + t.Errorf("duplicate message ID %q", item.message.ID) + continue + } + created[item.message.ID] = item.message + } + if count != total || len(created) != total { + t.Fatalf("sent %d requests, created %d unique messages; want %d", count, len(created), total) + } + + seen := make(map[string]bool, total) + cursor := "" + pages := 0 + for { + path := "/api/v1/chats/" + chatID + "/messages?limit=17" + if cursor != "" { + path += "&cursor=" + url.QueryEscape(cursor) + } + resp := api.request(t, http.MethodGet, path, users[0], "") + var page struct { + Items []createdMessage `json:"items"` + NextCursor *string `json:"next_cursor"` + } + decodeOK(t, resp, http.StatusOK, &page) + pages++ + if pages > total || len(page.Items) == 0 || len(page.Items) > 17 { + t.Fatalf("unexpected page %d: %s", pages, resp.body) + } + for _, message := range page.Items { + want, exists := created[message.ID] + if !exists || seen[message.ID] || message != want { + t.Fatalf("unexpected or repeated message on page %d: %+v", pages, message) + } + seen[message.ID] = true + } + if page.NextCursor == nil { + break + } + if *page.NextCursor == "" || *page.NextCursor == cursor { + t.Fatalf("invalid cursor on page %d: %s", pages, resp.body) + } + cursor = *page.NextCursor + } + if len(seen) != total { + t.Fatalf("history has %d unique messages across %d pages; want %d", len(seen), pages, total) + } + t.Logf("%d clients sent %d messages; history contains %d unique messages across %d pages", clients, total, len(seen), pages) +} diff --git a/service/tests/idempotency_test.go b/service/tests/idempotency_test.go new file mode 100644 index 0000000..9b317fd --- /dev/null +++ b/service/tests/idempotency_test.go @@ -0,0 +1,104 @@ +package tests + +import ( + "fmt" + "github.com/google/uuid" + "net/http" + "sync" + "testing" +) + +func TestIdempotencyContract(t *testing.T) { + api := newAPI(t) + alice := api.createUser(t, "alice") + bob := api.createUser(t, "bob") + eve := api.createUser(t, "eve") + chat := api.createChat(t, alice, alice, bob) + other := api.createChat(t, alice, alice) + path := "/api/v1/chats/" + chat + "/messages" + for _, key := range []string{"", "bad key", "кириллица", string(make([]byte, 129))} { + // NUL cannot be sent by net/http: use an overlength ASCII key instead. + if len(key) == 129 { + key = fmt.Sprintf("%0129d", 1) + } + assertError(t, api.requestKey(t, "POST", path, alice, `{"text":"hello"}`, key), 400, "invalid_request") + } + key := uuid.NewString() + first := api.requestKey(t, "POST", path, alice, `{"text":"hello"}`, key) + var want map[string]any + decodeOK(t, first, 201, &want) + var got map[string]any + decodeOK(t, api.requestKey(t, "POST", path, alice, `{ "text" : "hello" }`, key), 201, &got) + if jsonString(t, want) != jsonString(t, got) { + t.Fatal("replay changed message") + } + assertError(t, api.requestKey(t, "POST", path, alice, `{"text":"changed"}`, key), 409, "conflict") + assertError(t, api.requestKey(t, "POST", path, eve, `{"text":"hello"}`, key), 404, "not_found") + for _, tc := range []struct{ user, path string }{{bob, path}, {alice, "/api/v1/chats/" + other + "/messages"}} { + var msg map[string]any + decodeOK(t, api.requestKey(t, "POST", tc.path, tc.user, `{"text":"hello"}`, key), 201, &msg) + if msg["id"] == want["id"] { + t.Fatal("scope not isolated") + } + } + resp := api.request(t, http.MethodDelete, "/api/v1/auth/sessions/current", alice, "") + if resp.status != 204 { + t.Fatal(resp.status) + } + assertError(t, api.requestKey(t, "POST", path, alice, `{"text":"hello"}`, key), 401, "unauthorized") +} +func TestConcurrentIdempotencyAcrossReplicas(t *testing.T) { + api := newAPI(t) + user := api.createUser(t, "alice") + chat := api.createChat(t, user, user) + path := "/api/v1/chats/" + chat + "/messages" + key := uuid.NewString() + const count = 32 + type result struct { + response apiResponse + err error + } + responses := make(chan result, count) + start := make(chan struct{}) + var wg sync.WaitGroup + for range count { + wg.Go(func() { + <-start + response, err := api.send("POST", path, user, `{"text":"one logical message"}`, key) + responses <- result{response: response, err: err} + }) + } + close(start) + wg.Wait() + close(responses) + instances := map[string]bool{} + var id string + for result := range responses { + if result.err != nil { + t.Fatal(result.err) + } + resp := result.response + var msg struct { + ID string `json:"id"` + } + decodeOK(t, resp, 201, &msg) + if id != "" && id != msg.ID { + t.Fatal("duplicate side effect") + } + id = msg.ID + instances[resp.instanceID] = true + } + if len(instances) < 2 { + t.Fatalf("need at least 2 real API replicas, observed %d", len(instances)) + } + var page struct { + Items []struct { + ID string `json:"id"` + } `json:"items"` + } + decodeOK(t, api.request(t, "GET", path, user, ""), 200, &page) + if len(page.Items) != 1 || page.Items[0].ID != id { + t.Fatalf("history contains %d messages", len(page.Items)) + } + t.Logf("%d requests across %d replicas created exactly one message", count, len(instances)) +} diff --git a/student-tests/README.md b/student-tests/README.md new file mode 100644 index 0000000..dc16e6e --- /dev/null +++ b/student-tests/README.md @@ -0,0 +1,13 @@ +# Проверки реализации lab7 + +Перенесённые из lab6 сценарии используют только Compose-проект `mtusi2026lab7`, HTTPS `https://localhost:8443` и локальный CA. Публичные проверки в `tests/` сохранены. + +- `python3 student-tests/images.py`: геометрия, EXIF, alpha, форматы, checksums, ACL, отрицательные входы. +- `python3 student-tests/storage.py`: staging, cleanup, retention, restore/replay. +- `python3 student-tests/broker_persistence.py`: замена контейнера RabbitMQ с сохранением подтверждённой очереди. +- `python3 student-tests/pipeline.py --repeats 1`: перенесённые fault-сценарии pipeline; явно включает `compose.faults.yaml` и синтетические barriers. +- `make test-internal-security`: новые проверки TLS/mTLS, прав worker, OpenBao, rate limit и ограничений запроса. +- `python3 scripts/prove-auth.py`: rehash, измерения KDF/login и expiry. +- `make prove-worker`: три повтора incident response. + +Изменяющие состояние сценарии запускаются последовательно. Наличие скрипта не означает, что он был выполнен: фактические результаты перечислены в REPORT.md. Новые Go unit-тесты не добавлены. diff --git a/student-tests/broker_persistence.py b/student-tests/broker_persistence.py new file mode 100644 index 0000000..d013e51 --- /dev/null +++ b/student-tests/broker_persistence.py @@ -0,0 +1,47 @@ +#!/usr/bin/env python3 +"""Prove a confirmed queued message survives broker container replacement, not only restart.""" +from pathlib import Path +import json +import subprocess +import sys +import time +ROOT=Path(__file__).resolve().parents[1] +sys.path.insert(0,str(ROOT/'tests')) +from client import Client,check + +def compose(*args): + r=subprocess.run([str(ROOT/'scripts/compose.sh'),*args],stdout=subprocess.PIPE,stderr=subprocess.PIPE,text=True,timeout=150) + check(r.returncode==0,'broker persistence infrastructure command failed: '+args[0]);return r.stdout.strip() + +def queued(): + r=subprocess.run([str(ROOT/'scripts/compose.sh'),'exec','-T','broker','rabbitmqctl','-q','list_queues','-p','attachments','name','messages'],stdout=subprocess.PIPE,stderr=subprocess.PIPE,text=True,timeout=30) + if r.returncode:return 0 + out=r.stdout + for line in out.splitlines(): + fields=line.split() + if len(fields)==2 and fields[0]=='attachments.v1':return int(fields[1]) + return 0 + +def main(): + c=Client('https://localhost:8443', ca_file=str(ROOT/'secrets/lab7/lb/ca.crt'));alice,bob,eve,chat=c.bootstrap() + try: + compose('stop','worker');att,data=c.attachment(alice,chat) + deadline=time.monotonic()+20 + while queued()<1 and time.monotonic()=1,'message not durably queued before replacement') + published=compose('exec','-T','postgres','psql','-U','messenger','-d','messenger','-At','-c',"SELECT bool_and(status='published') FROM outbox WHERE job_id=(SELECT id FROM attachment_jobs WHERE attachment_id='%s')"%att['id']) + check(published=='t','outbox not marked published before replacement') + compose('stop','dispatcher') + old=compose('ps','-q','broker') + compose('up','-d','--no-deps','--force-recreate','--wait','--wait-timeout','120','broker') + new=compose('ps','-q','broker');check(new!=old,'broker container was not replaced') + deadline=time.monotonic()+30 + while queued()<1 and time.monotonic()=1,'confirmed queue message lost after replacement') + compose('start','worker');c.wait_ready(bob,att) + check(c.call('GET','/api/v1/attachments/'+att['id']+'/content',actor=bob)['bytes']==data,'original changed after broker replacement') + result={'result':'PASS','attachment_id':att['id'],'broker_hostname':'broker','old_container_id':old,'new_container_id':new,'message_present_before_and_after':True,'outbox_published_before_replacement':True,'dispatcher_stopped_during_replacement':True} + (ROOT/'evidence/broker-persistence.json').write_text(json.dumps(result,indent=2)+'\n') + print('PASS: confirmed message survived broker container replacement with the same volume and stable node identity') + finally:compose('start','worker','dispatcher') +if __name__=='__main__':main() diff --git a/student-tests/images.py b/student-tests/images.py new file mode 100644 index 0000000..50ef776 --- /dev/null +++ b/student-tests/images.py @@ -0,0 +1,53 @@ +#!/usr/bin/env python3 +"""Image contract checks using synthetic fixtures and real decoding in ImageMagick.""" +import hashlib +import json +from pathlib import Path +import struct +import subprocess +import sys +import urllib.parse +import zlib +ROOT=Path(__file__).resolve().parents[1] +sys.path.insert(0,str(ROOT/'tests')) +from client import Client,check +from image_probe import jpeg_size + +def magick(args,data=None): + r=subprocess.run([str(ROOT/'scripts/compose.sh'),'run','--rm','--no-deps','-T','--entrypoint','magick','operator']+args,input=data,stdout=subprocess.PIPE,stderr=subprocess.PIPE,timeout=30) + check(r.returncode==0,'fixture decoder failed') + return r.stdout + +def png(w,h,rgba): + def chunk(t,b):return struct.pack('>I',len(b))+t+b+struct.pack('>I',zlib.crc32(t+b)) + return b'\x89PNG\r\n\x1a\n'+chunk(b'IHDR',struct.pack('>IIBBBBB',w,h,8,6,0,0,0))+chunk(b'IDAT',zlib.compress(b''.join(b'\0'+b''.join(bytes(rgba(x,y)) for x in range(w)) for y in range(h)) if callable(rgba) else (b'\0'+bytes(rgba)*w)*h))+chunk(b'IEND',b'') + +def main(): + c=Client('https://localhost:8443', ca_file=str(ROOT/'secrets/lab7/lb/ca.crt'));alice,bob,eve,chat=c.bootstrap();results=[] + jpeg=magick(['png:-','jpeg:-'],png(300,150,(255,0,0,255))) + # EXIF TIFF orientation 6: display rotates the stored 300x150 pixels clockwise. + tiff=b'II'+struct.pack('H',len(exif)+2)+exif+jpeg[2:] + for name,data,mime,dimensions in [('small',png(16,8,(255,0,0,255)),'image/png',(16,8)),('floor',png(333,100,(255,0,0,255)),'image/png',(256,76)),('alpha',png(40,20,(0,0,0,0)),'image/png',(40,20)),('orientation',oriented,'image/jpeg',(128,256)),('thin',png(1,300,(0,0,0,255)),'image/png',(1,256)),('center_crop',png(300,100,lambda x,y:(255,0,0,255) if 100<=x<200 else (0,0,255,255)),'image/png',(256,85))]: + att=c.call('POST','/api/v1/chats/'+chat['id']+'/attachments?filename='+name,202,actor=alice,raw=data,headers={'Content-Type':mime},schema='Attachment')['json'];ready=c.wait_ready(bob,att) + original=c.call('GET','/api/v1/attachments/'+att['id']+'/content',actor=bob)['bytes'];check(original==data,'original changed') + for v in ready['variants']: + b=c.call('GET','/api/v1/attachments/'+att['id']+'/content?variant='+v['name'],actor=bob)['bytes'];check(hashlib.sha256(b).hexdigest()==v['sha256'],'checksum mismatch') + actual=tuple(map(int,magick(['identify','-format','%w %h','-'],b).split())) + check(actual==(v['width'],v['height']),'actual dimensions differ') + if v['name']=='thumbnail': + check(jpeg_size(b)==dimensions,'thumbnail geometry differs') + if name=='alpha':check(magick(['identify','-format','%[pixel:p{0,0}]','jpeg:-'],b).strip() in (b'gray(255)',b'srgb(255,255,255)'),'alpha was not composited on white') + if v['name']=='crop': + check(actual==(128,128),'crop geometry differs') + if name=='center_crop': + import re + color=tuple(map(int,re.findall(rb'\d+',magick(['identify','-format','%[pixel:p{0,0}]','jpeg:-'],b)))) + check(len(color)==3 and color[0]>=250 and color[1]<=5 and color[2]<=5,'crop is not centered') + check(b'Exif\0\0' not in b,'EXIF retained in derivative') + c.call('GET','/api/v1/attachments/'+att['id']+'/content',404,actor=eve) + results.append({'case':name,'result':'PASS','thumbnail':dimensions}) + for data,mime,status in [(b'not image','image/png',415),(png(2,2,(0,0,0,255)),'image/jpeg',415),(png(2,2,(0,0,0,255))[:40],'image/png',415),(b'x'*10485761,'image/png',413)]: + c.call('POST','/api/v1/chats/'+chat['id']+'/attachments?filename=bad',status,actor=alice,raw=data,headers={'Content-Type':mime}) + (ROOT/'evidence/images.json').write_text(json.dumps({'results':results,'negative_cases':4},indent=2)+'\n');print('PASS: image geometry, EXIF, alpha, real formats, checksums, ACL and negative inputs') +if __name__=='__main__':main() diff --git a/student-tests/pipeline.py b/student-tests/pipeline.py new file mode 100644 index 0000000..0dbda99 --- /dev/null +++ b/student-tests/pipeline.py @@ -0,0 +1,179 @@ +#!/usr/bin/env python3 +"""Repeatable lab5 experiment on the local synthetic Compose stand. Never prints tokens.""" +import argparse +import concurrent.futures +import datetime +import hashlib +import json +import os +from pathlib import Path +import subprocess +import sys +import time +import uuid + +ROOT=Path(__file__).resolve().parents[1] +sys.path.insert(0,str(ROOT/'tests')) +from client import Client, Failure, check +from image_probe import jpeg_size +FAULTS=ROOT/'.lab7-faults' +COMPOSE=[str(ROOT/'scripts/compose.sh'),'-f',str(ROOT/'compose.yaml'),'-f',str(ROOT/'compose.faults.yaml')] +RESULTS=[] +SELECTED=set() + +def compose(*args, allow_failure=False): + p=subprocess.run(COMPOSE+list(args),cwd=ROOT,stdout=subprocess.PIPE,stderr=subprocess.PIPE,text=True,timeout=180) + if p.returncode and not allow_failure: + # Infrastructure logs may contain settings; report only action and exit status. + raise Failure('Compose '+args[0]+' failed, exit '+str(p.returncode)) + return p + +def sql(query): + return compose('exec','-T','postgres','psql','-U','messenger','-d','messenger','-At','-v','ON_ERROR_STOP=1','-c',query).stdout.strip() + +def uid(value): + return str(uuid.UUID(value)) + +def job(att,version=1): + return sql("SELECT id FROM attachment_jobs WHERE attachment_id='%s' AND pipeline_version=%d"%(uid(att['id']),version)) + +def operator(action,id=None,version=None, allow_failure=False): + args=['run','--rm','--no-deps','-T','operator','jobs',action] + if id: args+=['--id',uid(id)] + if version:args+=['--pipeline-version',str(version)] + return compose(*args,allow_failure=allow_failure) + +def wait(predicate,timeout=65): + deadline=time.monotonic()+timeout + while time.monotonic()=2,'lease not recovered') + operator('reprocess',att['id'],2);second=ready(c,bob,att,data) + check(first['variants']!=second['variants'],'pipeline versions are indistinguishable') + operator('redeliver',jid);marker.unlink();time.sleep(1) + check(metadata(c,bob,att)==second,'stale worker/job overwrote version 2') + check(sql("SELECT pipeline_version FROM attachments WHERE id='%s'"%uid(att['id']))=='2','version rolled back') + ready(c,bob,att,data) + return {'attachment_id':att['id'],'old_job_id':jid,'pipeline_version':2,'manifest_rows':8} + record('lease_reorder_and_stale_version',run,version_fence) + def retries(): + att,data,jid=upload_stopped();marker=directive('before_objects',jid,'temporary');compose('start','worker');failed=state(c,bob,att,'failed',65) + check(failed['error_code']=='retry_exhausted','wrong terminal code');check(sql("SELECT attempt FROM attachment_jobs WHERE id='%s'"%uid(jid))=='5','attempt limit not honored') + c.call('POST','/api/v1/chats/'+chat['id']+'/messages',409,actor=alice,body={'text':'','attachment_ids':[att['id']]},headers={'Idempotency-Key':uuid.uuid4().hex}) + marker.unlink();operator('replay',jid);ready(c,bob,att,data);check(operator('replay',jid,allow_failure=True).returncode!=0,'successful job replay allowed') + return {'job_id':jid,'failed_attempt':5,'replayed_attempt':6} + record('bounded_retry_and_explicit_replay',run,retries) + def permanent(): + att,data,jid=upload_stopped();marker=directive('before_objects',jid,'permanent');compose('start','worker');failed=state(c,bob,att,'failed');check(failed['error_code']=='invalid_image','wrong permanent code');check(sql("SELECT attempt FROM attachment_jobs WHERE id='%s'"%uid(jid))=='1','permanent error was retried');marker.unlink();operator('replay',jid);ready(c,bob,att,data);return {'job_id':jid} + record('permanent_error',run,permanent) + for point in ('after_staging','after_upload_commit'): + def api_crash(point=point): + if point=='after_upload_commit':compose('stop','dispatcher') + marker=directive(point,'all','crash') + try:c.attachment(alice,chat) + except Failure:pass + aid=hit(marker);att={'id':aid} + if point=='after_staging':check(sql("SELECT count(*) FROM attachments WHERE id='%s'"%uid(aid))=='0','precommit row exists') + else: + check(sql("SELECT status FROM outbox WHERE job_id=(SELECT id FROM attachment_jobs WHERE attachment_id='%s')"%uid(aid))=='pending','COMMIT/publish window not reached') + compose('start','dispatcher');ready(c,bob,att,(ROOT/'tests/fixtures/sample.png').read_bytes()) + wait(lambda:health(c));return {'attachment_id':aid,'accepted_commit':point=='after_upload_commit'} + record('api_crash_'+point,run,api_crash) + payload={'date_utc':datetime.datetime.now(datetime.timezone.utc).isoformat(),'commit':subprocess.check_output(['git','rev-parse','HEAD'],cwd=ROOT,text=True).strip(),'working_tree':'implementation changes uncommitted','repeats':args.repeats,'worker':{'concurrency':2,'prefetch':2,'timeout_seconds':5,'lease_seconds':8,'memory':'1GiB','cpus':1},'results':RESULTS} + check(bool(RESULTS),'no scenarios selected') + payload['image_id']=subprocess.check_output(['docker','image','inspect','mtusi-messenger:lab7','--format','{{.Id}}'],text=True).strip() + (ROOT/args.output).write_text(json.dumps(payload,ensure_ascii=False,indent=2)+'\n') + finally: + for f in FAULTS.iterdir():f.unlink() + compose('stop','worker','dispatcher',allow_failure=True) + # Restore standard configuration, disabling fault injection and short experiment leases. + subprocess.run([str(ROOT/'scripts/compose.sh'),'up','-d','--wait','--wait-timeout','120'],cwd=ROOT,stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL,timeout=180) + +def health(c): + try:c.call('GET','/health/ready',schema='Health');return True + except Failure:return False + +if __name__=='__main__':main() diff --git a/student-tests/storage.py b/student-tests/storage.py new file mode 100644 index 0000000..7604096 --- /dev/null +++ b/student-tests/storage.py @@ -0,0 +1,62 @@ +#!/usr/bin/env python3 +"""Staging protection, retirement, retention and explicit restoration on synthetic data.""" +import json +from pathlib import Path +import sys +import time +ROOT=Path(__file__).resolve().parents[1] +sys.path.insert(0,str(ROOT/'tests')) +from client import Client,check +BASE=[str(ROOT/'scripts/compose.sh')] +import subprocess + +def command(*args,fail=False): + r=subprocess.run(BASE+list(args),stdout=subprocess.PIPE,stderr=subprocess.PIPE,text=True,timeout=120) + if not fail:check(r.returncode==0,'storage scenario infrastructure command failed: '+args[0]) + return r + +def sql(q):return command('exec','-T','postgres','psql','-U','messenger','-d','messenger','-At','-v','ON_ERROR_STOP=1','-c',q).stdout.strip() +def op(*args,fail=False):return command('run','--rm','--no-deps','-T','operator',*args,fail=fail) +def wait(fn): + deadline=time.monotonic()+20 + while time.monotonic()0 and exists(key),'reconciliation removed active source') + results.append({'scenario':'queued_staging_protected','result':'PASS','attachment_id':att['id']}) + command('start','worker');c.wait_ready(bob,att) + wait(lambda:sql("SELECT staging_key FROM attachments WHERE id='%s'"%att['id'])=='') + check(not exists(key),'ready source staging not retired') + results.append({'scenario':'staging_deleted_after_ready','result':'PASS','attachment_id':att['id']}) + command('stop','worker') + failed,data=c.attachment(alice,chat);key='staging/'+failed['id'];jid=sql("SELECT id FROM attachment_jobs WHERE attachment_id='%s'"%failed['id']) + op('storage','remove-object','--id',failed['id']);command('start','worker') + def is_failed():return c.call('GET','/api/v1/attachments/'+failed['id'],actor=bob,schema='Attachment')['json']['status']=='failed' + wait(is_failed) + check(sql("SELECT error_code FROM attachment_jobs WHERE id='%s'"%jid)=='source_missing','missing source not permanent') + check(op('jobs','replay','--id',jid,fail=True).returncode!=0,'replay accepted missing source') + op('storage','restore','--id',failed['id'],'--source','/fixtures/sample.png') + check(exists(key),'restored source missing') + # Advance only the synthetic record's age; do not wait a real day or change production retention. + sql("UPDATE attachments SET updated_at=clock_timestamp()-interval '25 hours' WHERE id='%s'"%failed['id']) + wait(lambda:sql("SELECT staging_key FROM attachments WHERE id='%s'"%failed['id'])=='') + check(not exists(key),'expired failed source not retired') + check(op('jobs','replay','--id',jid,fail=True).returncode!=0,'replay accepted retired source') + results.append({'scenario':'failed_retention_and_missing_source_replay','result':'PASS','age_simulated_hours':25}) + op('storage','restore','--id',failed['id'],'--source','/fixtures/sample.png');op('jobs','replay','--id',jid);c.wait_ready(bob,failed) + check(c.call('GET','/api/v1/attachments/'+failed['id']+'/content',actor=bob)['bytes']==data,'restored original changed') + results.append({'scenario':'verified_restore_then_replay','result':'PASS','job_id':jid}) + (ROOT/'evidence/storage.json').write_text(json.dumps({'results':results},indent=2)+'\n') + print('PASS: active source protected, ready staging removed, failed retention enforced, restore/replay verified') + finally:command('start','worker') +if __name__=='__main__':main()