diff --git a/.dockerignore b/.dockerignore index bc16e55..ddd2c73 100644 --- a/.dockerignore +++ b/.dockerignore @@ -17,4 +17,5 @@ backups __pycache__ node_modules evidence +/messenger .DS_Store diff --git a/.gitea/workflows/ci.yml b/.gitea/workflows/ci.yml new file mode 100644 index 0000000..33018c7 --- /dev/null +++ b/.gitea/workflows/ci.yml @@ -0,0 +1,101 @@ +name: CI + +on: + pull_request: + branches: [main] + push: + branches: [main] + workflow_dispatch: + +concurrency: + group: ci-${{ github.repository }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + verify: + runs-on: ubuntu-latest + timeout-minutes: 45 + env: + COMPOSE_PROJECT_NAME: mtusi2026lab6-${{ github.run_id }} + API_BIND_ADDRESS: 127.0.0.1:0 + SWAGGER_BIND_ADDRESS: 127.0.0.1:0 + PROMETHEUS_BIND_ADDRESS: 127.0.0.1:0 + GRAFANA_BIND_ADDRESS: 127.0.0.1:0 + LOKI_BIND_ADDRESS: 127.0.0.1:0 + TEMPO_BIND_ADDRESS: 127.0.0.1:0 + steps: + - name: Checkout exact commit + run: | + git init . + git remote add origin https://gitea.privcloud.tech/${{ github.repository }}.git + git fetch --depth=1 origin ${{ github.sha }} + git checkout --detach FETCH_HEAD + + - uses: actions/setup-go@v5 + with: + go-version-file: go.mod + cache: false + + - name: Check contract + run: make check + + - name: Validate OpenAPI + run: | + python3 -m venv .venv + .venv/bin/python -m pip install -r requirements-dev.txt + make validate PYTHON=.venv/bin/python + + - name: Check formatting and shell scripts + run: | + test -z "$(gofmt -l cmd internal migrations)" + sh -n infra/minio/bootstrap.sh scripts/restart.sh scripts/rolling-update.sh scripts/stop-one.sh scripts/test-lab5.sh scripts/test-lab5-acceptance.sh scripts/test-lab6.sh scripts/incident-worker.sh scripts/incident-database.sh scripts/incident-telemetry.sh + python3 -m py_compile scripts/lab6-traffic.py scripts/lab6-observe.py scripts/lab6-alert.py scripts/lab6-correlation.py scripts/validate-lab6-evidence.py + + - name: Lint + uses: golangci/golangci-lint-action@v7 + with: + version: v2.13.2 + args: --timeout=5m + skip-cache: true + + - name: Unit tests and build + run: | + go test -race ./... + go build ./cmd/... + + - name: Prepare local stack + run: | + cp lab6.env.example .env + docker compose config --quiet + docker compose up -d --build --scale api=2 --wait --wait-timeout 150 + docker network connect "${COMPOSE_PROJECT_NAME}_application" "$HOSTNAME" + docker network connect "${COMPOSE_PROJECT_NAME}_data" "$HOSTNAME" + docker network connect "${COMPOSE_PROJECT_NAME}_monitoring" "$HOSTNAME" + + - name: Integration checks + run: | + set -a + . ./.env + set +a + test "$(curl --noproxy '*' -s -o /dev/null -w '%{http_code}' "http://s3:9000/$S3_BUCKET")" = "403" + make test BASE_URL=http://nginx:8080 + docker compose --profile tests run --rm --build tests + go run ./cmd/dependencycheck -base-url=http://nginx:8080 + make test-persistence ACTION_SCRIPT=scripts/restart.sh BASE_URL=http://nginx:8080 + make test-failover ACTION_SCRIPT=scripts/stop-one.sh BASE_URL=http://nginx:8080 + docker compose up -d --scale api=2 --wait --wait-timeout 90 + make test-lab5-acceptance + make test-lab5 + ENV_FILE=.env PROMETHEUS_URL=http://prometheus:9090 TEMPO_URL=http://tempo:3200 LOKI_URL=http://loki:3100 make test-lab6-smoke BASE_URL=http://nginx:8080 + + - name: Container logs + if: failure() + run: if [ -f .env ]; then docker compose logs --no-color --tail=200; fi + + - name: Stop local stack + if: always() + run: | + docker network disconnect "${COMPOSE_PROJECT_NAME}_application" "$HOSTNAME" 2>/dev/null || true + docker network disconnect "${COMPOSE_PROJECT_NAME}_data" "$HOSTNAME" 2>/dev/null || true + docker network disconnect "${COMPOSE_PROJECT_NAME}_monitoring" "$HOSTNAME" 2>/dev/null || true + if [ -f .env ]; then docker compose --profile tests down -v --remove-orphans; fi diff --git a/.gitea/workflows/deploy.yml b/.gitea/workflows/deploy.yml new file mode 100644 index 0000000..b1b9a29 --- /dev/null +++ b/.gitea/workflows/deploy.yml @@ -0,0 +1,200 @@ +name: Deploy production + +on: + workflow_dispatch: + +concurrency: + group: production + cancel-in-progress: false + +jobs: + verify: + name: Verify selected commit + if: github.ref == 'refs/heads/main' + runs-on: ubuntu-latest + timeout-minutes: 45 + outputs: + api_image: ${{ steps.publish.outputs.api_image }} + nginx_image: ${{ steps.publish.outputs.nginx_image }} + swagger_image: ${{ steps.publish.outputs.swagger_image }} + prometheus_image: ${{ steps.publish.outputs.prometheus_image }} + grafana_image: ${{ steps.publish.outputs.grafana_image }} + s3_image: ${{ steps.publish.outputs.s3_image }} + s3_bootstrap_image: ${{ steps.publish.outputs.s3_bootstrap_image }} + env: + COMPOSE_PROJECT_NAME: mtusi2026lab6-deploy-${{ github.run_id }} + COMPOSE_FILE: compose.yaml:deploy/compose.images.yaml + API_BIND_ADDRESS: 127.0.0.1:0 + SWAGGER_BIND_ADDRESS: 127.0.0.1:0 + PROMETHEUS_BIND_ADDRESS: 127.0.0.1:0 + GRAFANA_BIND_ADDRESS: 127.0.0.1:0 + LOKI_BIND_ADDRESS: 127.0.0.1:0 + TEMPO_BIND_ADDRESS: 127.0.0.1:0 + API_IMAGE: gitea.privcloud.tech/${{ github.repository_owner }}/mtusi2026lab6-api:${{ github.sha }} + NGINX_IMAGE: gitea.privcloud.tech/${{ github.repository_owner }}/mtusi2026lab6-nginx:${{ github.sha }} + SWAGGER_IMAGE: gitea.privcloud.tech/${{ github.repository_owner }}/mtusi2026lab6-swagger:${{ github.sha }} + PROMETHEUS_IMAGE: gitea.privcloud.tech/${{ github.repository_owner }}/mtusi2026lab6-prometheus:${{ github.sha }} + GRAFANA_IMAGE: gitea.privcloud.tech/${{ github.repository_owner }}/mtusi2026lab6-grafana:${{ github.sha }} + S3_IMAGE: gitea.privcloud.tech/${{ github.repository_owner }}/mtusi2026lab6-minio:${{ github.sha }} + S3_BOOTSTRAP_IMAGE: gitea.privcloud.tech/${{ github.repository_owner }}/mtusi2026lab6-minio-client:${{ github.sha }} + steps: + - name: Checkout exact commit + run: | + git init . + git remote add origin https://gitea.privcloud.tech/${{ github.repository }}.git + git fetch --depth=1 origin ${{ github.sha }} + git checkout --detach FETCH_HEAD + + - uses: actions/setup-go@v5 + with: + go-version-file: go.mod + cache: false + + - name: Check contract + run: make check + + - name: Validate OpenAPI + run: | + python3 -m venv .venv + .venv/bin/python -m pip install -r requirements-dev.txt + make validate PYTHON=.venv/bin/python + + - name: Check formatting and shell scripts + run: | + test -z "$(gofmt -l cmd internal migrations)" + sh -n infra/minio/bootstrap.sh scripts/restart.sh scripts/rolling-update.sh scripts/stop-one.sh scripts/test-lab5.sh scripts/test-lab5-acceptance.sh scripts/test-lab6.sh scripts/incident-worker.sh scripts/incident-database.sh scripts/incident-telemetry.sh + python3 -m py_compile scripts/lab6-traffic.py scripts/lab6-observe.py scripts/lab6-alert.py scripts/lab6-correlation.py scripts/validate-lab6-evidence.py + + - name: Lint + uses: golangci/golangci-lint-action@v7 + with: + version: v2.13.2 + args: --timeout=5m + skip-cache: true + + - name: Unit tests and build + run: | + go test -race ./... + go build ./cmd/... + + - name: Prepare local stack + run: | + cp lab6.env.example .env + docker compose config --quiet + docker compose up -d --build --scale api=2 --wait --wait-timeout 150 + docker network connect "${COMPOSE_PROJECT_NAME}_application" "$HOSTNAME" + docker network connect "${COMPOSE_PROJECT_NAME}_data" "$HOSTNAME" + docker network connect "${COMPOSE_PROJECT_NAME}_monitoring" "$HOSTNAME" + + - name: Integration checks + run: | + set -a + . ./.env + set +a + test "$(curl --noproxy '*' -s -o /dev/null -w '%{http_code}' "http://s3:9000/$S3_BUCKET")" = "403" + make test BASE_URL=http://nginx:8080 + docker compose --profile tests run --rm --build tests + go run ./cmd/dependencycheck -base-url=http://nginx:8080 + make test-persistence ACTION_SCRIPT=scripts/restart.sh BASE_URL=http://nginx:8080 + make test-failover ACTION_SCRIPT=scripts/stop-one.sh BASE_URL=http://nginx:8080 + make test-lab5-acceptance + make test-lab5 + ENV_FILE=.env PROMETHEUS_URL=http://prometheus:9090 TEMPO_URL=http://tempo:3200 LOKI_URL=http://loki:3100 make test-lab6-smoke BASE_URL=http://nginx:8080 + + - name: Publish verified images + id: publish + env: + REGISTRY_USERNAME: ${{ secrets.REGISTRY_USERNAME }} + REGISTRY_PASSWORD: ${{ secrets.REGISTRY_PASSWORD }} + run: | + printf '%s' "$REGISTRY_PASSWORD" | docker login gitea.privcloud.tech --username "$REGISTRY_USERNAME" --password-stdin + + publish() { + output=$1 + image=$2 + push_output=$(docker push "$image") + printf '%s\n' "$push_output" + digest=$(printf '%s\n' "$push_output" | sed -n 's/^.*digest: \(sha256:[0-9a-f]*\).*$/\1/p' | tail -n 1) + test -n "$digest" + repository=${image%:*} + printf '%s=%s@%s\n' "$output" "$repository" "$digest" >> "$GITHUB_OUTPUT" + } + + publish api_image "$API_IMAGE" + publish nginx_image "$NGINX_IMAGE" + publish swagger_image "$SWAGGER_IMAGE" + publish prometheus_image "$PROMETHEUS_IMAGE" + publish grafana_image "$GRAFANA_IMAGE" + publish s3_image "$S3_IMAGE" + publish s3_bootstrap_image "$S3_BOOTSTRAP_IMAGE" + + - name: Container logs + if: failure() + run: if [ -f .env ]; then docker compose logs --no-color --tail=200; fi + + - name: Stop local stack + if: always() + run: | + docker network disconnect "${COMPOSE_PROJECT_NAME}_application" "$HOSTNAME" 2>/dev/null || true + docker network disconnect "${COMPOSE_PROJECT_NAME}_data" "$HOSTNAME" 2>/dev/null || true + docker network disconnect "${COMPOSE_PROJECT_NAME}_monitoring" "$HOSTNAME" 2>/dev/null || true + if [ -f .env ]; then docker compose --profile tests down -v --remove-orphans; fi + + deploy: + name: Deploy selected commit + needs: verify + runs-on: ubuntu-latest + environment: production + env: + DEPLOY_HOST: ${{ secrets.DEPLOY_HOST }} + DEPLOY_USER: ${{ secrets.DEPLOY_USER }} + DEPLOY_SSH_KEY: ${{ secrets.DEPLOY_SSH_KEY }} + DEPLOY_KNOWN_HOSTS: ${{ secrets.DEPLOY_KNOWN_HOSTS }} + REGISTRY_USERNAME: ${{ secrets.REGISTRY_USERNAME }} + REGISTRY_PASSWORD: ${{ secrets.REGISTRY_PASSWORD }} + DEPLOY_SHA: ${{ github.sha }} + steps: + - name: Configure SSH + run: | + umask 077 + mkdir -p ~/.ssh + printf '%s\n' "$DEPLOY_SSH_KEY" > ~/.ssh/id_ed25519 + printf '%s\n' "$DEPLOY_KNOWN_HOSTS" > ~/.ssh/known_hosts + + - name: Prepare immutable image manifest + run: | + cat > deploy-images.env </dev/null || true + docker network disconnect "${COMPOSE_PROJECT_NAME}_data" "$HOSTNAME" 2>/dev/null || true + docker network disconnect "${COMPOSE_PROJECT_NAME}_monitoring" "$HOSTNAME" 2>/dev/null || true + docker compose down -v --remove-orphans diff --git a/.gitignore b/.gitignore index b5a7189..7913498 100644 --- a/.gitignore +++ b/.gitignore @@ -30,3 +30,5 @@ coverage/ .vscode/ # Keep small, sanitized evidence summaries in evidence/ under version control. # Inspect git diff --cached: ignore rules are not a secret scanner. + +evidence/runtime/ diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..ab37655 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,45 @@ +FROM golang:1.26.5-alpine3.24 AS build + +WORKDIR /src +ENV GOMAXPROCS=2 GOMEMLIMIT=1GiB +COPY go.mod go.sum ./ +RUN go mod download +COPY . . +RUN go build -p=1 -o /out/messenger ./cmd/messenger +RUN go build -p=1 -o /out/lab5experiment ./cmd/lab5experiment + +FROM build AS test + +CMD ["go", "test", "./internal/postgres", "./internal/session"] + +FROM swaggerapi/swagger-ui:v5.31.2 AS swagger + +COPY contracts/openapi.json /spec/openapi.json +ENV SWAGGER_JSON=/spec/openapi.json + +FROM prom/prometheus:v3.13.3 AS prometheus + +COPY monitoring/prometheus.yml /etc/prometheus/prometheus.yml +COPY monitoring/rules /etc/prometheus/rules + +FROM grafana/grafana:13.2.4 AS grafana + +COPY monitoring/grafana/provisioning /etc/grafana/provisioning +COPY monitoring/grafana/dashboards /etc/grafana/dashboards + +FROM nginx:1.30.5-alpine AS loadbalancer + +COPY nginx/nginx.conf /etc/nginx/nginx.conf +COPY nginx/upstream.conf /etc/nginx/conf.d/upstream.conf + +FROM alpine:3.24.2 + +RUN addgroup -S app && adduser -S -G app -u 10001 app +COPY --from=build --chown=app:app /out/messenger /app/messenger +COPY --from=build --chown=app:app /out/lab5experiment /app/lab5experiment + +USER app +WORKDIR /app +HEALTHCHECK --interval=10s --timeout=2s --start-period=5s --retries=3 \ + CMD wget -q -O /dev/null "http://127.0.0.1:${APP_PORT}/health/ready" || exit 1 +CMD ["/app/messenger"] diff --git a/Makefile b/Makefile index 5d86d17..8a72b7e 100644 --- a/Makefile +++ b/Makefile @@ -4,7 +4,7 @@ CA_FILE ?= ACTION_SCRIPT ?= TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",) -.PHONY: help check validate test load test-persistence test-failover +.PHONY: help check validate test load test-persistence test-failover test-lab5 test-lab5-acceptance lab6-load test-lab6-smoke test-lab6 help: @echo "check: offline structure; validate: full OpenAPI (requirements-dev.txt); test: running API; load: synthetic writes" @@ -28,3 +28,18 @@ test-persistence: test-failover: @test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable stop-one hook"; exit 2) $(PYTHON) tests/resilience.py --mode failover --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS) + +test-lab5: + ./scripts/test-lab5.sh + +test-lab5-acceptance: + ./scripts/test-lab5-acceptance.sh + +lab6-load: + $(PYTHON) scripts/lab6-traffic.py --base-url "$(BASE_URL)" --mode mixed --duration 60 --concurrency 4 --wait-ready + +test-lab6-smoke: + BASE_URL="$(BASE_URL)" ./scripts/test-lab6.sh smoke + +test-lab6: + BASE_URL="$(BASE_URL)" ./scripts/test-lab6.sh full diff --git a/REPORT.md b/REPORT.md index b97ae42..c2cdea2 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,37 +1,260 @@ # Отчёт — лабораторная 6 -- Автор, группа: TODO -- Целевая оценка; выбранный трек 5, если нужен: TODO -- Commit/PR: TODO после публикации -- Использованные источники, библиотеки и помощь ИИ: TODO +- Автор, группа: Пластинин Денис, БВТ2303 +- Целевая оценка: 5 +- Выбранный трек: 2 — SLO и обнаружение инцидентов ## Запуск из чистого clone -TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных. +Требуются Docker Desktop с Compose v2, Go из `go.mod`, Python 3.9+ и `curl`. Все образы закреплены в Compose или Dockerfile. Основной observability stack: OpenTelemetry Collector `0.160.0`, Prometheus `3.13.3`, Loki `3.7.9`, Tempo `2.10.10`, Grafana `13.2.4`. -## Архитектура и решения +```bash +cp lab6.env.example .env +docker compose --env-file .env config --quiet +docker compose --env-file .env up -d --build --scale api=2 --wait --wait-timeout 180 +make test +make test-lab5-acceptance +make test-lab5 +make test-lab6-smoke +``` -TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab5, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации. +Интерфейсы локального стенда: + +- API: `http://127.0.0.1:8080`; +- Swagger: `http://127.0.0.1:8081`; +- Grafana: `http://127.0.0.1:3000`; +- Prometheus: `http://127.0.0.1:9090`; +- Loki API: `http://127.0.0.1:3100`; +- Tempo API: `http://127.0.0.1:3200`. + +Безопасная остановка сохраняет volumes: + +```bash +docker compose --env-file .env down +``` + +Намеренное удаление всех учебных данных выполняется отдельно: + +```bash +docker compose --env-file .env down -v +``` + +## Архитектура + +Возможности Lab 5 сохранены: две API-реплики за Nginx, PostgreSQL как источник бизнес-состояния, Valkey для сессий, MinIO/S3 для изображений, transactional outbox, Redis Streams и отдельный worker. Lab 6 добавляет единый telemetry pipeline. + +```text +Client → Nginx → API replicas → PostgreSQL / Valkey / MinIO + │ +PostgreSQL outbox → Dispatcher → Redis Streams → Worker → PostgreSQL / MinIO + │ │ + └──────── OTLP ──────────┤ +API ────────────────────────────── OTLP ──────────┤ + ▼ + OpenTelemetry Collector + ├─ metrics → Prometheus + ├─ logs → Loki + └─ traces → Tempo + │ + ▼ + Grafana +``` + +Приложения экспортируют OTLP только в Collector. Они не знают адреса Loki и Tempo. Collector выполняет `memory_limiter` до `batch`, ограничивает очереди exporter и прекращает retry через 30 секунд. Evidence проверяет queue size/capacity, failures и drops отдельно для `otlp_grpc/tempo` и `otlp_http/loki`, а память — по пиковому RSS за двухминутное окно. Недоступность telemetry backend поэтому не блокирует бизнес-запросы и не вызывает неограниченный рост памяти. + +Prometheus, Loki, Tempo и Grafana используют отдельные volumes. Retention локальных logs/traces — 7 суток; Prometheus хранит 7 суток. Collector является транспортом и процессором, а не долговременным хранилищем. + +### Resource attributes и среды + +Каждый процесс объявляет: + +```text +service.name = messenger-api | messenger-dispatcher | messenger-worker +service.version = значение SERVICE_VERSION +deployment.environment.name = lab +service.instance.id = стабильный ID процесса +``` + +В учебном Compose одна среда `lab`. Атрибут среды всё равно передаётся: один и тот же dashboard и TraceQL остаются пригодны для `test`/`production`, если стенд будет развёрнут там позднее. Среда и service name имеют низкую кардинальность. + +### Middleware и сквозной контекст + +HTTP middleware создаёт server span, нормализует route template, считает request counter/histogram и добавляет в JSON-лог `request_id`, `trace_id` и `span_id`. Реальный UUID из URL не становится label. + +При upload текущий W3C context сохраняется в outbox envelope как `traceparent`/`tracestate`. Dispatcher создаёт producer span. Worker извлекает context и создаёт consumer span. Поэтому один trace показывает цепочку: + +```text +HTTP upload → staging S3 → PostgreSQL/outbox → Redis publish +→ Redis consume → image transform → S3 variants → PostgreSQL ready +``` + +Идентификаторы пользователя, чата, сообщения, вложения, filename, raw URL и request ID запрещены в metric labels. Ограниченные технические ID допустимы в logs/spans синтетического стенда для расследования. `Authorization`, session token, пароль, тело сообщения, байты изображения, S3 credentials и signed URL не экспортируются. + +## Хранилища и назначение сигналов + +| Сигнал | Хранилище | На какой вопрос отвечает | +| --- | --- | --- | +| Метрика | Prometheus TSDB | Есть ли массовая деградация, когда началась и каков масштаб | +| Лог | Loki | Какое событие и ошибка произошли в конкретном процессе | +| Trace | Tempo | На каком участке сквозного запроса потрачено время или возникла ошибка | +| Dashboard / Explore | Grafana | Единый поиск, корреляция и переходы между тремя сигналами | + +Grafana provisioning создаёт Prometheus, Loki и Tempo без ручной настройки. Tempo datasource включает поиск TraceQL, node graph, service map, переходы trace → logs и trace → metrics. Loki derived field открывает trace по `trace_id`; Prometheus exemplars ведут из histogram в Tempo. + +## Карта сигналов + +| Контрактный сигнал | Реализация и единица | +| --- | --- | +| HTTP requests | `messenger_http_requests_total{method,route,status_class}`, requests | +| HTTP duration | `messenger_http_request_duration_seconds_bucket`, seconds | +| Delivery outcomes | `messenger_attachment_worker_jobs_total{job_type,result}`, deliveries | +| Terminal job transitions | `messenger_attachment_terminal_transitions_total{status}`, first durable outcome per job | +| Job duration | `messenger_attachment_worker_job_duration_seconds_bucket`, seconds | +| Time to ready | `messenger_attachment_time_to_ready_seconds_bucket`, seconds | +| Queue states | `messenger_attachment_jobs_count{status}`, jobs | +| Oldest job | `messenger_attachment_jobs_oldest_age_seconds{status}`, seconds | +| Worker saturation | `messenger_attachment_worker_jobs_in_flight` / `messenger_attachment_worker_capacity`, jobs | +| Outbox | `messenger_outbox_pending`, events | +| Logs | OTLP log records: timestamp, severity, service, event, trace/span IDs | +| Traces | HTTP server, producer, consumer, PostgreSQL, Redis and S3 spans | + +p50/p95/p99 вычисляются `histogram_quantile` из суммы bucket всех реплик. Перцентили реплик не усредняются. Queue depth и oldest age показываются отдельно: количество элементов не измеряет длительность ожидания. + +## SLO + +### Доступность API + +SLI — доля бизнес-запросов без `5xx`. Health и `/metrics` исключены. Ожидаемые клиентские `4xx` не считаются отказом сервиса. + +```text +bad = business responses 5xx +total = all business responses +SLI = 1 - bad / total +SLO = 99.5% за 30 дней +error budget = 0.5% +``` + +Цель 99,5% выбрана для учебного single-host стенда без multi-zone HA: она требует реагировать на деградацию, но не заявляет недостижимую для локального Compose доступность. + +### Время готовности вложений + +SLI — доля уникальных durable attachment jobs, чей первый terminal outcome равен `ready` и наступил не позднее 30 секунд после acceptance. Время acceptance назначает PostgreSQL при создании job, поэтому начало и конец интервала принадлежат одному clock domain. PostgreSQL записывает первый outcome в append-only ledger в той же транзакции, что и переход job. Миграция переносит уже завершённые jobs с `accepted_at = created_at` и `terminal_at = updated_at`. Повторная delivery и ручной replay не меняют уже учтённый результат: job, впервые завершившаяся `failed`, остаётся bad event даже после успешного replay. + +```text +bad = failed + ready later than 30s +total = ready + failed +SLI = 1 - bad / total +SLO = 99% за 30 дней +error budget = 1% +``` + +`messenger_attachment_terminal_transitions_total` и cumulative buckets `messenger_attachment_time_to_ready_seconds_bucket` экспортируются из ledger. Bucket `le="+Inf"` является durable количеством первых outcomes `ready` и используется как знаменатель ready jobs. + +30 секунд отделяют нормальную асинхронную обработку от видимой пользователю задержки и существенно превышают обычное время небольшого синтетического изображения. + +### Multi-window burn-rate alerts + +Для обоих SLO используются две пары окон: + +| Alert | Burn rate | Короткое окно | Длинное окно | Назначение | +| --- | ---: | ---: | ---: | --- | +| Fast | 14,4× | 5 минут | 1 час | быстро обнаружить сильный инцидент | +| Slow | 6× | 30 минут | 6 часов | обнаружить устойчивую деградацию | + +Условие должно выполняться в обоих окнах пары. Короткое окно быстро реагирует, длинное отсекает одиночный шум. Отдельные operational alerts контролируют отсутствие worker, рост oldest queued age и Redis Stream lag. + +## Dashboard и поиск + +Dashboard **Messenger Lab 6 overview** содержит: + +- API RPS, error ratio, p50/p95/p99; +- состояния API/worker/dispatcher и DB pool; +- accepted/ready/failed jobs, processing duration и end-to-end ready time; +- queue depth, oldest age, Redis lag/PEL, outbox pending; +- worker busy/capacity; +- SLO и firing alerts; +- Collector export failures и переходы к Explore. + +Поиск trace выполняется в Grafana Explore → Tempo по `service.name`, span name, route, status и duration. Из trace доступен переход в Loki по тому же `trace_id`. Пошаговые процедуры: + +- `monitoring/runbooks/api-availability.md`; +- `monitoring/runbooks/attachment-pipeline.md`. ## Доказательства критериев -| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | +| Критерий | Команда/сценарий | Проверяемый результат | Доказательство | | --- | --- | --- | --- | -| Публичный smoke | make test | TODO | TODO | -| Остальные пункты уровня 3 | TODO | TODO | TODO | -| Все пункты уровня 4, если заявлен | TODO | TODO | TODO | -| Выбранный трек 5, если заявлен | TODO | TODO | TODO | +| Offline contract | `make check` | структура и обязательные файлы не ослаблены | stdout CI | +| OpenAPI | `make validate` | OpenAPI 3.1 проходит полную проверку | stdout CI | +| Публичный smoke | `make test` | бизнес-API предыдущих лабораторных работает | stdout CI | +| Lab 5 regression | `make test-lab5-acceptance && make test-lab5` | durable jobs, retry, replay и idempotency сохранены | stdout CI | +| Три telemetry signal | `make test-lab6-smoke` | нагрузка создаёт metrics/logs/traces, attachments доходят до terminal state | `evidence/runtime/.../baseline-*.json` | +| Worker incident | `scripts/incident-worker.sh short` | alert firing, после запуска worker backlog дренирован и alert resolved | `worker-short-*/` | +| DB incident | `scripts/incident-database.sh` | видны 5xx и DB error spans, burn-rate firing, API восстанавливается, оперативный alert resolved | `database-*/` | +| Длительная деградация | `scripts/incident-worker.sh long` | растут oldest age/lag, срабатывает backlog alert, все jobs затем ready | `worker-long-*/` | +| Telemetry outage | `scripts/incident-telemetry.sh` | API readiness остаётся 200, Collector queue/retry ограничены, backend восстанавливается | `telemetry-*/` | +| Полный трек 2 | `make test-lab6` | выполнены short/long incidents, firing/resolved и schema validation | `evidence/runtime/lab6-*/` | -TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены. +Runtime evidence создаётся локально и игнорируется Git до проверки содержимого. В нём нет session tokens, cookies, passwords, payload envelope и пользовательских текстов. Для публикации выбираются только небольшие агрегированные JSON. -## Эксперименты и отказы +## Методика экспериментов -TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения. +`scripts/lab6-traffic.py` создаёт изолированных синтетических пользователей и чат, затем генерирует сообщения и PNG-вложения. Нагрузка closed-loop, concurrency и duration фиксируются в JSON; HTTP status и p50/p95/p99 рассчитываются клиентом только как вспомогательные данные. Серверные SLO используют Prometheus histograms. -| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | -| --- | --- | --- | --- | -| TODO | TODO | TODO | TODO | +Полный сценарий: -## Ограничения и следующие шаги +```bash +make test-lab6 +``` -TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза. +Он выполняет: + +1. baseline mixed load; +2. короткую остановку worker; +3. краткую недоступность PostgreSQL под message traffic; +4. длительную остановку worker; +5. остановку Loki и Tempo под mixed traffic; +6. проверку evidence и переходов alert `firing → resolved`. + +Каждый incident script устанавливает trap и восстанавливает затронутые сервисы. `docker compose down -v` не используется. Baseline/degraded/recovered snapshots содержат одни и те же PromQL, поэтому сравнение не зависит от ручного выбора панели. + +Гипотезы трека: + +- короткий сильный отказ расходует бюджет быстрее 14,4× и обнаруживается fast alert; +- длительная деградация выявляется ростом oldest queued age и multi-window SLO; +- после восстановления worker backlog уменьшается, все принятые jobs достигают terminal state, alerts переходят в resolved; +- отказ Loki/Tempo не влияет на бизнес-ответы, а ограниченные очереди Collector отражают потери telemetry без неограниченного роста RAM. + +Фактические времена обнаружения и восстановления берутся из `started_at`, `alert-firing.json`, `recovery_started_at` и `alert-resolved.json`. Отсутствие перехода считается падением сценария, а не успешным экспериментом. Ложное срабатывание проверяется baseline snapshot: до fault operational и burn-rate alerts не должны находиться в `firing`. + +## Результаты прогона + +Полный прогон `2026-11-04` сохранён локально в `evidence/runtime/lab6-20261104T174049Z` и прошёл schema validation без ошибок. Обезличенная сводка включена в `evidence/lab6-results.json`. Время обнаружения является верхней границей: сценарий опрашивает alert после выдержки заданной длительности отказа. + +| Инцидент | Наблюдение | Обнаружение | Восстановление | +| --- | --- | ---: | ---: | +| Worker, короткий | `AttachmentWorkerUnavailable` firing, очередь затем обработана | не более 76 с | alert resolved через 13 с, полное наблюдение 33 с | +| PostgreSQL | operational 5xx и fast burn-rate firing; сохранены 3 DB error traces, один связан с API-логом | не более 78 с | operational alert resolved через 134 с, API стабилен через 154 с | +| Worker, длительный | oldest queued age рос; backlog, fast и slow burn alerts firing | не более 181 с | alert resolved через 20 с, backlog обработан за 91 с | +| Loki и Tempo | очереди выросли до 15/512 и 16/512, затем дренировались до нуля; peak RSS 226.7 MB | 30 с отказа | `/health/ready` вернул 200, backends healthy через 31 с | + +Перед первым fault список firing alerts был пуст. Burn-rate alerts после инцидента закономерно остаются активными дольше оперативных alerts: их длинные окна сохраняют уже потраченный error budget. Это не считается ложным срабатыванием следующей фазы. + +## Ограничения + +- Стенд single-host и не проверяет потерю всего Docker host или volume. +- Loki и Tempo работают в single-binary режиме без репликации. +- 30-дневный SLO определяется production-подобными окнами, но учебный эксперимент создаёт искусственно высокий error ratio и не заменяет месячное наблюдение. +- Closed-loop generator подвержен coordinated omission; capacity conclusions по его RPS не делаются. +- Остановка telemetry backend проверяет ограниченное поведение exporter, но не гарантирует отсутствие потерь: после заполнения bounded queue часть telemetry намеренно отбрасывается. +- Внешняя доставка alerts не настраивается; проверяются локальные состояния Prometheus `firing` и `resolved`. + +## Источники + +ИИ-ассистент использовался при подготовке кода, конфигурации и текста; результат проверен тестами, CI-сценариями и воспроизводимым fault-прогоном. + +- OpenTelemetry Collector configuration и Go SDK; +- W3C Trace Context; +- Prometheus histograms и multi-window burn-rate alerts; +- Grafana provisioning, Loki OTLP ingestion и Tempo TraceQL; +- документация Redis Streams, PostgreSQL, MinIO и Docker Compose. diff --git a/cmd/dependencycheck/main.go b/cmd/dependencycheck/main.go new file mode 100644 index 0000000..eaa3d64 --- /dev/null +++ b/cmd/dependencycheck/main.go @@ -0,0 +1,330 @@ +package main + +import ( + "bytes" + "encoding/json" + "flag" + "fmt" + "io" + "net/http" + "os" + "os/exec" + "strings" + "sync" + "time" + + "github.com/google/uuid" +) + +type apiClient struct { + baseURL string + client *http.Client + token string + userID string +} + +func main() { + if err := run(); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + fmt.Println("PASS: replica idempotency and dependency recovery") +} + +func run() error { + baseURL := flag.String("base-url", "http://localhost:8080", "API base URL") + flag.Parse() + client := &apiClient{baseURL: *baseURL, client: &http.Client{Timeout: 10 * time.Second}} + if err := client.registerAndLogin(); err != nil { + return err + } + if err := client.expect(http.MethodGet, "/api/v1/users/me", http.StatusOK, true); err != nil { + return err + } + if err := client.verifyConcurrentIdempotency(); err != nil { + return err + } + if err := verifyOutage(client, "valkey"); err != nil { + return err + } + return verifyOutage(client, "postgres") +} + +func (c *apiClient) registerAndLogin() error { + username := "dep_" + strings.ReplaceAll(uuid.NewString(), "-", "")[:16] + password := "T3st-" + uuid.NewString() + body := map[string]string{"username": username, "display_name": "Dependency check", "password": password} + payload, err := json.Marshal(body) + if err != nil { + return err + } + request, err := http.NewRequest(http.MethodPost, c.baseURL+"/api/v1/users", bytes.NewReader(payload)) + if err != nil { + return err + } + request.Header.Set("Content-Type", "application/json") + response, err := c.client.Do(request) + if err != nil { + return fmt.Errorf("register: %w", err) + } + registrationBody := response.Body + defer func() { _ = registrationBody.Close() }() + if response.StatusCode != http.StatusCreated { + return fmt.Errorf("register status = %d", response.StatusCode) + } + var user struct { + ID string `json:"id"` + } + if err := json.NewDecoder(io.LimitReader(response.Body, 1<<20)).Decode(&user); err != nil { + return fmt.Errorf("decode user: %w", err) + } + if user.ID == "" { + return fmt.Errorf("register returned an empty user id") + } + c.userID = user.ID + + request, err = http.NewRequest(http.MethodPost, c.baseURL+"/api/v1/auth/sessions", nil) + if err != nil { + return err + } + request.SetBasicAuth(username, password) + response, err = c.client.Do(request) + if err != nil { + return fmt.Errorf("login: %w", err) + } + defer func() { _ = response.Body.Close() }() + if response.StatusCode != http.StatusCreated { + return fmt.Errorf("login status = %d", response.StatusCode) + } + var session struct { + Token string `json:"token"` + } + if err := json.NewDecoder(io.LimitReader(response.Body, 1<<20)).Decode(&session); err != nil { + return fmt.Errorf("decode session: %w", err) + } + if session.Token == "" { + return fmt.Errorf("login returned an empty token") + } + c.token = session.Token + return nil +} + +type messageResult struct { + messageID string + instanceID string + err error +} + +func (c *apiClient) verifyConcurrentIdempotency() error { + chatID, err := c.createChat() + if err != nil { + return err + } + + const workers = 16 + var ( + key = uuid.NewString() + start = make(chan struct{}) + results = make(chan messageResult, workers) + group sync.WaitGroup + instances = make(map[string]struct{}) + messageID string + ) + transport := &http.Transport{DisableKeepAlives: true, MaxConnsPerHost: workers} + client := &http.Client{Transport: transport, Timeout: 10 * time.Second} + defer transport.CloseIdleConnections() + + for range workers { + group.Add(1) + go func() { + defer group.Done() + <-start + id, instance, requestErr := c.createMessage(client, chatID, key, "concurrent message") + results <- messageResult{messageID: id, instanceID: instance, err: requestErr} + }() + } + close(start) + group.Wait() + close(results) + + for result := range results { + if result.err != nil { + return result.err + } + if result.instanceID == "" { + return fmt.Errorf("message response has an empty X-Instance-Id") + } + instances[result.instanceID] = struct{}{} + if messageID == "" { + messageID = result.messageID + } + if result.messageID != messageID { + return fmt.Errorf("idempotent responses returned different message ids") + } + } + if len(instances) < 2 { + return fmt.Errorf("requests reached %d API instance, want at least 2", len(instances)) + } + return c.verifyMessageHistory(chatID, messageID) +} + +func (c *apiClient) createChat() (string, error) { + payload, err := json.Marshal(map[string]any{ + "title": "Replica check", "member_ids": []string{c.userID}, + }) + if err != nil { + return "", err + } + request, err := http.NewRequest(http.MethodPost, c.baseURL+"/api/v1/chats", bytes.NewReader(payload)) + if err != nil { + return "", err + } + request.Header.Set("Authorization", "Bearer "+c.token) + request.Header.Set("Content-Type", "application/json") + response, err := c.client.Do(request) + if err != nil { + return "", fmt.Errorf("create chat: %w", err) + } + defer func() { _ = response.Body.Close() }() + if response.StatusCode != http.StatusCreated { + return "", fmt.Errorf("create chat status = %d", response.StatusCode) + } + var chat struct { + ID string `json:"id"` + } + if err := json.NewDecoder(io.LimitReader(response.Body, 1<<20)).Decode(&chat); err != nil { + return "", fmt.Errorf("decode chat: %w", err) + } + return chat.ID, nil +} + +func (c *apiClient) createMessage(client *http.Client, chatID, key, text string) (string, string, error) { + payload, err := json.Marshal(map[string]string{"text": text}) + if err != nil { + return "", "", err + } + request, err := http.NewRequest( + http.MethodPost, c.baseURL+"/api/v1/chats/"+chatID+"/messages", bytes.NewReader(payload), + ) + if err != nil { + return "", "", err + } + request.Header.Set("Authorization", "Bearer "+c.token) + request.Header.Set("Content-Type", "application/json") + request.Header.Set("Idempotency-Key", key) + response, err := client.Do(request) + if err != nil { + return "", "", fmt.Errorf("create message: %w", err) + } + defer func() { _ = response.Body.Close() }() + if response.StatusCode != http.StatusCreated { + return "", response.Header.Get("X-Instance-Id"), fmt.Errorf("create message status = %d", response.StatusCode) + } + var message struct { + ID string `json:"id"` + } + if err := json.NewDecoder(io.LimitReader(response.Body, 1<<20)).Decode(&message); err != nil { + return "", response.Header.Get("X-Instance-Id"), fmt.Errorf("decode message: %w", err) + } + return message.ID, response.Header.Get("X-Instance-Id"), nil +} + +func (c *apiClient) verifyMessageHistory(chatID, messageID string) error { + request, err := http.NewRequest(http.MethodGet, c.baseURL+"/api/v1/chats/"+chatID+"/messages?limit=100", nil) + if err != nil { + return err + } + request.Header.Set("Authorization", "Bearer "+c.token) + response, err := c.client.Do(request) + if err != nil { + return fmt.Errorf("list messages: %w", err) + } + defer func() { _ = response.Body.Close() }() + if response.StatusCode != http.StatusOK { + return fmt.Errorf("list messages status = %d", response.StatusCode) + } + var page struct { + Items []struct { + ID string `json:"id"` + } `json:"items"` + } + if err := json.NewDecoder(io.LimitReader(response.Body, 1<<20)).Decode(&page); err != nil { + return fmt.Errorf("decode messages: %w", err) + } + if len(page.Items) != 1 || page.Items[0].ID != messageID { + return fmt.Errorf("message history contains %d items with an unexpected id", len(page.Items)) + } + return nil +} + +func verifyOutage(client *apiClient, service string) (returnErr error) { + if err := compose("stop", service); err != nil { + return err + } + defer func() { + if err := compose("up", "-d", "--wait", "--wait-timeout", "90", service); returnErr == nil && err != nil { + returnErr = err + } + }() + if err := client.expectEventually("/health/ready", http.StatusServiceUnavailable, false); err != nil { + return fmt.Errorf("%s readiness outage: %w", service, err) + } + if err := client.expectEventually("/api/v1/users/me", http.StatusServiceUnavailable, true); err != nil { + return fmt.Errorf("%s business outage: %w", service, err) + } + if err := compose("up", "-d", "--wait", "--wait-timeout", "90", service); err != nil { + return err + } + if err := client.expectEventually("/health/ready", http.StatusOK, false); err != nil { + return fmt.Errorf("%s readiness recovery: %w", service, err) + } + if err := client.expectEventually("/api/v1/users/me", http.StatusOK, true); err != nil { + return fmt.Errorf("%s session recovery: %w", service, err) + } + return nil +} + +func compose(arguments ...string) error { + command := exec.Command("docker", append([]string{"compose"}, arguments...)...) + if output, err := command.CombinedOutput(); err != nil { + return fmt.Errorf("docker compose %v: %w: %s", arguments, err, bytes.TrimSpace(output)) + } + return nil +} + +func (c *apiClient) expectEventually(path string, status int, authenticated bool) error { + deadline := time.Now().Add(20 * time.Second) + var lastErr error + for time.Now().Before(deadline) { + lastErr = c.expect(http.MethodGet, path, status, authenticated) + if lastErr == nil { + return nil + } + time.Sleep(200 * time.Millisecond) + } + return lastErr +} + +func (c *apiClient) expect(method, path string, status int, authenticated bool) error { + request, err := http.NewRequest(method, c.baseURL+path, nil) + if err != nil { + return err + } + if authenticated { + request.Header.Set("Authorization", "Bearer "+c.token) + } + response, err := c.client.Do(request) + if err != nil { + return err + } + return closeResponse(response, status) +} + +func closeResponse(response *http.Response, status int) error { + defer func() { _ = response.Body.Close() }() + _, _ = io.Copy(io.Discard, io.LimitReader(response.Body, 1<<20)) + if response.StatusCode != status { + return fmt.Errorf("status = %d, want %d", response.StatusCode, status) + } + return nil +} diff --git a/cmd/lab5experiment/api_scenario.go b/cmd/lab5experiment/api_scenario.go new file mode 100644 index 0000000..048a121 --- /dev/null +++ b/cmd/lab5experiment/api_scenario.go @@ -0,0 +1,298 @@ +package main + +import ( + "bytes" + "context" + "encoding/base64" + "encoding/json" + "errors" + "flag" + "fmt" + "image" + _ "image/jpeg" + "io" + "net/http" + "strings" + "time" + + "github.com/google/uuid" + + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model" +) + +type apiState struct { + BaseURL string `json:"base_url"` + Token string `json:"token"` + AttachmentID string `json:"attachment_id"` +} + +type apiUser struct { + ID string `json:"id"` +} + +type apiSession struct { + Token string `json:"token"` +} + +type apiChat struct { + ID string `json:"id"` +} + +func runAPIProduce(arguments []string, output io.Writer) error { + flags := flag.NewFlagSet("api-produce", flag.ContinueOnError) + flags.SetOutput(io.Discard) + baseURL := flags.String("base-url", "http://nginx:8080", "messenger base URL") + if err := flags.Parse(arguments); err != nil || flags.NArg() != 0 { + return errors.New("usage: lab5experiment api-produce [-base-url URL]") + } + client := &http.Client{Timeout: 10 * time.Second} + suffix := strings.ReplaceAll(uuid.NewString(), "-", "")[:12] + password := "Lab5-" + uuid.NewString() + owner, err := registerAPIUser(client, *baseURL, "qa_"+suffix, password) + if err != nil { + return err + } + member, err := registerAPIUser(client, *baseURL, "qb_"+suffix, password) + if err != nil { + return err + } + token, err := createAPISession(client, *baseURL, "qa_"+suffix, password) + if err != nil { + return err + } + chat, err := createAPIChat(client, *baseURL, token, owner.ID, member.ID) + if err != nil { + return err + } + fixture, err := pngFixture(320, 200) + if err != nil { + return err + } + var attachment model.Attachment + response, err := apiRequest( + client, http.MethodPost, + *baseURL+"/api/v1/chats/"+chat.ID+"/attachments?filename=queued.png", + token, "image/png", fixture, &attachment, + ) + if err != nil { + return err + } + if response.StatusCode != http.StatusAccepted || attachment.Status != model.AttachmentQueued { + return fmt.Errorf("upload returned status=%d attachment_status=%s", response.StatusCode, attachment.Status) + } + messageBody, err := json.Marshal(map[string]any{"text": "", "attachment_ids": []string{attachment.ID}}) + if err != nil { + return err + } + response, err = apiRequestWithHeaders( + client, http.MethodPost, *baseURL+"/api/v1/chats/"+chat.ID+"/messages", + token, "application/json", messageBody, nil, + map[string]string{"Idempotency-Key": uuid.NewString()}, + ) + if err != nil { + return err + } + if response.StatusCode != http.StatusCreated { + return fmt.Errorf("link queued attachment returned %d", response.StatusCode) + } + response, err = apiRequest(client, http.MethodGet, *baseURL+"/api/v1/attachments/"+attachment.ID+"/content", token, "", nil, nil) + if err != nil { + return err + } + if response.StatusCode != http.StatusConflict { + return fmt.Errorf("queued content returned %d, expected 409", response.StatusCode) + } + return json.NewEncoder(output).Encode(apiState{BaseURL: *baseURL, Token: token, AttachmentID: attachment.ID}) +} + +func runAPIVerify(arguments []string, input io.Reader, output io.Writer) error { + flags := flag.NewFlagSet("api-verify", flag.ContinueOnError) + flags.SetOutput(io.Discard) + waitTimeout := flags.Duration("wait-timeout", time.Minute, "ready wait timeout") + if err := flags.Parse(arguments); err != nil || flags.NArg() != 0 { + return errors.New("usage: lab5experiment api-verify [-wait-timeout 1m]") + } + var state apiState + if err := json.NewDecoder(input).Decode(&state); err != nil { + return fmt.Errorf("decode API scenario state: %w", err) + } + client := &http.Client{Timeout: 10 * time.Second} + ctx, cancel := context.WithTimeout(context.Background(), *waitTimeout) + defer cancel() + var attachment model.Attachment + for { + response, err := apiRequest(client, http.MethodGet, state.BaseURL+"/api/v1/attachments/"+state.AttachmentID, state.Token, "", nil, &attachment) + if err != nil { + return err + } + if response.StatusCode != http.StatusOK { + return fmt.Errorf("attachment metadata returned %d", response.StatusCode) + } + if attachment.Status == model.AttachmentReady { + break + } + if attachment.Status == model.AttachmentFailed { + return fmt.Errorf("attachment failed with %v", attachment.ErrorCode) + } + if err := wait(ctx, 250*time.Millisecond); err != nil { + return err + } + } + if len(attachment.Variants) != 3 { + return fmt.Errorf("public variants = %d, expected 3", len(attachment.Variants)) + } + response, original, err := apiBytes(client, state.BaseURL+"/api/v1/attachments/"+state.AttachmentID+"/content", state.Token) + if err != nil { + return err + } + fixture, err := pngFixture(320, 200) + if err != nil { + return err + } + if response.StatusCode != http.StatusOK || !bytes.Equal(original, fixture) { + return errors.New("original changed after worker and broker restart") + } + response, thumbnail, err := apiBytes( + client, state.BaseURL+"/api/v1/attachments/"+state.AttachmentID+"/content?variant=thumbnail", state.Token, + ) + if err != nil { + return err + } + if response.StatusCode != http.StatusOK || response.Header.Get("Content-Type") != "image/jpeg" { + return errors.New("thumbnail response is invalid") + } + imageConfig, format, err := image.DecodeConfig(bytes.NewReader(thumbnail)) + if err != nil || format != "jpeg" || imageConfig.Width != 256 || imageConfig.Height != 160 { + return errors.New("thumbnail dimensions are invalid") + } + return json.NewEncoder(output).Encode(map[string]any{ + "accepted_while_worker_stopped": true, + "accepted_before_outbox_publish": true, + "linked_while_queued": true, + "content_conflict_while_queued": true, + "ready_after_api_broker_restart": true, + "original_preserved": true, + "thumbnail_dimensions": "256x160", + }) +} + +func registerAPIUser(client *http.Client, baseURL, username, password string) (apiUser, error) { + body, err := json.Marshal(map[string]string{ + "username": username, "display_name": username, "password": password, + }) + if err != nil { + return apiUser{}, err + } + var user apiUser + response, err := apiRequest(client, http.MethodPost, baseURL+"/api/v1/users", "", "application/json", body, &user) + if err != nil { + return apiUser{}, err + } + if response.StatusCode != http.StatusCreated { + return apiUser{}, fmt.Errorf("register user returned %d", response.StatusCode) + } + return user, nil +} + +func createAPISession(client *http.Client, baseURL, username, password string) (string, error) { + credentials := base64.StdEncoding.EncodeToString([]byte(username + ":" + password)) + request, err := http.NewRequest(http.MethodPost, baseURL+"/api/v1/auth/sessions", nil) + if err != nil { + return "", err + } + request.Header.Set("Authorization", "Basic "+credentials) + response, err := client.Do(request) + if err != nil { + return "", err + } + defer func() { _ = response.Body.Close() }() + if response.StatusCode != http.StatusCreated { + return "", fmt.Errorf("create session returned %d", response.StatusCode) + } + var session apiSession + if err := json.NewDecoder(io.LimitReader(response.Body, 1<<20)).Decode(&session); err != nil { + return "", err + } + return session.Token, nil +} + +func createAPIChat(client *http.Client, baseURL, token, ownerID, memberID string) (apiChat, error) { + body, err := json.Marshal(map[string]any{ + "title": "Lab 5 resilience", "member_ids": []string{ownerID, memberID}, + }) + if err != nil { + return apiChat{}, err + } + var chat apiChat + response, err := apiRequest(client, http.MethodPost, baseURL+"/api/v1/chats", token, "application/json", body, &chat) + if err != nil { + return apiChat{}, err + } + if response.StatusCode != http.StatusCreated { + return apiChat{}, fmt.Errorf("create chat returned %d", response.StatusCode) + } + return chat, nil +} + +func apiRequest( + client *http.Client, + method string, + requestURL string, + token string, + contentType string, + body []byte, + result any, +) (*http.Response, error) { + return apiRequestWithHeaders(client, method, requestURL, token, contentType, body, result, nil) +} + +func apiRequestWithHeaders( + client *http.Client, + method string, + requestURL string, + token string, + contentType string, + body []byte, + result any, + headers map[string]string, +) (*http.Response, error) { + request, err := http.NewRequest(method, requestURL, bytes.NewReader(body)) + if err != nil { + return nil, err + } + if token != "" { + request.Header.Set("Authorization", "Bearer "+token) + } + if contentType != "" { + request.Header.Set("Content-Type", contentType) + } + for name, value := range headers { + request.Header.Set(name, value) + } + response, err := client.Do(request) + if err != nil { + return nil, err + } + defer func() { _ = response.Body.Close() }() + if result != nil && response.StatusCode < http.StatusBadRequest { + if err := json.NewDecoder(io.LimitReader(response.Body, 16<<20)).Decode(result); err != nil { + return nil, err + } + } + return response, nil +} + +func apiBytes(client *http.Client, requestURL, token string) (*http.Response, []byte, error) { + request, err := http.NewRequest(http.MethodGet, requestURL, nil) + if err != nil { + return nil, nil, err + } + request.Header.Set("Authorization", "Bearer "+token) + response, err := client.Do(request) + if err != nil { + return nil, nil, err + } + defer func() { _ = response.Body.Close() }() + body, err := io.ReadAll(io.LimitReader(response.Body, 16<<20)) + return response, body, err +} diff --git a/cmd/lab5experiment/main.go b/cmd/lab5experiment/main.go new file mode 100644 index 0000000..35c5241 --- /dev/null +++ b/cmd/lab5experiment/main.go @@ -0,0 +1,607 @@ +package main + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "flag" + "fmt" + "image" + "image/color" + "image/png" + "io" + "log/slog" + "os" + "strings" + "time" + + "github.com/google/uuid" + "github.com/redis/go-redis/v9" + + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/broker/redisstream" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/config" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/database" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/messenger/attachment/transform" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/messenger/attachment/worker" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/objectstore" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/outbox" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/postgres" +) + +const ( + experimentLease = 2 * time.Second + maxAttempts = 5 +) + +type result struct { + StartedAt string `json:"started_at"` + DurationMillis int64 `json:"duration_ms"` + PipelineVersion int `json:"pipeline_version"` + FaultScenarios []scenario `json:"fault_scenarios"` + DuplicateDeliveries int `json:"duplicate_deliveries"` + V1ObjectCount int `json:"v1_object_count"` + V2ObjectCount int `json:"v2_object_count"` + AcceptedJobs int `json:"accepted_jobs"` + TerminalJobs int `json:"terminal_jobs"` + ReadyJobs int `json:"ready_jobs"` + FailedJobs int `json:"failed_jobs"` + PermanentErrorCode string `json:"permanent_error_code"` + ReplayAttempt int `json:"replay_attempt"` + InvariantResults map[string]bool `json:"invariant_results"` +} + +type scenario struct { + Name string `json:"name"` + AttemptAfterRecovery int `json:"attempt_after_recovery"` + ObjectsBeforeRecovery int `json:"objects_before_recovery"` + ObjectsAfterRecovery int `json:"objects_after_recovery"` + FinalStatus string `json:"final_status"` +} + +type experiment struct { + settings config.WorkerConfig + repository *postgres.Store + objects *objectstore.Store + broker *redisstream.Broker + dispatcher *outbox.Dispatcher + worker *worker.Worker + ownerID string + chatID string + fixture []byte +} + +func main() { + if err := run(os.Args[1:], os.Stdout); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } +} + +func run(arguments []string, output io.Writer) error { + if len(arguments) > 0 { + switch arguments[0] { + case "api-produce": + return runAPIProduce(arguments[1:], output) + case "api-verify": + return runAPIVerify(arguments[1:], os.Stdin, output) + } + } + flags := flag.NewFlagSet("lab5experiment", flag.ContinueOnError) + flags.SetOutput(io.Discard) + timeout := flags.Duration("timeout", 2*time.Minute, "experiment timeout") + if err := flags.Parse(arguments); err != nil || flags.NArg() != 0 || *timeout <= 0 { + return errors.New("usage: lab5experiment [-timeout 2m]") + } + started := time.Now() + ctx, cancel := context.WithTimeout(context.Background(), *timeout) + defer cancel() + experiment, closeExperiment, err := newExperiment(ctx) + if err != nil { + return err + } + defer closeExperiment() + + report, err := experiment.run(ctx) + if err != nil { + return err + } + report.StartedAt = started.UTC().Format(time.RFC3339) + report.DurationMillis = time.Since(started).Milliseconds() + encoder := json.NewEncoder(output) + encoder.SetIndent("", " ") + return encoder.Encode(report) +} + +func newExperiment(ctx context.Context) (*experiment, func(), error) { + settings, err := config.LoadWorker() + if err != nil { + return nil, nil, err + } + pool, err := database.Open(ctx, database.Config{ + URL: settings.DatabaseURL, MinConnections: settings.DBMinConns, MaxConnections: settings.DBMaxConns, + MaxConnectionAge: settings.DBMaxConnectionAge, MaxConnectionIdle: settings.DBMaxConnectionIdle, + HealthCheckPeriod: settings.DBHealthCheckPeriod, ConnectTimeout: settings.DependencyTimeout, + }) + if err != nil { + return nil, nil, err + } + objects, err := objectstore.New(objectstore.Config{ + Endpoint: settings.S3Endpoint, Region: settings.S3Region, Bucket: settings.S3Bucket, + AccessKey: settings.S3AccessKey, SecretKey: settings.S3SecretKey, + UseTLS: settings.S3UseTLS, Timeout: settings.S3Timeout, + }) + if err != nil { + pool.Close() + return nil, nil, err + } + redisOptions, err := redis.ParseURL(settings.BrokerURL) + if err != nil { + pool.Close() + return nil, nil, err + } + client := redis.NewClient(redisOptions) + if err := client.Ping(ctx).Err(); err != nil { + pool.Close() + _ = client.Close() + return nil, nil, err + } + consumer := "lab5experiment-" + uuid.NewString() + broker, err := redisstream.New(client, redisstream.Config{ + Stream: settings.Stream, Group: settings.ConsumerGroup, Consumer: consumer, + FailedStream: settings.FailedStream, FailedMaxLen: settings.FailedStreamMaxLength, Block: time.Second, + ClaimMinIdle: experimentLease, ReadCount: 32, + }) + if err != nil { + pool.Close() + _ = client.Close() + return nil, nil, err + } + if err := broker.Ensure(ctx); err != nil { + pool.Close() + _ = client.Close() + return nil, nil, err + } + repository := postgres.New(pool) + dispatcher, err := outbox.New(repository, broker, nil, outbox.Config{ + BatchSize: 32, LeaseDuration: 10 * time.Second, PollInterval: time.Second, + RecoveryInterval: 10 * time.Second, RetryDelay: time.Second, + Retention: 24 * time.Hour, CleanupInterval: time.Hour, CleanupBatchSize: 1000, + }) + if err != nil { + pool.Close() + _ = client.Close() + return nil, nil, err + } + processor, err := transform.NewProcessor(transform.DefaultConfig(settings.AttachmentMaxBytes, settings.AttachmentMaxPixels)) + if err != nil { + pool.Close() + _ = client.Close() + return nil, nil, err + } + processorWorker, err := worker.New( + slog.New(slog.NewTextHandler(io.Discard, nil)), repository, broker, objects, processor, nil, + worker.Config{ + WorkerID: consumer, Concurrency: 1, + LeaseDuration: experimentLease, HeartbeatInterval: 500 * time.Millisecond, + ProcessingTimeout: time.Second, + ReclaimInterval: experimentLease, + MaxAttempts: maxAttempts, RetryBase: time.Second, RetryMax: 4 * time.Second, + MaxBytes: settings.AttachmentMaxBytes, MaxPixels: settings.AttachmentMaxPixels, + MemoryBytes: settings.MemoryBytes, + }, + ) + if err != nil { + pool.Close() + _ = client.Close() + return nil, nil, err + } + fixture, err := pngFixture(320, 200) + if err != nil { + pool.Close() + _ = client.Close() + return nil, nil, err + } + value := &experiment{ + settings: settings, repository: repository, objects: objects, broker: broker, + dispatcher: dispatcher, worker: processorWorker, fixture: fixture, + } + if err := value.seed(ctx); err != nil { + pool.Close() + _ = client.Close() + return nil, nil, err + } + closeExperiment := func() { + pool.Close() + _ = client.Close() + } + return value, closeExperiment, nil +} + +func (e *experiment) seed(ctx context.Context) error { + suffix := strings.ReplaceAll(uuid.NewString(), "-", "")[:12] + owner, err := e.repository.CreateUser(ctx, "l5a_"+suffix, "Lab 5 A", "$argon2id$experiment") + if err != nil { + return err + } + member, err := e.repository.CreateUser(ctx, "l5b_"+suffix, "Lab 5 B", "$argon2id$experiment") + if err != nil { + return err + } + chat, err := e.repository.CreateChat(ctx, "Lab 5 experiment", []string{owner.ID, member.ID}, owner.ID) + if err != nil { + return err + } + e.ownerID = owner.ID + e.chatID = chat.ID + return nil +} + +func (e *experiment) run(ctx context.Context) (result, error) { + report := result{ + PipelineVersion: 1, + InvariantResults: map[string]bool{ + "accepted_jobs_reach_terminal_state": false, + "duplicate_effect_is_unique": false, + "old_pipeline_cannot_replace_new": false, + }, + } + for _, fault := range []worker.FaultPoint{worker.FaultBeforeSideEffect, worker.FaultAfterSideEffect} { + observed, err := e.runLeaseRecovery(ctx, fault) + if err != nil { + return result{}, err + } + report.FaultScenarios = append(report.FaultScenarios, observed) + report.AcceptedJobs++ + if observed.FinalStatus == model.AttachmentReady { + report.ReadyJobs++ + report.TerminalJobs++ + } + } + permanentCode, replayAttempt, err := e.runPermanentFailureReplay(ctx) + if err != nil { + return result{}, err + } + report.AcceptedJobs++ + report.TerminalJobs++ + report.FailedJobs++ + report.PermanentErrorCode = permanentCode + report.ReplayAttempt = replayAttempt + + commitScenario, envelope, attachmentID, err := e.runAfterCommit(ctx) + if err != nil { + return result{}, err + } + report.FaultScenarios = append(report.FaultScenarios, commitScenario) + report.AcceptedJobs++ + if commitScenario.FinalStatus == model.AttachmentReady { + report.ReadyJobs++ + report.TerminalJobs++ + } + + payload, err := redisstream.EncodeEnvelope(envelope) + if err != nil { + return result{}, err + } + for range 2 { + if _, err := e.broker.PublishConfirmed(ctx, payload); err != nil { + return result{}, err + } + } + duplicateDeliveries, err := e.readDeliveries(ctx, 2) + if err != nil { + return result{}, err + } + for _, delivery := range duplicateDeliveries { + if err := e.worker.ProcessDelivery(ctx, delivery, worker.FaultNone); err != nil { + return result{}, err + } + } + report.DuplicateDeliveries = len(duplicateDeliveries) + report.V1ObjectCount, err = e.objectCount(ctx, attachmentID) + if err != nil { + return result{}, err + } + if report.V1ObjectCount != 4 { + return result{}, fmt.Errorf("duplicate delivery produced %d objects, expected 4", report.V1ObjectCount) + } + report.InvariantResults["duplicate_effect_is_unique"] = true + + oldPayload := payload + v2Job := model.AttachmentJob{ID: uuid.NewString(), PipelineVersion: envelope.PipelineVersion + 1} + if _, err := e.repository.QueueAttachmentPipelineVersion(ctx, attachmentID, v2Job, "", ""); err != nil { + return result{}, err + } + if _, err := e.dispatcher.DispatchOnce(ctx); err != nil { + return result{}, err + } + v2Delivery, err := e.readDelivery(ctx) + if err != nil { + return result{}, err + } + if err := e.worker.ProcessDelivery(ctx, v2Delivery, worker.FaultNone); err != nil { + return result{}, err + } + if _, err := e.broker.PublishConfirmed(ctx, oldPayload); err != nil { + return result{}, err + } + staleDelivery, err := e.readDelivery(ctx) + if err != nil { + return result{}, err + } + if err := e.worker.ProcessDelivery(ctx, staleDelivery, worker.FaultNone); err != nil { + return result{}, err + } + metadata, err := e.repository.AttachmentForUser(ctx, attachmentID, e.ownerID) + if err != nil { + return result{}, err + } + if metadata.Status != model.AttachmentReady || metadata.PipelineVersion != v2Job.PipelineVersion || len(metadata.Variants) != 3 { + return result{}, errors.New("new pipeline version was not preserved after stale delivery") + } + report.V2ObjectCount, err = e.objectCount(ctx, attachmentID) + if err != nil { + return result{}, err + } + if report.V2ObjectCount != 8 { + return result{}, fmt.Errorf("versioned pipeline produced %d objects, expected 8", report.V2ObjectCount) + } + report.InvariantResults["old_pipeline_cannot_replace_new"] = true + report.AcceptedJobs++ + report.ReadyJobs++ + report.TerminalJobs++ + report.InvariantResults["accepted_jobs_reach_terminal_state"] = report.AcceptedJobs == report.TerminalJobs + return report, nil +} + +func (e *experiment) runPermanentFailureReplay(ctx context.Context) (string, int, error) { + attachmentID, _, _, err := e.queueAttachmentData(ctx, []byte("corrupted image"), "image/png") + if err != nil { + return "", 0, err + } + delivery, err := e.readDelivery(ctx) + if err != nil { + return "", 0, err + } + if err := e.worker.ProcessDelivery(ctx, delivery, worker.FaultNone); err != nil { + return "", 0, err + } + metadata, err := e.repository.AttachmentForUser(ctx, attachmentID, e.ownerID) + if err != nil { + return "", 0, err + } + if metadata.Status != model.AttachmentFailed || metadata.ErrorCode == nil { + return "", 0, errors.New("corrupted staging object did not reach failed") + } + firstCode := *metadata.ErrorCode + replayed, err := e.repository.ReplayFailedAttachment( + ctx, attachmentID, "lab5experiment", "verify explicit replay fencing", + ) + if err != nil { + return "", 0, err + } + if _, err := e.dispatcher.DispatchOnce(ctx); err != nil { + return "", 0, err + } + replayDelivery, err := e.readDelivery(ctx) + if err != nil { + return "", 0, err + } + if err := e.worker.ProcessDelivery(ctx, replayDelivery, worker.FaultNone); err != nil { + return "", 0, err + } + metadata, err = e.repository.AttachmentForUser(ctx, attachmentID, e.ownerID) + if err != nil { + return "", 0, err + } + if metadata.Status != model.AttachmentFailed || metadata.ErrorCode == nil || *metadata.ErrorCode != firstCode { + return "", 0, errors.New("replayed permanent failure changed terminal classification") + } + return firstCode, replayed.Attempt, nil +} + +func (e *experiment) runLeaseRecovery(ctx context.Context, fault worker.FaultPoint) (scenario, error) { + attachmentID, _, err := e.queueAttachment(ctx) + if err != nil { + return scenario{}, err + } + delivery, err := e.readDelivery(ctx) + if err != nil { + return scenario{}, err + } + if err := e.worker.ProcessDelivery(ctx, delivery, fault); !errors.Is(err, worker.ErrInjectedFailure) { + return scenario{}, fmt.Errorf("fault %s returned %v", fault, err) + } + before, err := e.objectCount(ctx, attachmentID) + if err != nil { + return scenario{}, err + } + if fault == worker.FaultBeforeSideEffect && before != 0 { + return scenario{}, errors.New("objects were written before the injected side effect") + } + if fault == worker.FaultAfterSideEffect && before != 4 { + return scenario{}, fmt.Errorf("side effect wrote %d objects, expected 4", before) + } + if err := wait(ctx, experimentLease+250*time.Millisecond); err != nil { + return scenario{}, err + } + recovered, err := e.repository.RecoverExpiredAttachmentJobs(ctx, 10, maxAttempts, time.Now()) + if err != nil { + return scenario{}, err + } + if recovered.Requeued != 1 { + return scenario{}, fmt.Errorf("requeued jobs = %d, expected 1", recovered.Requeued) + } + if _, err := e.dispatcher.DispatchOnce(ctx); err != nil { + return scenario{}, err + } + retry, err := e.readDelivery(ctx) + if err != nil { + return scenario{}, err + } + if err := e.worker.ProcessDelivery(ctx, retry, worker.FaultNone); err != nil { + return scenario{}, err + } + if err := e.ackReclaimed(ctx); err != nil { + return scenario{}, err + } + metadata, err := e.repository.AttachmentForUser(ctx, attachmentID, e.ownerID) + if err != nil { + return scenario{}, err + } + after, err := e.objectCount(ctx, attachmentID) + if err != nil { + return scenario{}, err + } + job, err := e.repository.AttachmentJob(ctx, retry.Envelope.JobID.String()) + if err != nil { + return scenario{}, err + } + return scenario{ + Name: string(fault), AttemptAfterRecovery: job.Attempt, + ObjectsBeforeRecovery: before, ObjectsAfterRecovery: after, FinalStatus: metadata.Status, + }, nil +} + +func (e *experiment) runAfterCommit(ctx context.Context) (scenario, redisstream.Envelope, string, error) { + attachmentID, _, err := e.queueAttachment(ctx) + if err != nil { + return scenario{}, redisstream.Envelope{}, "", err + } + delivery, err := e.readDelivery(ctx) + if err != nil { + return scenario{}, redisstream.Envelope{}, "", err + } + if err := e.worker.ProcessDelivery(ctx, delivery, worker.FaultAfterCommit); !errors.Is(err, worker.ErrInjectedFailure) { + return scenario{}, redisstream.Envelope{}, "", fmt.Errorf("after_commit returned %v", err) + } + metadata, err := e.repository.AttachmentForUser(ctx, attachmentID, e.ownerID) + if err != nil { + return scenario{}, redisstream.Envelope{}, "", err + } + before, err := e.objectCount(ctx, attachmentID) + if err != nil { + return scenario{}, redisstream.Envelope{}, "", err + } + if err := wait(ctx, experimentLease+250*time.Millisecond); err != nil { + return scenario{}, redisstream.Envelope{}, "", err + } + if err := e.ackReclaimed(ctx); err != nil { + return scenario{}, redisstream.Envelope{}, "", err + } + after, err := e.objectCount(ctx, attachmentID) + if err != nil { + return scenario{}, redisstream.Envelope{}, "", err + } + return scenario{ + Name: string(worker.FaultAfterCommit), AttemptAfterRecovery: delivery.Envelope.Attempt, + ObjectsBeforeRecovery: before, ObjectsAfterRecovery: after, FinalStatus: metadata.Status, + }, delivery.Envelope, attachmentID, nil +} + +func (e *experiment) queueAttachment(ctx context.Context) (string, model.AttachmentJob, error) { + attachmentID, _, job, err := e.queueAttachmentData(ctx, e.fixture, "image/png") + return attachmentID, job, err +} + +func (e *experiment) queueAttachmentData( + ctx context.Context, + data []byte, + contentType string, +) (string, string, model.AttachmentJob, error) { + attachmentID := uuid.NewString() + stagingKey := "staging/" + attachmentID + "/experiment" + digest := sha256.Sum256(data) + _, err := e.repository.CreatePendingAttachment(ctx, model.Attachment{ + ID: attachmentID, ChatID: e.chatID, OwnerID: e.ownerID, Filename: "experiment.png", + ContentType: contentType, SizeBytes: int64(len(data)), SHA256: hex.EncodeToString(digest[:]), + ObjectKey: stagingKey, StagingKey: stagingKey, Status: model.AttachmentPending, PipelineVersion: 1, + }) + if err != nil { + return "", "", model.AttachmentJob{}, err + } + if err := e.objects.Put(ctx, stagingKey, contentType, data); err != nil { + return "", "", model.AttachmentJob{}, err + } + job := model.AttachmentJob{ID: uuid.NewString(), PipelineVersion: 1, Attempt: 1} + if _, err := e.repository.QueueAttachmentWithJobAndOutbox(ctx, attachmentID, stagingKey, job, "", ""); err != nil { + return "", "", model.AttachmentJob{}, err + } + if _, err := e.dispatcher.DispatchOnce(ctx); err != nil { + return "", "", model.AttachmentJob{}, err + } + return attachmentID, stagingKey, job, nil +} + +func (e *experiment) readDelivery(ctx context.Context) (redisstream.Delivery, error) { + deliveries, err := e.readDeliveries(ctx, 1) + if err != nil { + return redisstream.Delivery{}, err + } + return deliveries[0], nil +} + +func (e *experiment) readDeliveries(ctx context.Context, expected int) ([]redisstream.Delivery, error) { + var result []redisstream.Delivery + for len(result) < expected { + deliveries, err := e.broker.Read(ctx) + if err != nil { + return nil, err + } + result = append(result, deliveries...) + } + if len(result) != expected { + return nil, fmt.Errorf("deliveries = %d, expected %d", len(result), expected) + } + return result, nil +} + +func (e *experiment) ackReclaimed(ctx context.Context) error { + deliveries, _, err := e.broker.Reclaim(ctx, "0-0") + if err != nil { + return err + } + if len(deliveries) != 1 { + return fmt.Errorf("reclaimed deliveries = %d, expected 1", len(deliveries)) + } + return e.worker.ProcessDelivery(ctx, deliveries[0], worker.FaultNone) +} + +func (e *experiment) objectCount(ctx context.Context, attachmentID string) (int, error) { + objects, next, err := e.objects.List(ctx, "attachments/"+attachmentID+"/", "", 100) + if err != nil { + return 0, err + } + if next != "" { + return 0, errors.New("unexpected object pagination") + } + return len(objects), nil +} + +func pngFixture(width, height int) ([]byte, error) { + value := image.NewNRGBA(image.Rect(0, 0, width, height)) + for y := range height { + for x := range width { + value.SetNRGBA(x, y, color.NRGBA{R: uint8(x), G: uint8(y), B: uint8(x + y), A: 255}) + } + } + var output bytes.Buffer + if err := png.Encode(&output, value); err != nil { + return nil, err + } + return output.Bytes(), nil +} + +func wait(ctx context.Context, duration time.Duration) error { + timer := time.NewTimer(duration) + defer timer.Stop() + select { + case <-ctx.Done(): + return ctx.Err() + case <-timer.C: + return nil + } +} diff --git a/cmd/messenger/main.go b/cmd/messenger/main.go new file mode 100644 index 0000000..c2c6c96 --- /dev/null +++ b/cmd/messenger/main.go @@ -0,0 +1,40 @@ +package main + +import ( + "errors" + "log/slog" + "os" + + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/bootstrap" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/logger" +) + +func main() { + appLogger := logger.New() + if err := run(appLogger, os.Args[1:]); err != nil { + appLogger.Error("application stopped", "error", err) + os.Exit(1) + } +} + +func run(logger *slog.Logger, arguments []string) error { + if len(arguments) == 0 { + return bootstrap.RunServer(logger) + } + if len(arguments) == 1 && arguments[0] == "migrate" { + return bootstrap.RunMigrations() + } + if len(arguments) == 1 && arguments[0] == "dispatch" { + return bootstrap.RunDispatcher(logger) + } + if len(arguments) == 1 && arguments[0] == "worker" { + return bootstrap.RunWorker(logger) + } + if arguments[0] == "reconcile-attachments" { + return bootstrap.RunReconciliation(arguments[1:], os.Stdout) + } + if arguments[0] == "replay-attachment" { + return bootstrap.RunAttachmentReplay(arguments[1:], os.Stdout) + } + return errors.New("usage: messenger [migrate|dispatch|worker|reconcile-attachments|replay-attachment]") +} diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 0000000..d21c2eb --- /dev/null +++ b/compose.yaml @@ -0,0 +1,526 @@ +x-messenger-config-volume: &messenger-config-volume + type: bind + source: ./config/messenger.yaml + target: /etc/messenger/messenger.yaml + read_only: true + +x-telemetry-environment: &telemetry-environment + OTEL_EXPORTER_OTLP_ENDPOINT: ${OTEL_EXPORTER_OTLP_ENDPOINT} + OTEL_EXPORTER_OTLP_PROTOCOL: ${OTEL_EXPORTER_OTLP_PROTOCOL} + OTEL_RESOURCE_ATTRIBUTES: deployment.environment.name=${DEPLOYMENT_ENVIRONMENT},service.version=${SERVICE_VERSION} + +services: + postgres: + image: postgres:17.6-alpine3.22 + environment: + POSTGRES_USER: ${POSTGRES_USER} + POSTGRES_DB: ${POSTGRES_DB} + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} + volumes: + - postgres_data:/var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER} -d ${POSTGRES_DB}"] + interval: 5s + timeout: 3s + retries: 20 + cpus: ${POSTGRES_CPUS} + mem_limit: ${POSTGRES_MEMORY} + restart: unless-stopped + networks: [data] + + valkey: + image: valkey/valkey:8.1.4-alpine + command: ["valkey-server", "--appendonly", "yes", "--appendfsync", "everysec", "--requirepass", "${VALKEY_PASSWORD}"] + environment: + VALKEY_PASSWORD: ${VALKEY_PASSWORD} + volumes: + - valkey_data:/data + healthcheck: + test: ["CMD-SHELL", "valkey-cli --no-auth-warning -a \"$$VALKEY_PASSWORD\" ping | grep -q PONG"] + interval: 5s + timeout: 3s + retries: 20 + cpus: ${VALKEY_CPUS} + mem_limit: ${VALKEY_MEMORY} + restart: unless-stopped + networks: [data] + + redis-broker: + image: redis:8.2.2-alpine + command: + - redis-server + - --appendonly + - "yes" + - --appendfsync + - always + - --maxmemory-policy + - noeviction + - --requirepass + - ${REDIS_BROKER_PASSWORD} + environment: + REDIS_BROKER_PASSWORD: ${REDIS_BROKER_PASSWORD} + volumes: + - redis_broker_data:/data + healthcheck: + test: ["CMD-SHELL", "redis-cli --no-auth-warning -a \"$$REDIS_BROKER_PASSWORD\" ping | grep -q PONG"] + interval: 5s + timeout: 3s + retries: 20 + cpus: ${REDIS_BROKER_CPUS} + mem_limit: ${REDIS_BROKER_MEMORY} + restart: unless-stopped + networks: [data] + + s3: + build: + context: . + dockerfile: infra/minio/server.Dockerfile + args: + MINIO_VERSION: RELEASE.2025-10-15T17-29-55Z + image: mtusi-minio:RELEASE.2025-10-15T17-29-55Z + command: ["server", "/data", "--address", ":9000"] + environment: + MINIO_ROOT_USER: ${MINIO_ROOT_USER} + MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD} + MINIO_BROWSER: "off" + MINIO_PROMETHEUS_AUTH_TYPE: public + volumes: + - s3_data:/data + expose: + - "9000" + healthcheck: + test: ["CMD", "wget", "-q", "-O", "/dev/null", "http://127.0.0.1:9000/minio/health/ready"] + interval: 5s + timeout: 3s + retries: 20 + cpus: ${S3_CPUS} + mem_limit: ${S3_MEMORY} + restart: unless-stopped + networks: [data] + + s3-bootstrap: + build: + context: . + dockerfile: infra/minio/client.Dockerfile + args: + MC_VERSION: RELEASE.2025-08-13T08-35-41Z + image: mtusi-mc:RELEASE.2025-08-13T08-35-41Z + entrypoint: ["/bin/sh", "/usr/local/bin/bootstrap.sh"] + environment: + MINIO_ROOT_USER: ${MINIO_ROOT_USER} + MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD} + S3_BUCKET: ${S3_BUCKET} + S3_ACCESS_KEY: ${S3_ACCESS_KEY} + S3_SECRET_KEY: ${S3_SECRET_KEY} + depends_on: + s3: + condition: service_healthy + healthcheck: + disable: true + cpus: ${S3_BOOTSTRAP_CPUS} + mem_limit: ${S3_BOOTSTRAP_MEMORY} + restart: "no" + networks: [data] + + migrate: + image: ${API_IMAGE:-mtusi-messenger:lab6} + command: ["/app/messenger", "migrate"] + environment: + MESSENGER_CONFIG_PATH: /etc/messenger/messenger.yaml + DATABASE_URL: ${DATABASE_URL} + volumes: + - *messenger-config-volume + depends_on: + postgres: + condition: service_healthy + healthcheck: + disable: true + cpus: ${MIGRATE_CPUS} + mem_limit: ${MIGRATE_MEMORY} + restart: "no" + networks: [data] + + api: + build: . + image: ${API_IMAGE:-mtusi-messenger:lab6} + environment: + <<: *telemetry-environment + OTEL_SERVICE_NAME: messenger-api + MESSENGER_CONFIG_PATH: /etc/messenger/messenger.yaml + APP_HOST: ${APP_HOST} + APP_PORT: ${APP_PORT} + DATABASE_URL: ${DATABASE_URL} + CACHE_URL: ${CACHE_URL} + SWAGGER_PORT: ${SWAGGER_PORT} + S3_ENDPOINT: ${S3_ENDPOINT} + S3_REGION: ${S3_REGION} + S3_BUCKET: ${S3_BUCKET} + S3_ACCESS_KEY: ${S3_ACCESS_KEY} + S3_SECRET_KEY: ${S3_SECRET_KEY} + volumes: + - *messenger-config-volume + expose: + - "8080" + depends_on: + migrate: + condition: service_completed_successfully + valkey: + condition: service_healthy + s3-bootstrap: + condition: service_completed_successfully + healthcheck: + test: ["CMD-SHELL", "wget -q -O /dev/null http://127.0.0.1:${APP_PORT}/health/ready || exit 1"] + interval: 2s + timeout: 1s + start_period: 5s + retries: 2 + deploy: + replicas: ${API_REPLICAS:-2} + cpus: ${API_CPUS} + mem_limit: ${API_MEMORY} + stop_grace_period: 8s + restart: unless-stopped + networks: [application, data] + + dispatcher: + image: ${API_IMAGE:-mtusi-messenger:lab6} + command: ["/app/messenger", "dispatch"] + environment: + <<: *telemetry-environment + OTEL_SERVICE_NAME: messenger-dispatcher + MESSENGER_CONFIG_PATH: /etc/messenger/messenger.yaml + DATABASE_URL: ${DATABASE_URL} + BROKER_URL: ${BROKER_URL} + volumes: + - *messenger-config-volume + expose: + - "9091" + depends_on: + migrate: + condition: service_completed_successfully + redis-broker: + condition: service_healthy + healthcheck: + test: ["CMD-SHELL", "wget -q -O /dev/null http://127.0.0.1:9091/health/live || exit 1"] + interval: 5s + timeout: 2s + retries: 20 + cpus: ${DISPATCHER_CPUS} + mem_limit: ${DISPATCHER_MEMORY} + restart: unless-stopped + networks: [data, monitoring] + + worker: + image: ${API_IMAGE:-mtusi-messenger:lab6} + command: ["/app/messenger", "worker"] + environment: + <<: *telemetry-environment + OTEL_SERVICE_NAME: messenger-worker + MESSENGER_CONFIG_PATH: /etc/messenger/messenger.yaml + DATABASE_URL: ${DATABASE_URL} + S3_ENDPOINT: ${S3_ENDPOINT} + S3_REGION: ${S3_REGION} + S3_BUCKET: ${S3_BUCKET} + S3_ACCESS_KEY: ${S3_ACCESS_KEY} + S3_SECRET_KEY: ${S3_SECRET_KEY} + BROKER_URL: ${BROKER_URL} + volumes: + - *messenger-config-volume + expose: + - "9091" + depends_on: + migrate: + condition: service_completed_successfully + redis-broker: + condition: service_healthy + s3-bootstrap: + condition: service_completed_successfully + healthcheck: + test: ["CMD-SHELL", "wget -q -O /dev/null http://127.0.0.1:9091/health/live || exit 1"] + interval: 5s + timeout: 2s + retries: 20 + cpus: ${WORKER_CPUS} + mem_limit: ${WORKER_MEMORY} + stop_grace_period: 8s + restart: unless-stopped + networks: [data, monitoring] + + nginx: + build: + context: . + target: loadbalancer + image: ${NGINX_IMAGE:-mtusi-nginx:lab6} + ports: + - "${API_BIND_ADDRESS}:8080" + depends_on: + api: + condition: service_healthy + healthcheck: + test: ["CMD-SHELL", "wget -q -O /dev/null http://127.0.0.1:8081/stub_status || exit 1"] + interval: 2s + timeout: 1s + retries: 10 + cpus: ${NGINX_CPUS} + mem_limit: ${NGINX_MEMORY} + restart: unless-stopped + networks: [edge, application, monitoring] + + swagger: + build: + context: . + target: swagger + image: ${SWAGGER_IMAGE:-mtusi-swagger:lab6} + ports: + - "${SWAGGER_BIND_ADDRESS}:8080" + depends_on: + nginx: + condition: service_healthy + cpus: ${SWAGGER_CPUS} + mem_limit: ${SWAGGER_MEMORY} + restart: unless-stopped + networks: [edge] + + tests: + profiles: ["tests"] + build: + context: . + target: test + environment: + TEST_DATABASE_URL: ${DATABASE_URL} + TEST_CACHE_URL: ${CACHE_URL} + depends_on: + migrate: + condition: service_completed_successfully + valkey: + condition: service_healthy + cpus: ${TEST_CPUS} + mem_limit: ${TEST_MEMORY} + networks: [data] + + lab5-experiment: + profiles: ["experiments"] + image: ${API_IMAGE:-mtusi-messenger:lab6} + command: ["/app/lab5experiment"] + environment: + MESSENGER_CONFIG_PATH: /etc/messenger/messenger.yaml + DATABASE_URL: ${DATABASE_URL} + S3_ENDPOINT: ${S3_ENDPOINT} + S3_REGION: ${S3_REGION} + S3_BUCKET: ${S3_BUCKET} + S3_ACCESS_KEY: ${S3_ACCESS_KEY} + S3_SECRET_KEY: ${S3_SECRET_KEY} + BROKER_URL: ${BROKER_URL} + volumes: + - *messenger-config-volume + depends_on: + migrate: + condition: service_completed_successfully + redis-broker: + condition: service_healthy + s3-bootstrap: + condition: service_completed_successfully + cpus: ${TEST_CPUS} + mem_limit: ${TEST_MEMORY} + networks: [application, data] + + postgres-exporter: + image: quay.io/prometheuscommunity/postgres-exporter:v0.20.1 + environment: + DATA_SOURCE_URI: postgres:5432/${POSTGRES_DB}?sslmode=disable + DATA_SOURCE_USER: ${POSTGRES_USER} + DATA_SOURCE_PASS: ${POSTGRES_PASSWORD} + PG_EXPORTER_COLLECTION_TIMEOUT: 5s + PG_EXPORTER_EXTEND_QUERY_PATH: /etc/postgres_exporter/queries.yml + volumes: + - ./monitoring/postgres-queries.yml:/etc/postgres_exporter/queries.yml:ro + depends_on: + postgres: + condition: service_healthy + cpus: ${EXPORTER_CPUS} + mem_limit: ${EXPORTER_MEMORY} + restart: unless-stopped + networks: [data, monitoring] + + valkey-exporter: + image: oliver006/redis_exporter:v1.89.0-alpine + environment: + REDIS_ADDR: redis://valkey:6379 + REDIS_USER: default + REDIS_PASSWORD: ${VALKEY_PASSWORD} + depends_on: + valkey: + condition: service_healthy + cpus: ${EXPORTER_CPUS} + mem_limit: ${EXPORTER_MEMORY} + restart: unless-stopped + networks: [data, monitoring] + + redis-broker-exporter: + image: oliver006/redis_exporter:v1.89.0-alpine + environment: + REDIS_ADDR: redis://redis-broker:6379 + REDIS_USER: default + REDIS_PASSWORD: ${REDIS_BROKER_PASSWORD} + REDIS_EXPORTER_CHECK_SINGLE_STREAMS: attachment-jobs,attachment-jobs-failed + depends_on: + redis-broker: + condition: service_healthy + cpus: ${EXPORTER_CPUS} + mem_limit: ${EXPORTER_MEMORY} + restart: unless-stopped + networks: [data, monitoring] + + nginx-exporter: + image: nginx/nginx-prometheus-exporter:1.5.1 + command: ["--nginx.scrape-uri=http://nginx:8081/stub_status"] + depends_on: + nginx: + condition: service_healthy + cpus: ${EXPORTER_CPUS} + mem_limit: ${EXPORTER_MEMORY} + restart: unless-stopped + networks: [monitoring] + + prometheus: + build: + context: . + target: prometheus + image: ${PROMETHEUS_IMAGE:-mtusi-prometheus:lab6} + command: + - --config.file=/etc/prometheus/prometheus.yml + - --storage.tsdb.path=/prometheus + - --storage.tsdb.retention.time=7d + - --web.enable-remote-write-receiver + volumes: + - prometheus_data:/prometheus + ports: + - "${PROMETHEUS_BIND_ADDRESS}:9090" + healthcheck: + test: ["CMD", "promtool", "query", "instant", "http://127.0.0.1:9090", "up"] + interval: 5s + timeout: 3s + retries: 20 + depends_on: + - api + - nginx-exporter + - postgres-exporter + - valkey-exporter + - redis-broker-exporter + - dispatcher + - worker + - s3 + - otel-collector + - loki + - tempo + cpus: ${PROMETHEUS_CPUS} + mem_limit: ${PROMETHEUS_MEMORY} + restart: unless-stopped + networks: [edge, application, data, monitoring] + + grafana: + build: + context: . + target: grafana + image: ${GRAFANA_IMAGE:-mtusi-grafana:lab6} + environment: + GF_SECURITY_ADMIN_USER: admin + GF_SECURITY_ADMIN_PASSWORD: ${GRAFANA_ADMIN_PASSWORD} + GF_USERS_ALLOW_SIGN_UP: "false" + GF_EXPLORE_ENABLED: "true" + volumes: + - grafana_data:/var/lib/grafana + ports: + - "${GRAFANA_BIND_ADDRESS}:3000" + depends_on: + - prometheus + - loki + - tempo + healthcheck: + test: ["CMD-SHELL", "wget -q -O /dev/null http://127.0.0.1:3000/api/health || exit 1"] + interval: 5s + timeout: 3s + retries: 20 + cpus: ${GRAFANA_CPUS} + mem_limit: ${GRAFANA_MEMORY} + restart: unless-stopped + networks: [edge, monitoring] + + loki: + image: grafana/loki:3.7.9 + command: ["-config.file=/etc/loki/loki.yaml"] + volumes: + - ./infra/loki/loki.yaml:/etc/loki/loki.yaml:ro + - loki_data:/loki + ports: + - "${LOKI_BIND_ADDRESS}:3100" + healthcheck: + test: ["CMD", "/usr/bin/loki", "-version"] + interval: 10s + timeout: 3s + retries: 10 + cpus: ${LOKI_CPUS} + mem_limit: ${LOKI_MEMORY} + restart: unless-stopped + networks: [edge, monitoring] + + tempo: + image: grafana/tempo:2.10.10 + command: ["-config.file=/etc/tempo/tempo.yaml"] + volumes: + - ./infra/tempo/tempo.yaml:/etc/tempo/tempo.yaml:ro + - tempo_data:/var/tempo + ports: + - "${TEMPO_BIND_ADDRESS}:3200" + healthcheck: + test: ["CMD", "/tempo", "-version"] + interval: 10s + timeout: 3s + retries: 10 + cpus: ${TEMPO_CPUS} + mem_limit: ${TEMPO_MEMORY} + restart: unless-stopped + networks: [edge, monitoring] + + otel-collector: + image: otel/opentelemetry-collector-contrib:0.160.0 + command: ["--config=/etc/otelcol-contrib/collector.yaml"] + volumes: + - ./infra/otel/collector.yaml:/etc/otelcol-contrib/collector.yaml:ro + expose: + - "4317" + - "4318" + - "8888" + - "8889" + - "13133" + depends_on: + - loki + - tempo + healthcheck: + test: ["CMD", "/otelcol-contrib", "components"] + interval: 10s + timeout: 3s + retries: 10 + cpus: ${OTEL_COLLECTOR_CPUS} + mem_limit: ${OTEL_COLLECTOR_MEMORY} + restart: unless-stopped + networks: [application, data, monitoring] + +volumes: + postgres_data: + valkey_data: + redis_broker_data: + prometheus_data: + grafana_data: + loki_data: + tempo_data: + s3_data: + +networks: + edge: + application: + internal: true + data: + internal: true + monitoring: + internal: true diff --git a/config/messenger.yaml b/config/messenger.yaml new file mode 100644 index 0000000..031e152 --- /dev/null +++ b/config/messenger.yaml @@ -0,0 +1,76 @@ +version: 1 + +server: + session_store: valkey + session_ttl: 1h + idempotency_ttl: 24h + request_timeout: 5s + http: + read_header_timeout: 5s + read_timeout: 10s + write_timeout: 10s + idle_timeout: 1m + shutdown_timeout: 5s + password: + memory_kib: 65536 + iterations: 3 + parallelism: 2 + salt_length: 16 + key_length: 32 + +database: + min_connections: 1 + max_connections: 10 + max_connection_age: 30m + max_connection_idle: 5m + health_check_period: 30s + dependency_timeout: 5s + +migration: + timeout: 30s + +object_storage: + use_tls: false + timeout: 5s + +attachments: + max_bytes: 10485760 + max_pixels: 40000000 + body_memory_bytes: 67108864 + decode_memory_bytes: 402653184 + +pipeline: + version: 1 + +broker: + stream: attachment-jobs + consumer_group: attachment-workers + failed_stream: attachment-jobs-failed + failed_stream_max_length: 10000 + +outbox: + batch_size: 32 + lease: 15s + poll_interval: 1s + recovery_interval: 10s + retry_delay: 2s + retention: 168h + cleanup_interval: 1m + cleanup_batch_size: 1000 + +worker: + concurrency: 2 + prefetch: 4 + lease: 30s + heartbeat_interval: 5s + processing_timeout: 20s + read_block: 2s + reclaim_interval: 5s + max_attempts: 5 + retry: + base: 1s + max: 30s + memory_bytes: 738197504 + +monitoring: + metrics_address: 0.0.0.0:9091 diff --git a/deploy/compose.images.yaml b/deploy/compose.images.yaml new file mode 100644 index 0000000..7d98319 --- /dev/null +++ b/deploy/compose.images.yaml @@ -0,0 +1,30 @@ +services: + api: + image: ${API_IMAGE:?API_IMAGE is required} + + migrate: + image: ${API_IMAGE:?API_IMAGE is required} + + dispatcher: + image: ${API_IMAGE:?API_IMAGE is required} + + worker: + image: ${API_IMAGE:?API_IMAGE is required} + + nginx: + image: ${NGINX_IMAGE:?NGINX_IMAGE is required} + + swagger: + image: ${SWAGGER_IMAGE:?SWAGGER_IMAGE is required} + + prometheus: + image: ${PROMETHEUS_IMAGE:?PROMETHEUS_IMAGE is required} + + grafana: + image: ${GRAFANA_IMAGE:?GRAFANA_IMAGE is required} + + s3: + image: ${S3_IMAGE:?S3_IMAGE is required} + + s3-bootstrap: + image: ${S3_BOOTSTRAP_IMAGE:?S3_BOOTSTRAP_IMAGE is required} diff --git a/deploy/compose.runtime.yaml b/deploy/compose.runtime.yaml new file mode 100644 index 0000000..b9dcd47 --- /dev/null +++ b/deploy/compose.runtime.yaml @@ -0,0 +1,4 @@ +services: + nginx: + volumes: + - ${NGINX_UPSTREAM_FILE:?NGINX_UPSTREAM_FILE is required}:/etc/nginx/conf.d/upstream.conf:ro diff --git a/evidence/lab6-results.json b/evidence/lab6-results.json new file mode 100644 index 0000000..c09c6b9 --- /dev/null +++ b/evidence/lab6-results.json @@ -0,0 +1,80 @@ +{ + "schema_version": 1, + "captured_at": "2026-11-04T17:53:52Z", + "baseline": { + "api_request_rate_per_second": 1.014325, + "firing_alerts": 0, + "traces_by_service": { + "messenger-api": 100, + "messenger-worker": 72, + "messenger-dispatcher": 64 + }, + "logs_by_service": { + "messenger-api": 207, + "messenger-worker": 41, + "messenger-dispatcher": 1 + }, + "api_worker_shared_traces": 22, + "traces_with_correlated_logs": 101 + }, + "incidents": [ + { + "name": "worker_unavailable_short", + "injected_duration_seconds": 76, + "alert_detection_upper_bound_seconds": 76, + "alert_resolution_seconds": 13, + "recovery_observation_seconds": 33 + }, + { + "name": "postgres_unavailable", + "injected_duration_seconds": 78, + "alert_detection_upper_bound_seconds": 78, + "operational_alert_resolution_seconds": 134, + "recovery_observation_seconds": 154, + "fast_burn_alert_observed": true, + "database_error_traces": 3, + "database_error_traces_with_logs": 1 + }, + { + "name": "worker_unavailable_long", + "injected_duration_seconds": 181, + "alert_detection_upper_bound_seconds": 181, + "alert_resolution_seconds": 20, + "recovery_observation_seconds": 91, + "fast_burn_alert_observed": true, + "slow_burn_alert_observed": true + }, + { + "name": "telemetry_backends_unavailable", + "injected_duration_seconds": 30, + "api_ready_http_status": 200, + "recovery_observation_seconds": 31, + "exporters": { + "otlp_grpc/tempo": { + "capacity": 512, + "degraded_queue_size": 15, + "recovered_queue_size": 0, + "export_failures": 16, + "dropped_records": 0 + }, + "otlp_http/loki": { + "capacity": 512, + "degraded_queue_size": 16, + "recovered_queue_size": 0, + "export_failures": 8, + "dropped_records": 0 + } + }, + "collector_peak_rss_bytes": { + "baseline": 225923072, + "degraded": 226549760, + "recovered": 226746368 + } + } + ], + "validation": { + "valid": true, + "errors": [], + "incident_summaries": 4 + } +} diff --git a/evidence/lab6-schema.json b/evidence/lab6-schema.json new file mode 100644 index 0000000..c119c53 --- /dev/null +++ b/evidence/lab6-schema.json @@ -0,0 +1,37 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://gitea.privcloud.tech/mtusi2026/mtusi2026lab6/evidence/lab6-schema.json", + "title": "Lab 6 incident summary", + "type": "object", + "required": [ + "schema_version", + "incident", + "started_at", + "recovery_started_at", + "recovered_at", + "injected_duration_seconds", + "recovery_observation_seconds", + "artifacts" + ], + "properties": { + "schema_version": {"const": 1}, + "incident": { + "enum": ["worker_unavailable", "postgres_unavailable", "telemetry_backends_unavailable"] + }, + "profile": {"enum": ["short", "long"]}, + "started_at": {"type": "string", "format": "date-time"}, + "recovery_started_at": {"type": "string", "format": "date-time"}, + "recovered_at": {"type": "string", "format": "date-time"}, + "planned_injection_seconds": {"type": "integer", "minimum": 1}, + "injected_duration_seconds": {"type": "integer", "minimum": 1}, + "recovery_observation_seconds": {"type": "integer", "minimum": 0}, + "api_ready_http_status_during_outage": {"type": "integer", "minimum": 100, "maximum": 599}, + "artifacts": { + "type": "array", + "items": {"type": "string", "pattern": "^[A-Za-z0-9._-]+$"}, + "minItems": 4, + "uniqueItems": true + } + }, + "additionalProperties": false +} diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..37c0a6e --- /dev/null +++ b/go.mod @@ -0,0 +1,83 @@ +module gitea.privcloud.tech/dizi/mtusi2026lab6 + +go 1.26.0 + +require ( + github.com/deepteams/webp v1.2.8 + github.com/google/uuid v1.6.0 + github.com/jackc/pgx/v5 v5.11.0 + github.com/minio/minio-go/v7 v7.0.99 + github.com/prometheus/client_golang v1.23.2 + github.com/redis/go-redis/v9 v9.17.2 + github.com/spf13/viper v1.21.0 + github.com/stretchr/testify v1.11.1 + github.com/valkey-io/valkey-go v1.0.78 + go.opentelemetry.io/contrib/bridges/otelslog v0.13.0 + go.opentelemetry.io/contrib/instrumentation/runtime v0.60.0 + go.opentelemetry.io/otel v1.38.0 + go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc v0.14.0 + go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.38.0 + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.38.0 + go.opentelemetry.io/otel/exporters/prometheus v0.60.0 + go.opentelemetry.io/otel/metric v1.38.0 + go.opentelemetry.io/otel/sdk v1.38.0 + go.opentelemetry.io/otel/sdk/log v0.14.0 + go.opentelemetry.io/otel/sdk/metric v1.38.0 + go.opentelemetry.io/otel/trace v1.38.0 + golang.org/x/crypto v0.57.0 + golang.org/x/sync v0.23.0 +) + +require ( + github.com/beorn7/perks v1.0.1 // indirect + github.com/cenkalti/backoff/v5 v5.0.3 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect + github.com/dustin/go-humanize v1.0.1 // indirect + github.com/fsnotify/fsnotify v1.9.0 // indirect + github.com/go-ini/ini v1.67.0 // indirect + github.com/go-logr/logr v1.4.3 // indirect + github.com/go-logr/stdr v1.2.2 // indirect + github.com/go-viper/mapstructure/v2 v2.4.0 // indirect + github.com/grafana/regexp v0.0.0-20240518133315-a468a5bfb3bc // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.2 // indirect + github.com/jackc/pgpassfile v1.0.0 // indirect + github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect + github.com/jackc/puddle/v2 v2.2.2 // indirect + github.com/klauspost/compress v1.18.2 // indirect + github.com/klauspost/cpuid/v2 v2.2.11 // indirect + github.com/klauspost/crc32 v1.3.0 // indirect + github.com/minio/crc64nvme v1.1.1 // indirect + github.com/minio/md5-simd v1.1.2 // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/pelletier/go-toml/v2 v2.2.4 // indirect + github.com/philhofer/fwd v1.2.0 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/prometheus/client_model v0.6.2 // indirect + github.com/prometheus/common v0.66.1 // indirect + github.com/prometheus/otlptranslator v0.0.2 // indirect + github.com/prometheus/procfs v0.17.0 // indirect + github.com/rs/xid v1.6.0 // indirect + github.com/sagikazarmark/locafero v0.11.0 // indirect + github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8 // indirect + github.com/spf13/afero v1.15.0 // indirect + github.com/spf13/cast v1.10.0 // indirect + github.com/spf13/pflag v1.0.10 // indirect + github.com/subosito/gotenv v1.6.0 // indirect + github.com/tinylib/msgp v1.6.1 // indirect + go.opentelemetry.io/auto/sdk v1.1.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.38.0 // indirect + go.opentelemetry.io/otel/log v0.14.0 // indirect + go.opentelemetry.io/proto/otlp v1.7.1 // indirect + go.yaml.in/yaml/v2 v2.4.2 // indirect + go.yaml.in/yaml/v3 v3.0.5 // indirect + golang.org/x/net v0.58.0 // indirect + golang.org/x/sys v0.48.0 // indirect + golang.org/x/text v0.42.0 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20250825161204-c5933d9347a5 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20250825161204-c5933d9347a5 // indirect + google.golang.org/grpc v1.75.0 // indirect + google.golang.org/protobuf v1.36.8 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..63eb5b2 --- /dev/null +++ b/go.sum @@ -0,0 +1,184 @@ +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs= +github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c= +github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA= +github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0= +github.com/cenkalti/backoff/v5 v5.0.3 h1:ZN+IMa753KfX5hd8vVaMixjnqRZ3y8CuJKRKj1xcsSM= +github.com/cenkalti/backoff/v5 v5.0.3/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/deepteams/webp v1.2.8 h1:iFLBxo4V34ixqay6+MTB18zujCM8VSe1ejsYh+ue1nA= +github.com/deepteams/webp v1.2.8/go.mod h1:J8Ap+HAixxpKKRN9IpEeSKlfvhsef1v43jKTO7m3f4c= +github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= +github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= +github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= +github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k= +github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= +github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A= +github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= +github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/go-viper/mapstructure/v2 v2.4.0 h1:EBsztssimR/CONLSZZ04E8qAkxNYq4Qp9LvH92wZUgs= +github.com/go-viper/mapstructure/v2 v2.4.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/grafana/regexp v0.0.0-20240518133315-a468a5bfb3bc h1:GN2Lv3MGO7AS6PrRoT6yV5+wkrOpcszoIsO4+4ds248= +github.com/grafana/regexp v0.0.0-20240518133315-a468a5bfb3bc/go.mod h1:+JKpmjMGhpgPL+rXZ5nsZieVzvarn86asRlBg4uNGnk= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.2 h1:8Tjv8EJ+pM1xP8mK6egEbD1OgnVTyacbefKhmbLhIhU= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.2/go.mod h1:pkJQ2tZHJ0aFOVEEot6oZmaVEZcRme73eIFmhiVuRWs= +github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= +github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= +github.com/jackc/pgx/v5 v5.11.0 h1:IzBBtyK9AHqf98cctWFifYSci2hgQR/cd56wB4p+ogg= +github.com/jackc/pgx/v5 v5.11.0/go.mod h1:mal1tBGAFfLHvZzaYh77YS/eC6IX9OWbRV1QIIM0Jn4= +github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo= +github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= +github.com/klauspost/compress v1.18.2 h1:iiPHWW0YrcFgpBYhsA6D1+fqHssJscY/Tm/y2Uqnapk= +github.com/klauspost/compress v1.18.2/go.mod h1:R0h/fSBs8DE4ENlcrlib3PsXS61voFxhIs2DeRhCvJ4= +github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/klauspost/cpuid/v2 v2.2.11 h1:0OwqZRYI2rFrjS4kvkDnqJkKHdHaRnCm68/DY4OxRzU= +github.com/klauspost/cpuid/v2 v2.2.11/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= +github.com/klauspost/crc32 v1.3.0 h1:sSmTt3gUt81RP655XGZPElI0PelVTZ6YwCRnPSupoFM= +github.com/klauspost/crc32 v1.3.0/go.mod h1:D7kQaZhnkX/Y0tstFGf8VUzv2UofNGqCjnC3zdHB0Hw= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= +github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= +github.com/minio/crc64nvme v1.1.1 h1:8dwx/Pz49suywbO+auHCBpCtlW1OfpcLN7wYgVR6wAI= +github.com/minio/crc64nvme v1.1.1/go.mod h1:eVfm2fAzLlxMdUGc0EEBGSMmPwmXD5XiNRpnu9J3bvg= +github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= +github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= +github.com/minio/minio-go/v7 v7.0.99 h1:2vH/byrwUkIpFQFOilvTfaUpvAX3fEFhEzO+DR3DlCE= +github.com/minio/minio-go/v7 v7.0.99/go.mod h1:EtGNKtlX20iL2yaYnxEigaIvj0G0GwSDnifnG8ClIdw= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/onsi/gomega v1.42.1 h1:iN1rCUX+44NZ1Dc97MPoeFYbFR0vh8zxoxMFwKdyZ6I= +github.com/onsi/gomega v1.42.1/go.mod h1:REff/hsDsodHoKlWsP2mAPhu1+5/6hVYNf9rIEBpeSg= +github.com/pelletier/go-toml/v2 v2.2.4 h1:mye9XuhQ6gvn5h28+VilKrrPoQVanw5PMw/TB0t5Ec4= +github.com/pelletier/go-toml/v2 v2.2.4/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY= +github.com/philhofer/fwd v1.2.0 h1:e6DnBTl7vGY+Gz322/ASL4Gyp1FspeMvx1RNDoToZuM= +github.com/philhofer/fwd v1.2.0/go.mod h1:RqIHx9QI14HlwKwm98g9Re5prTQ6LdeRQn+gXJFxsJM= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h0RJWRi/o0o= +github.com/prometheus/client_golang v1.23.2/go.mod h1:Tb1a6LWHB3/SPIzCoaDXI4I8UHKeFTEQ1YCr+0Gyqmg= +github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk= +github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE= +github.com/prometheus/common v0.66.1 h1:h5E0h5/Y8niHc5DlaLlWLArTQI7tMrsfQjHV+d9ZoGs= +github.com/prometheus/common v0.66.1/go.mod h1:gcaUsgf3KfRSwHY4dIMXLPV0K/Wg1oZ8+SbZk/HH/dA= +github.com/prometheus/otlptranslator v0.0.2 h1:+1CdeLVrRQ6Psmhnobldo0kTp96Rj80DRXRd5OSnMEQ= +github.com/prometheus/otlptranslator v0.0.2/go.mod h1:P8AwMgdD7XEr6QRUJ2QWLpiAZTgTE2UYgjlu3svompI= +github.com/prometheus/procfs v0.17.0 h1:FuLQ+05u4ZI+SS/w9+BWEM2TXiHKsUQ9TADiRH7DuK0= +github.com/prometheus/procfs v0.17.0/go.mod h1:oPQLaDAMRbA+u8H5Pbfq+dl3VDAvHxMUOVhe0wYB2zw= +github.com/redis/go-redis/v9 v9.17.2 h1:P2EGsA4qVIM3Pp+aPocCJ7DguDHhqrXNhVcEp4ViluI= +github.com/redis/go-redis/v9 v9.17.2/go.mod h1:u410H11HMLoB+TP67dz8rL9s6QW2j76l0//kSOd3370= +github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII= +github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o= +github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU= +github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0= +github.com/sagikazarmark/locafero v0.11.0 h1:1iurJgmM9G3PA/I+wWYIOw/5SyBtxapeHDcg+AAIFXc= +github.com/sagikazarmark/locafero v0.11.0/go.mod h1:nVIGvgyzw595SUSUE6tvCp3YYTeHs15MvlmU87WwIik= +github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8 h1:+jumHNA0Wrelhe64i8F6HNlS8pkoyMv5sreGx2Ry5Rw= +github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8/go.mod h1:3n1Cwaq1E1/1lhQhtRK2ts/ZwZEhjcQeJQ1RuC6Q/8U= +github.com/spf13/afero v1.15.0 h1:b/YBCLWAJdFWJTN9cLhiXXcD7mzKn9Dm86dNnfyQw1I= +github.com/spf13/afero v1.15.0/go.mod h1:NC2ByUVxtQs4b3sIUphxK0NioZnmxgyCrfzeuq8lxMg= +github.com/spf13/cast v1.10.0 h1:h2x0u2shc1QuLHfxi+cTJvs30+ZAHOGRic8uyGTDWxY= +github.com/spf13/cast v1.10.0/go.mod h1:jNfB8QC9IA6ZuY2ZjDp0KtFO2LZZlg4S/7bzP6qqeHo= +github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk= +github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/viper v1.21.0 h1:x5S+0EU27Lbphp4UKm1C+1oQO+rKx36vfCoaVebLFSU= +github.com/spf13/viper v1.21.0/go.mod h1:P0lhsswPGWD/1lZJ9ny3fYnVqxiegrlNrEmgLjbTCAY= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8= +github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= +github.com/tinylib/msgp v1.6.1 h1:ESRv8eL3u+DNHUoSAAQRE50Hm162zqAnBoGv9PzScPY= +github.com/tinylib/msgp v1.6.1/go.mod h1:RSp0LW9oSxFut3KzESt5Voq4GVWyS+PSulT77roAqEA= +github.com/valkey-io/valkey-go v1.0.78 h1:1Ce8hRaaNXpKfqmqZQRhnbVh1eW8MGR873pdXVvE4y8= +github.com/valkey-io/valkey-go v1.0.78/go.mod h1:gvC/r2m3eW4Hbj0YnjogTzNtFdPSM/D+NCqen+5OABM= +go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA= +go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A= +go.opentelemetry.io/contrib/bridges/otelslog v0.13.0 h1:bwnLpizECbPr1RrQ27waeY2SPIPeccCx/xLuoYADZ9s= +go.opentelemetry.io/contrib/bridges/otelslog v0.13.0/go.mod h1:3nWlOiiqA9UtUnrcNk82mYasNxD8ehOspL0gOfEo6Y4= +go.opentelemetry.io/contrib/instrumentation/runtime v0.60.0 h1:0NgN/3SYkqYJ9NBlDfl/2lzVlwos/YQLvi8sUrzJRBE= +go.opentelemetry.io/contrib/instrumentation/runtime v0.60.0/go.mod h1:oxpUfhTkhgQaYIjtBt3T3w135dLoxq//qo3WPlPIKkE= +go.opentelemetry.io/otel v1.38.0 h1:RkfdswUDRimDg0m2Az18RKOsnI8UDzppJAtj01/Ymk8= +go.opentelemetry.io/otel v1.38.0/go.mod h1:zcmtmQ1+YmQM9wrNsTGV/q/uyusom3P8RxwExxkZhjM= +go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc v0.14.0 h1:OMqPldHt79PqWKOMYIAQs3CxAi7RLgPxwfFSwr4ZxtM= +go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc v0.14.0/go.mod h1:1biG4qiqTxKiUCtoWDPpL3fB3KxVwCiGw81j3nKMuHE= +go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.38.0 h1:vl9obrcoWVKp/lwl8tRE33853I8Xru9HFbw/skNeLs8= +go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.38.0/go.mod h1:GAXRxmLJcVM3u22IjTg74zWBrRCKq8BnOqUVLodpcpw= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.38.0 h1:GqRJVj7UmLjCVyVJ3ZFLdPRmhDUp2zFmQe3RHIOsw24= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.38.0/go.mod h1:ri3aaHSmCTVYu2AWv44YMauwAQc0aqI9gHKIcSbI1pU= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.38.0 h1:lwI4Dc5leUqENgGuQImwLo4WnuXFPetmPpkLi2IrX54= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.38.0/go.mod h1:Kz/oCE7z5wuyhPxsXDuaPteSWqjSBD5YaSdbxZYGbGk= +go.opentelemetry.io/otel/exporters/prometheus v0.60.0 h1:cGtQxGvZbnrWdC2GyjZi0PDKVSLWP/Jocix3QWfXtbo= +go.opentelemetry.io/otel/exporters/prometheus v0.60.0/go.mod h1:hkd1EekxNo69PTV4OWFGZcKQiIqg0RfuWExcPKFvepk= +go.opentelemetry.io/otel/log v0.14.0 h1:2rzJ+pOAZ8qmZ3DDHg73NEKzSZkhkGIua9gXtxNGgrM= +go.opentelemetry.io/otel/log v0.14.0/go.mod h1:5jRG92fEAgx0SU/vFPxmJvhIuDU9E1SUnEQrMlJpOno= +go.opentelemetry.io/otel/metric v1.38.0 h1:Kl6lzIYGAh5M159u9NgiRkmoMKjvbsKtYRwgfrA6WpA= +go.opentelemetry.io/otel/metric v1.38.0/go.mod h1:kB5n/QoRM8YwmUahxvI3bO34eVtQf2i4utNVLr9gEmI= +go.opentelemetry.io/otel/sdk v1.38.0 h1:l48sr5YbNf2hpCUj/FoGhW9yDkl+Ma+LrVl8qaM5b+E= +go.opentelemetry.io/otel/sdk v1.38.0/go.mod h1:ghmNdGlVemJI3+ZB5iDEuk4bWA3GkTpW+DOoZMYBVVg= +go.opentelemetry.io/otel/sdk/log v0.14.0 h1:JU/U3O7N6fsAXj0+CXz21Czg532dW2V4gG1HE/e8Zrg= +go.opentelemetry.io/otel/sdk/log v0.14.0/go.mod h1:imQvII+0ZylXfKU7/wtOND8Hn4OpT3YUoIgqJVksUkM= +go.opentelemetry.io/otel/sdk/log/logtest v0.14.0 h1:Ijbtz+JKXl8T2MngiwqBlPaHqc4YCaP/i13Qrow6gAM= +go.opentelemetry.io/otel/sdk/log/logtest v0.14.0/go.mod h1:dCU8aEL6q+L9cYTqcVOk8rM9Tp8WdnHOPLiBgp0SGOA= +go.opentelemetry.io/otel/sdk/metric v1.38.0 h1:aSH66iL0aZqo//xXzQLYozmWrXxyFkBJ6qT5wthqPoM= +go.opentelemetry.io/otel/sdk/metric v1.38.0/go.mod h1:dg9PBnW9XdQ1Hd6ZnRz689CbtrUp0wMMs9iPcgT9EZA= +go.opentelemetry.io/otel/trace v1.38.0 h1:Fxk5bKrDZJUH+AMyyIXGcFAPah0oRcT+LuNtJrmcNLE= +go.opentelemetry.io/otel/trace v1.38.0/go.mod h1:j1P9ivuFsTceSWe1oY+EeW3sc+Pp42sO++GHkg4wwhs= +go.opentelemetry.io/proto/otlp v1.7.1 h1:gTOMpGDb0WTBOP8JaO72iL3auEZhVmAQg4ipjOVAtj4= +go.opentelemetry.io/proto/otlp v1.7.1/go.mod h1:b2rVh6rfI/s2pHWNlB7ILJcRALpcNDzKhACevjI+ZnE= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.yaml.in/yaml/v2 v2.4.2 h1:DzmwEr2rDGHl7lsFgAHxmNz/1NlQ7xLIrlN2h5d1eGI= +go.yaml.in/yaml/v2 v2.4.2/go.mod h1:081UH+NErpNdqlCXm3TtEran0rJZGxAYx9hb/ELlsPU= +go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw= +go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg= +golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M= +golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA= +golang.org/x/net v0.58.0 h1:ynWG7rqYi4ccpTEuPZ2QGWHktVEM9DMCj9yzDE0Q7To= +golang.org/x/net v0.58.0/go.mod h1:YwCddHnFlT7eLQqVprV19OnhLGtc5xOKgE0RyqgfWAU= +golang.org/x/sync v0.23.0 h1:KameEIfc1IkluZyXWLn39Wd4tURc6GbCiISGiZm2bQk= +golang.org/x/sync v0.23.0/go.mod h1:sUUOizhqBxiL6pEWpqNLUiaJn1ShEbZ6BBqskPbjZm0= +golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo= +golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= +golang.org/x/text v0.42.0 h1:JbOZXgfeCPU9gacVtYliJqOhD+zhrEqK4LfdpmlUZqI= +golang.org/x/text v0.42.0/go.mod h1:ojzP1Z+2QtioaF8DTtO8K5q7JWVVYwZKenzujK0Zd0E= +gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk= +gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E= +google.golang.org/genproto/googleapis/api v0.0.0-20250825161204-c5933d9347a5 h1:BIRfGDEjiHRrk0QKZe3Xv2ieMhtgRGeLcZQ0mIVn4EY= +google.golang.org/genproto/googleapis/api v0.0.0-20250825161204-c5933d9347a5/go.mod h1:j3QtIyytwqGr1JUDtYXwtMXWPKsEa5LtzIFN1Wn5WvE= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250825161204-c5933d9347a5 h1:eaY8u2EuxbRv7c3NiGK0/NedzVsCcV6hDuU5qPX5EGE= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250825161204-c5933d9347a5/go.mod h1:M4/wBTSeyLxupu3W3tJtOgB14jILAS/XWPSSa3TAlJc= +google.golang.org/grpc v1.75.0 h1:+TW+dqTd2Biwe6KKfhE5JpiYIBWq865PhKGSXiivqt4= +google.golang.org/grpc v1.75.0/go.mod h1:JtPAzKiq4v1xcAB2hydNlWI2RnF85XXcV0mhKXr2ecQ= +google.golang.org/protobuf v1.36.8 h1:xHScyCOEuuwZEc6UtSOvPbAT4zRh0xcNRYekJwfqyMc= +google.golang.org/protobuf v1.36.8/go.mod h1:fuxRtAxBytpl4zzqUh6/eyUujkJdNiuEkXntxiD/uRU= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/infra/loki/loki.yaml b/infra/loki/loki.yaml new file mode 100644 index 0000000..4c04d5d --- /dev/null +++ b/infra/loki/loki.yaml @@ -0,0 +1,48 @@ +auth_enabled: false + +server: + http_listen_address: 0.0.0.0 + http_listen_port: 3100 + grpc_listen_address: 0.0.0.0 + grpc_listen_port: 9096 + +common: + path_prefix: /loki + replication_factor: 1 + ring: + kvstore: + store: inmemory + storage: + filesystem: + chunks_directory: /loki/chunks + rules_directory: /loki/rules + +schema_config: + configs: + - from: 2024-01-01 + store: tsdb + object_store: filesystem + schema: v13 + index: + prefix: index_ + period: 24h + +limits_config: + allow_structured_metadata: true + retention_period: 168h + volume_enabled: true + otlp_config: + resource_attributes: + attributes_config: + - action: index_label + attributes: + - service.name + - deployment.environment.name + +compactor: + working_directory: /loki/compactor + retention_enabled: true + delete_request_store: filesystem + +analytics: + reporting_enabled: false diff --git a/infra/minio/bootstrap.sh b/infra/minio/bootstrap.sh new file mode 100644 index 0000000..68a2888 --- /dev/null +++ b/infra/minio/bootstrap.sh @@ -0,0 +1,36 @@ +#!/bin/sh + +set -eu + +alias_name=bootstrap +policy_name=messenger-attachments +policy_file=/tmp/messenger-attachments-policy.json + +mc alias set "$alias_name" http://s3:9000 "$MINIO_ROOT_USER" "$MINIO_ROOT_PASSWORD" +mc mb --ignore-existing "$alias_name/$S3_BUCKET" +mc anonymous set none "$alias_name/$S3_BUCKET" + +cat >"$policy_file" </dev/null diff --git a/infra/minio/client.Dockerfile b/infra/minio/client.Dockerfile new file mode 100644 index 0000000..fe8ee0a --- /dev/null +++ b/infra/minio/client.Dockerfile @@ -0,0 +1,16 @@ +FROM golang:1.24.8-alpine3.22 AS build + +ARG MC_VERSION +RUN apk add --no-cache git +RUN git clone --depth 1 --branch "${MC_VERSION}" https://github.com/minio/mc.git /src/mc +WORKDIR /src/mc +RUN go mod download +RUN LDFLAGS="$(MC_RELEASE=RELEASE go run buildscripts/gen-ldflags.go)" && \ + GOMAXPROCS=2 GOMEMLIMIT=1GiB CGO_ENABLED=0 go build -p=1 -tags kqueue -trimpath -ldflags "$LDFLAGS" -o /out/mc . + +FROM alpine:3.22.2 + +COPY --from=build /out/mc /usr/local/bin/mc +COPY infra/minio/bootstrap.sh /usr/local/bin/bootstrap.sh + +ENTRYPOINT ["mc"] diff --git a/infra/minio/server.Dockerfile b/infra/minio/server.Dockerfile new file mode 100644 index 0000000..44b75c4 --- /dev/null +++ b/infra/minio/server.Dockerfile @@ -0,0 +1,18 @@ +FROM golang:1.24.8-alpine3.22 AS build + +ARG MINIO_VERSION +RUN apk add --no-cache git +RUN git clone --depth 1 --branch "${MINIO_VERSION}" https://github.com/minio/minio.git /src/minio +WORKDIR /src/minio +RUN go mod download +RUN LDFLAGS="$(MINIO_RELEASE=RELEASE go run buildscripts/gen-ldflags.go)" && \ + GOMAXPROCS=2 GOMEMLIMIT=1GiB CGO_ENABLED=0 go build -p=1 -tags kqueue -trimpath -ldflags "$LDFLAGS" -o /out/minio . + +FROM alpine:3.22.2 + +RUN addgroup -S minio && adduser -S -G minio minio && mkdir /data && chown minio:minio /data +COPY --from=build /out/minio /usr/local/bin/minio + +USER minio +VOLUME ["/data"] +ENTRYPOINT ["minio"] diff --git a/infra/otel/collector.yaml b/infra/otel/collector.yaml new file mode 100644 index 0000000..b22fa0e --- /dev/null +++ b/infra/otel/collector.yaml @@ -0,0 +1,80 @@ +extensions: + health_check: + endpoint: 0.0.0.0:13133 + +receivers: + otlp: + protocols: + grpc: + endpoint: 0.0.0.0:4317 + http: + endpoint: 0.0.0.0:4318 + +processors: + memory_limiter: + check_interval: 1s + limit_mib: 192 + spike_limit_mib: 48 + batch: + send_batch_size: 1024 + send_batch_max_size: 2048 + timeout: 2s + +exporters: + prometheus: + endpoint: 0.0.0.0:8889 + resource_constant_labels: + included: + - service.name + - service.version + - deployment.environment.name + otlp_grpc/tempo: + endpoint: tempo:4317 + tls: + insecure: true + sending_queue: + enabled: true + num_consumers: 2 + queue_size: 512 + retry_on_failure: + enabled: true + initial_interval: 1s + max_interval: 5s + max_elapsed_time: 30s + otlp_http/loki: + endpoint: http://loki:3100/otlp + sending_queue: + enabled: true + num_consumers: 2 + queue_size: 512 + retry_on_failure: + enabled: true + initial_interval: 1s + max_interval: 5s + max_elapsed_time: 30s + +service: + extensions: [health_check] + telemetry: + logs: + level: info + metrics: + readers: + - pull: + exporter: + prometheus: + host: 0.0.0.0 + port: 8888 + pipelines: + metrics: + receivers: [otlp] + processors: [memory_limiter, batch] + exporters: [prometheus] + logs: + receivers: [otlp] + processors: [memory_limiter, batch] + exporters: [otlp_http/loki] + traces: + receivers: [otlp] + processors: [memory_limiter, batch] + exporters: [otlp_grpc/tempo] diff --git a/infra/tempo/tempo.yaml b/infra/tempo/tempo.yaml new file mode 100644 index 0000000..0d8a229 --- /dev/null +++ b/infra/tempo/tempo.yaml @@ -0,0 +1,48 @@ +server: + http_listen_address: 0.0.0.0 + http_listen_port: 3200 + grpc_listen_address: 0.0.0.0 + grpc_listen_port: 9095 + +distributor: + receivers: + otlp: + protocols: + grpc: + endpoint: 0.0.0.0:4317 + http: + endpoint: 0.0.0.0:4318 + +ingester: + max_block_duration: 5m + +compactor: + compaction: + block_retention: 168h + +storage: + trace: + backend: local + wal: + path: /var/tempo/wal + local: + path: /var/tempo/traces + +metrics_generator: + registry: + external_labels: + source: tempo + deployment_environment: lab + storage: + path: /var/tempo/generator/wal + remote_write: + - url: http://prometheus:9090/api/v1/write + send_exemplars: true + +overrides: + defaults: + metrics_generator: + processors: [service-graphs, span-metrics] + +usage_report: + reporting_enabled: false diff --git a/internal/bootstrap/migration.go b/internal/bootstrap/migration.go new file mode 100644 index 0000000..0e7dd49 --- /dev/null +++ b/internal/bootstrap/migration.go @@ -0,0 +1,25 @@ +package bootstrap + +import ( + "context" + + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/config" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/database" +) + +func RunMigrations() error { + settings, err := config.LoadMigration() + if err != nil { + return err + } + pool, err := database.Open(context.Background(), database.Config{ + URL: settings.DatabaseURL, MinConnections: settings.DBMinConns, MaxConnections: settings.DBMaxConns, + MaxConnectionAge: settings.DBMaxConnectionAge, MaxConnectionIdle: settings.DBMaxConnectionIdle, + HealthCheckPeriod: settings.DBHealthCheckPeriod, ConnectTimeout: settings.DependencyTimeout, + }) + if err != nil { + return err + } + defer pool.Close() + return database.Migrate(context.Background(), pool, settings.MigrationTimeout) +} diff --git a/internal/bootstrap/pipeline.go b/internal/bootstrap/pipeline.go new file mode 100644 index 0000000..1539107 --- /dev/null +++ b/internal/bootstrap/pipeline.go @@ -0,0 +1,222 @@ +package bootstrap + +import ( + "context" + "errors" + "log/slog" + "net" + "net/http" + "os" + "os/signal" + "syscall" + "time" + + "github.com/google/uuid" + "github.com/redis/go-redis/v9" + + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/broker/redisstream" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/config" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/database" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/messenger/attachment/pipeline" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/messenger/attachment/transform" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/messenger/attachment/worker" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/objectstore" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/outbox" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/postgres" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/telemetry" +) + +func RunDispatcher(logger *slog.Logger) error { + settings, err := config.LoadDispatcher() + if err != nil { + return err + } + instanceID, err := os.Hostname() + if err != nil { + return err + } + observability, logger, err := startTelemetry("messenger-dispatcher", instanceID) + if err != nil { + return err + } + defer shutdownTelemetry(observability) + pool, err := database.Open(context.Background(), dispatcherDatabaseConfig(settings)) + if err != nil { + return err + } + defer pool.Close() + if err := telemetry.ObserveDatabasePool(observability.MeterProvider().Meter("messenger-dispatcher/database"), pool); err != nil { + return err + } + client, err := newRedisClient(settings.BrokerURL) + if err != nil { + return err + } + defer func() { _ = client.Close() }() + broker, err := redisstream.New(client, redisstream.Config{ + Stream: settings.Stream, Group: settings.ConsumerGroup, Consumer: "dispatcher", + FailedStream: settings.FailedStream, FailedMaxLen: settings.FailedStreamMaxLength, + ClaimMinIdle: settings.LeaseDuration, ReadCount: int64(settings.BatchSize), + }) + if err != nil { + return err + } + if err := broker.Ensure(context.Background()); err != nil { + return err + } + metrics := pipeline.NewMetrics(observability.MeterProvider().Meter("messenger-dispatcher"), observability.MetricsHandler(), 0) + dispatcher, err := outbox.New(postgres.New(pool), broker, metrics, outbox.Config{ + BatchSize: settings.BatchSize, LeaseDuration: settings.LeaseDuration, + PollInterval: settings.PollInterval, RecoveryInterval: settings.RecoveryInterval, + RetryDelay: settings.RetryDelay, Retention: settings.Retention, + CleanupInterval: settings.CleanupInterval, CleanupBatchSize: settings.CleanupBatchSize, + }) + if err != nil { + return err + } + logger.Info("outbox dispatcher started", "stream", settings.Stream) + return runPipelineProcess(logger, settings.MetricsAddress, metrics.Handler(), dispatcher.Run) +} + +func RunWorker(logger *slog.Logger) error { + settings, err := config.LoadWorker() + if err != nil { + return err + } + hostname, err := os.Hostname() + if err != nil { + return err + } + observability, logger, err := startTelemetry("messenger-worker", hostname) + if err != nil { + return err + } + defer shutdownTelemetry(observability) + pool, err := database.Open(context.Background(), workerDatabaseConfig(settings)) + if err != nil { + return err + } + defer pool.Close() + if err := telemetry.ObserveDatabasePool(observability.MeterProvider().Meter("messenger-worker/database"), pool); err != nil { + return err + } + objects, err := objectstore.New(objectstore.Config{ + Endpoint: settings.S3Endpoint, Region: settings.S3Region, Bucket: settings.S3Bucket, + AccessKey: settings.S3AccessKey, SecretKey: settings.S3SecretKey, + UseTLS: settings.S3UseTLS, Timeout: settings.S3Timeout, + }) + if err != nil { + return err + } + client, err := newRedisClient(settings.BrokerURL) + if err != nil { + return err + } + defer func() { _ = client.Close() }() + consumer := hostname + "-" + uuid.NewString() + broker, err := redisstream.New(client, redisstream.Config{ + Stream: settings.Stream, Group: settings.ConsumerGroup, Consumer: consumer, + FailedStream: settings.FailedStream, FailedMaxLen: settings.FailedStreamMaxLength, Block: settings.ReadBlock, + ClaimMinIdle: settings.LeaseDuration, ReadCount: int64(settings.Prefetch), + }) + if err != nil { + return err + } + if err := broker.Ensure(context.Background()); err != nil { + return err + } + processor, err := transform.NewProcessor(transform.DefaultConfig(settings.AttachmentMaxBytes, settings.AttachmentMaxPixels)) + if err != nil { + return err + } + metrics := pipeline.NewMetrics(observability.MeterProvider().Meter("messenger-worker"), observability.MetricsHandler(), settings.Concurrency) + processorWorker, err := worker.New( + logger, postgres.New(pool), broker, objects, processor, metrics, + worker.Config{ + WorkerID: consumer, Concurrency: settings.Concurrency, LeaseDuration: settings.LeaseDuration, + HeartbeatInterval: settings.HeartbeatInterval, ProcessingTimeout: settings.ProcessingTimeout, + ReclaimInterval: settings.ReclaimInterval, MaxAttempts: settings.MaxAttempts, + RetryBase: settings.RetryBase, RetryMax: settings.RetryMax, + MaxBytes: settings.AttachmentMaxBytes, MaxPixels: settings.AttachmentMaxPixels, + MemoryBytes: settings.MemoryBytes, + }, + ) + if err != nil { + return err + } + logger.Info("attachment worker started", "consumer", consumer, "concurrency", settings.Concurrency) + return runPipelineProcess(logger, settings.MetricsAddress, metrics.Handler(), processorWorker.Run) +} + +func newRedisClient(rawURL string) (*redis.Client, error) { + options, err := redis.ParseURL(rawURL) + if err != nil { + return nil, err + } + client := redis.NewClient(options) + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + if err := client.Ping(ctx).Err(); err != nil { + _ = client.Close() + return nil, err + } + return client, nil +} + +func runPipelineProcess( + logger *slog.Logger, + metricsAddress string, + metricsHandler http.Handler, + run func(context.Context) error, +) error { + mux := http.NewServeMux() + mux.Handle("/metrics", metricsHandler) + mux.HandleFunc("/health/live", func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusNoContent) }) + server := &http.Server{Addr: metricsAddress, Handler: mux, ReadHeaderTimeout: 5 * time.Second} + listener, err := net.Listen("tcp", metricsAddress) + if err != nil { + return err + } + serveResult := make(chan error, 1) + go func() { serveResult <- server.Serve(listener) }() + + ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) + defer stop() + runResult := make(chan error, 1) + go func() { runResult <- run(ctx) }() + + var result error + select { + case result = <-runResult: + stop() + case result = <-serveResult: + stop() + case <-ctx.Done(): + } + shutdownCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + if err := server.Shutdown(shutdownCtx); err != nil && result == nil { + result = err + } + if errors.Is(result, http.ErrServerClosed) || errors.Is(result, context.Canceled) { + return nil + } + logger.Info("pipeline process stopped") + return result +} + +func dispatcherDatabaseConfig(settings config.DispatcherConfig) database.Config { + return database.Config{ + URL: settings.DatabaseURL, MinConnections: settings.DBMinConns, MaxConnections: settings.DBMaxConns, + MaxConnectionAge: settings.DBMaxConnectionAge, MaxConnectionIdle: settings.DBMaxConnectionIdle, + HealthCheckPeriod: settings.DBHealthCheckPeriod, ConnectTimeout: settings.DependencyTimeout, + } +} + +func workerDatabaseConfig(settings config.WorkerConfig) database.Config { + return database.Config{ + URL: settings.DatabaseURL, MinConnections: settings.DBMinConns, MaxConnections: settings.DBMaxConns, + MaxConnectionAge: settings.DBMaxConnectionAge, MaxConnectionIdle: settings.DBMaxConnectionIdle, + HealthCheckPeriod: settings.DBHealthCheckPeriod, ConnectTimeout: settings.DependencyTimeout, + } +} diff --git a/internal/bootstrap/reconciliation.go b/internal/bootstrap/reconciliation.go new file mode 100644 index 0000000..3f09a3f --- /dev/null +++ b/internal/bootstrap/reconciliation.go @@ -0,0 +1,86 @@ +package bootstrap + +import ( + "context" + "encoding/json" + "errors" + "flag" + "io" + "os/signal" + "syscall" + "time" + + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/config" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/database" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/objectstore" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/postgres" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/reconcile" +) + +type reconciliationOutput struct { + Mode string `json:"mode"` + GracePeriod string `json:"grace_period"` + UnreferencedRetention string `json:"unreferenced_retention"` + PageSize int `json:"page_size"` + DurationMillis int64 `json:"duration_ms"` + Report reconcile.Report `json:"report"` +} + +func RunReconciliation(arguments []string, output io.Writer) error { + flags := flag.NewFlagSet("reconcile-attachments", flag.ContinueOnError) + flags.SetOutput(io.Discard) + apply := flags.Bool("apply", false, "apply safe repairs") + gracePeriod := flags.Duration("grace-period", time.Hour, "minimum anomaly age") + retention := flags.Duration("unreferenced-retention", 24*time.Hour, "retention for ready unreferenced attachments") + pageSize := flags.Int("page-size", 500, "S3 and PostgreSQL page size") + if err := flags.Parse(arguments); err != nil || flags.NArg() != 0 { + return errors.New("usage: messenger reconcile-attachments [--apply] [--grace-period=1h] [--unreferenced-retention=24h] [--page-size=500]") + } + if *apply && *gracePeriod <= 0 { + return errors.New("grace-period must be positive in apply mode") + } + settings, err := config.LoadReconciliation() + if err != nil { + return err + } + ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) + defer stop() + pool, err := database.Open(ctx, database.Config{ + URL: settings.DatabaseURL, MinConnections: settings.DBMinConns, MaxConnections: settings.DBMaxConns, + MaxConnectionAge: settings.DBMaxConnectionAge, MaxConnectionIdle: settings.DBMaxConnectionIdle, + HealthCheckPeriod: settings.DBHealthCheckPeriod, ConnectTimeout: settings.DependencyTimeout, + }) + if err != nil { + return err + } + defer pool.Close() + objects, err := objectstore.New(objectstore.Config{ + Endpoint: settings.S3Endpoint, Region: settings.S3Region, Bucket: settings.S3Bucket, + AccessKey: settings.S3AccessKey, SecretKey: settings.S3SecretKey, + UseTLS: settings.S3UseTLS, Timeout: settings.S3Timeout, + }) + if err != nil { + return err + } + reconciler, err := reconcile.New(postgres.New(pool), objects, reconcile.Config{ + Prefix: "attachments/", PageSize: *pageSize, GraceTime: *gracePeriod, + OrphanPrefixes: []string{"staging/"}, + UnreferencedRetention: *retention, Apply: *apply, + }) + if err != nil { + return err + } + started := time.Now() + report, err := reconciler.Run(ctx) + if err != nil { + return err + } + mode := "dry-run" + if *apply { + mode = "apply" + } + return json.NewEncoder(output).Encode(reconciliationOutput{ + Mode: mode, GracePeriod: gracePeriod.String(), UnreferencedRetention: retention.String(), + PageSize: *pageSize, DurationMillis: time.Since(started).Milliseconds(), Report: report, + }) +} diff --git a/internal/bootstrap/reconciliation_test.go b/internal/bootstrap/reconciliation_test.go new file mode 100644 index 0000000..9a9d3b0 --- /dev/null +++ b/internal/bootstrap/reconciliation_test.go @@ -0,0 +1,13 @@ +package bootstrap + +import ( + "io" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestApplyReconciliationRequiresPositiveGracePeriod(t *testing.T) { + err := RunReconciliation([]string{"--apply", "--grace-period=0"}, io.Discard) + require.EqualError(t, err, "grace-period must be positive in apply mode") +} diff --git a/internal/bootstrap/replay.go b/internal/bootstrap/replay.go new file mode 100644 index 0000000..70cc3e8 --- /dev/null +++ b/internal/bootstrap/replay.go @@ -0,0 +1,39 @@ +package bootstrap + +import ( + "context" + "errors" + "fmt" + "io" + + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/config" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/database" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/postgres" +) + +func RunAttachmentReplay(arguments []string, output io.Writer) error { + if len(arguments) != 3 { + return errors.New("usage: messenger replay-attachment ") + } + settings, err := config.LoadJobAdmin() + if err != nil { + return err + } + pool, err := database.Open(context.Background(), database.Config{ + URL: settings.DatabaseURL, MinConnections: settings.DBMinConns, MaxConnections: settings.DBMaxConns, + MaxConnectionAge: settings.DBMaxConnectionAge, MaxConnectionIdle: settings.DBMaxConnectionIdle, + HealthCheckPeriod: settings.DBHealthCheckPeriod, ConnectTimeout: settings.DependencyTimeout, + }) + if err != nil { + return err + } + defer pool.Close() + job, err := postgres.New(pool).ReplayFailedAttachment( + context.Background(), arguments[0], arguments[1], arguments[2], + ) + if err != nil { + return err + } + _, err = fmt.Fprintf(output, "replayed attachment=%s job=%s attempt=%d\n", job.AttachmentID, job.ID, job.Attempt) + return err +} diff --git a/internal/bootstrap/server.go b/internal/bootstrap/server.go new file mode 100644 index 0000000..473c0b7 --- /dev/null +++ b/internal/bootstrap/server.go @@ -0,0 +1,123 @@ +package bootstrap + +import ( + "context" + "log/slog" + "net" + "net/http" + "os" + "os/signal" + "syscall" + + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/config" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/database" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/messenger/attachment" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/messenger/httpapi" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/objectstore" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/password" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/postgres" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/session" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/telemetry" +) + +func RunServer(logger *slog.Logger) error { + settings, err := config.LoadServer() + if err != nil { + return err + } + instanceID, err := os.Hostname() + if err != nil { + return err + } + observability, logger, err := startTelemetry("messenger-api", instanceID) + if err != nil { + return err + } + defer shutdownTelemetry(observability) + pool, err := database.Open(context.Background(), databaseConfig(settings)) + if err != nil { + return err + } + defer pool.Close() + if err := telemetry.ObserveDatabasePool(observability.MeterProvider().Meter("messenger-api/database"), pool); err != nil { + return err + } + sessions, err := session.New(settings.CacheURL) + if err != nil { + return err + } + defer func() { _ = sessions.Close() }() + passwords, err := password.New(password.Config{ + MemoryKiB: settings.PasswordMemoryKiB, Iterations: settings.PasswordIterations, + Parallelism: settings.PasswordParallelism, SaltLength: settings.PasswordSaltLength, + KeyLength: settings.PasswordKeyLength, + }) + if err != nil { + return err + } + objects, err := objectstore.New(objectStoreConfig(settings)) + if err != nil { + return err + } + repository := postgres.New(pool) + attachments, err := attachment.NewService(logger, repository, objects, attachment.Config{ + MaxBytes: settings.AttachmentMaxBytes, MaxPixels: settings.AttachmentMaxPixels, + BodyMemoryBytes: settings.AttachmentBodyMemory, DecodeMemoryBytes: settings.AttachmentDecodeMemory, + CleanupTimeout: settings.RequestTimeout, PipelineVersion: settings.PipelineVersion, + }) + if err != nil { + return err + } + app := httpapi.NewServer(logger, httpapi.Dependencies{ + Users: repository, Chats: repository, Messages: repository, Attachments: attachments, + Sessions: sessions, Passwords: passwords, Database: repository, Objects: objects, + Meter: observability.MeterProvider().Meter("messenger-api"), Metrics: observability.MetricsHandler(), + }, httpapi.Config{ + SessionTTL: settings.SessionTTL, IdempotencyTTL: settings.IdempotencyTTL, + RequestTTL: settings.RequestTimeout, InstanceID: instanceID, SwaggerPort: settings.SwaggerPort, + }) + server := &http.Server{ + Addr: settings.Address, Handler: app.Handler(), ReadHeaderTimeout: settings.HTTPReadHeaderTimeout, + ReadTimeout: settings.HTTPReadTimeout, WriteTimeout: settings.HTTPWriteTimeout, + IdleTimeout: settings.HTTPIdleTimeout, + } + listener, err := net.Listen("tcp", server.Addr) + if err != nil { + return err + } + serveResult := make(chan error, 1) + go func() { serveResult <- server.Serve(listener) }() + logger.Info("server started", "address", server.Addr) + ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) + defer stop() + select { + case err := <-serveResult: + return err + case <-ctx.Done(): + logger.Info("shutdown requested") + app.SetReady(false) + shutdownCtx, cancel := context.WithTimeout(context.Background(), settings.HTTPShutdownTimeout) + defer cancel() + if err := server.Shutdown(shutdownCtx); err != nil { + _ = server.Close() + return err + } + return nil + } +} + +func databaseConfig(settings config.ServerConfig) database.Config { + return database.Config{ + URL: settings.DatabaseURL, MinConnections: settings.DBMinConns, MaxConnections: settings.DBMaxConns, + MaxConnectionAge: settings.DBMaxConnectionAge, MaxConnectionIdle: settings.DBMaxConnectionIdle, + HealthCheckPeriod: settings.DBHealthCheckPeriod, ConnectTimeout: settings.DependencyTimeout, + } +} + +func objectStoreConfig(settings config.ServerConfig) objectstore.Config { + return objectstore.Config{ + Endpoint: settings.S3Endpoint, Region: settings.S3Region, Bucket: settings.S3Bucket, + AccessKey: settings.S3AccessKey, SecretKey: settings.S3SecretKey, + UseTLS: settings.S3UseTLS, Timeout: settings.S3Timeout, + } +} diff --git a/internal/bootstrap/telemetry.go b/internal/bootstrap/telemetry.go new file mode 100644 index 0000000..bb1dcd9 --- /dev/null +++ b/internal/bootstrap/telemetry.go @@ -0,0 +1,23 @@ +package bootstrap + +import ( + "context" + "log/slog" + "time" + + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/telemetry" +) + +func startTelemetry(serviceName, instanceID string) (*telemetry.Runtime, *slog.Logger, error) { + runtime, err := telemetry.New(context.Background(), telemetry.ConfigFromEnvironment(serviceName, instanceID)) + if err != nil { + return nil, nil, err + } + return runtime, runtime.Logger(), nil +} + +func shutdownTelemetry(runtime *telemetry.Runtime) { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + _ = runtime.Shutdown(ctx) +} diff --git a/internal/broker/redisstream/broker.go b/internal/broker/redisstream/broker.go new file mode 100644 index 0000000..0788997 --- /dev/null +++ b/internal/broker/redisstream/broker.go @@ -0,0 +1,309 @@ +package redisstream + +import ( + "context" + "errors" + "fmt" + "regexp" + "strconv" + "strings" + "time" + + "github.com/redis/go-redis/v9" + + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/codes" + "go.opentelemetry.io/otel/trace" + + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/outbox" +) + +const ( + payloadField = "payload" + failedPayloadMaxBytes = 64 << 10 +) + +var failureCodePattern = regexp.MustCompile(`^[a-z][a-z0-9_]{0,63}$`) + +var ErrDeliveryNotOwned = errors.New("redis stream delivery is not owned by this consumer") + +var _ outbox.Publisher = (*Broker)(nil) + +type Config struct { + Stream string + Group string + Consumer string + FailedStream string + FailedMaxLen int64 + Block time.Duration + ClaimMinIdle time.Duration + ReadCount int64 +} + +type Broker struct { + client redis.UniversalClient + config Config +} + +type Delivery struct { + ID string + Envelope Envelope + DecodeError error + + broker *Broker + payload []byte +} + +type FailedDelivery struct { + Code string + FailedAt time.Time +} + +func New(client redis.UniversalClient, config Config) (*Broker, error) { + if client == nil { + return nil, errors.New("redis client is required") + } + if err := config.validate(); err != nil { + return nil, err + } + return &Broker{client: client, config: config}, nil +} + +func (c Config) validate() error { + switch { + case c.Stream == "": + return errors.New("stream is required") + case c.Group == "": + return errors.New("consumer group is required") + case c.Consumer == "": + return errors.New("consumer name is required") + case c.FailedStream == "": + return errors.New("failed stream is required") + case c.FailedStream == c.Stream: + return errors.New("failed stream must differ from job stream") + case c.FailedMaxLen <= 0: + return errors.New("failed stream maximum length must be positive") + case c.Block < 0: + return errors.New("block duration must not be negative") + case c.ClaimMinIdle <= 0: + return errors.New("claim minimum idle time must be positive") + case c.ReadCount <= 0: + return errors.New("read count must be positive") + default: + return nil + } +} + +func (b *Broker) Ensure(ctx context.Context) error { + err := b.client.XGroupCreateMkStream(ctx, b.config.Stream, b.config.Group, "0").Err() + if err != nil && !isBusyGroup(err) { + return fmt.Errorf("create redis stream consumer group: %w", err) + } + return nil +} + +func isBusyGroup(err error) bool { + return err != nil && strings.HasPrefix(err.Error(), "BUSYGROUP") +} + +func (b *Broker) PublishConfirmed(ctx context.Context, payload []byte) (string, error) { + ctx, span := otel.Tracer("redisstream").Start(ctx, "redis XADD", trace.WithSpanKind(trace.SpanKindClient), + trace.WithAttributes(attribute.String("db.system", "redis"), attribute.String("db.operation.name", "XADD"), + attribute.String("messaging.destination.name", b.config.Stream))) + defer span.End() + envelope, err := DecodeEnvelope(payload) + if err != nil { + return "", err + } + payload, err = EncodeEnvelope(envelope) + if err != nil { + return "", err + } + + id, err := b.client.XAdd(ctx, &redis.XAddArgs{ + Stream: b.config.Stream, + Values: map[string]any{payloadField: string(payload)}, + }).Result() + if err != nil { + span.RecordError(err) + span.SetStatus(codes.Error, "XADD failed") + return "", fmt.Errorf("publish redis stream job: %w", err) + } + span.SetAttributes(attribute.String("messaging.message.id", id)) + return id, nil +} + +func (b *Broker) Read(ctx context.Context) ([]Delivery, error) { + ctx, span := otel.Tracer("redisstream").Start(ctx, "redis XREADGROUP", trace.WithSpanKind(trace.SpanKindClient), + trace.WithAttributes(attribute.String("db.system", "redis"), attribute.String("db.operation.name", "XREADGROUP"), + attribute.String("messaging.destination.name", b.config.Stream))) + defer span.End() + streams, err := b.client.XReadGroup(ctx, &redis.XReadGroupArgs{ + Group: b.config.Group, + Consumer: b.config.Consumer, + Streams: []string{b.config.Stream, ">"}, + Count: b.config.ReadCount, + Block: b.config.Block, + }).Result() + if errors.Is(err, redis.Nil) { + return nil, nil + } + if err != nil { + span.RecordError(err) + span.SetStatus(codes.Error, "XREADGROUP failed") + return nil, fmt.Errorf("read redis stream jobs: %w", err) + } + return b.deliveries(streams) +} + +func (b *Broker) Reclaim(ctx context.Context, start string) ([]Delivery, string, error) { + messages, next, err := b.client.XAutoClaim(ctx, &redis.XAutoClaimArgs{ + Stream: b.config.Stream, + Group: b.config.Group, + Consumer: b.config.Consumer, + MinIdle: b.config.ClaimMinIdle, + Start: start, + Count: b.config.ReadCount, + }).Result() + if err != nil { + return nil, start, fmt.Errorf("reclaim redis stream jobs: %w", err) + } + + deliveries, err := b.deliveries([]redis.XStream{{Stream: b.config.Stream, Messages: messages}}) + if err != nil { + return nil, start, err + } + return deliveries, next, nil +} + +func (b *Broker) deliveries(streams []redis.XStream) ([]Delivery, error) { + var deliveries []Delivery + for _, stream := range streams { + for _, message := range stream.Messages { + delivery := Delivery{ID: message.ID, broker: b} + payload, err := messagePayload(message) + if err != nil { + delivery.DecodeError = fmt.Errorf("decode redis stream message %s: %w", message.ID, err) + deliveries = append(deliveries, delivery) + continue + } + delivery.payload = boundedPayload(payload) + envelope, err := DecodeEnvelope(payload) + if err != nil { + delivery.DecodeError = fmt.Errorf("decode redis stream message %s: %w", message.ID, err) + deliveries = append(deliveries, delivery) + continue + } + delivery.Envelope = envelope + delivery.payload, err = EncodeEnvelope(envelope) + if err != nil { + return nil, fmt.Errorf("encode validated redis stream message %s: %w", message.ID, err) + } + deliveries = append(deliveries, delivery) + } + } + return deliveries, nil +} + +func boundedPayload(payload []byte) []byte { + if len(payload) > failedPayloadMaxBytes { + payload = payload[:failedPayloadMaxBytes] + } + return append([]byte(nil), payload...) +} + +func messagePayload(message redis.XMessage) ([]byte, error) { + value, ok := message.Values[payloadField] + if !ok { + return nil, errors.New("payload field is missing") + } + switch payload := value.(type) { + case string: + return []byte(payload), nil + case []byte: + return payload, nil + default: + return nil, fmt.Errorf("payload has type %T", value) + } +} + +func (d Delivery) Ack(ctx context.Context) error { + if d.broker == nil { + return errors.New("delivery is detached from broker") + } + acked, err := d.broker.client.Eval(ctx, ackOwnedScript, []string{d.broker.config.Stream}, + d.broker.config.Group, d.ID, d.broker.config.Consumer, + ).Int64() + if err != nil { + return fmt.Errorf("ack redis stream job: %w", err) + } + if acked == 0 { + return ErrDeliveryNotOwned + } + return nil +} + +const ackOwnedScript = ` +local pending = redis.call('XPENDING', KEYS[1], ARGV[1], ARGV[2], ARGV[2], 1) +if #pending == 0 or pending[1][2] ~= ARGV[3] then + return 0 +end +local acked = redis.call('XACK', KEYS[1], ARGV[1], ARGV[2]) +if acked == 1 then + redis.call('XDEL', KEYS[1], ARGV[2]) +end +return acked +` + +func (d Delivery) ToFailed(ctx context.Context, failure FailedDelivery) error { + if d.broker == nil { + return errors.New("delivery is detached from broker") + } + if !failureCodePattern.MatchString(failure.Code) { + return errors.New("failure code is invalid") + } + if failure.FailedAt.IsZero() { + return errors.New("failure time is required") + } + + payload := d.payload + if d.DecodeError == nil && len(payload) == 0 { + var err error + payload, err = EncodeEnvelope(d.Envelope) + if err != nil { + return err + } + } + moved, err := d.broker.client.Eval(ctx, moveToFailedScript, + []string{d.broker.config.FailedStream, d.broker.config.Stream}, + d.broker.config.Group, d.ID, d.broker.config.Consumer, string(payload), failure.Code, + failure.FailedAt.UTC().Format(time.RFC3339Nano), strconv.Itoa(d.Envelope.Attempt), + strconv.FormatInt(d.broker.config.FailedMaxLen, 10), + ).Int64() + if err != nil { + return fmt.Errorf("move redis stream job to failed stream: %w", err) + } + if moved == 0 { + return ErrDeliveryNotOwned + } + return nil +} + +const moveToFailedScript = ` +local pending = redis.call('XPENDING', KEYS[2], ARGV[1], ARGV[2], ARGV[2], 1) +if #pending == 0 or pending[1][2] ~= ARGV[3] then + return 0 +end +redis.call('XADD', KEYS[1], 'MAXLEN', '~', ARGV[8], '*', + 'payload', ARGV[4], + 'source_message_id', ARGV[2], + 'error_code', ARGV[5], + 'failed_at', ARGV[6], + 'attempt', ARGV[7]) +local acked = redis.call('XACK', KEYS[2], ARGV[1], ARGV[2]) +if acked == 1 then + redis.call('XDEL', KEYS[2], ARGV[2]) +end +return acked +` diff --git a/internal/broker/redisstream/broker_test.go b/internal/broker/redisstream/broker_test.go new file mode 100644 index 0000000..5d96c42 --- /dev/null +++ b/internal/broker/redisstream/broker_test.go @@ -0,0 +1,93 @@ +package redisstream + +import ( + "testing" + "time" + + "github.com/redis/go-redis/v9" + "github.com/stretchr/testify/require" +) + +func TestBrokerDecodesDeliveries(t *testing.T) { + payload, err := EncodeEnvelope(validEnvelope()) + require.NoError(t, err) + + broker := &Broker{config: Config{Stream: "attachment-jobs"}} + deliveries, err := broker.deliveries([]redis.XStream{{ + Stream: "attachment-jobs", + Messages: []redis.XMessage{{ + ID: "1-0", + Values: map[string]any{payloadField: string(payload)}, + }}, + }}) + require.NoError(t, err) + require.Len(t, deliveries, 1) + require.Equal(t, "1-0", deliveries[0].ID) + require.Equal(t, validEnvelope(), deliveries[0].Envelope) +} + +func TestBrokerRejectsPayloadWithStorageMetadata(t *testing.T) { + broker := &Broker{config: Config{Stream: "attachment-jobs"}} + deliveries, err := broker.deliveries([]redis.XStream{{ + Stream: "attachment-jobs", + Messages: []redis.XMessage{{ + ID: "1-0", + Values: map[string]any{payloadField: `{ + "schema_version":1, + "job_id":"11111111-1111-4111-8111-111111111111", + "type":"attachment.process.v1", + "attachment_id":"22222222-2222-4222-8222-222222222222", + "pipeline_version":1, + "attempt":1, + "created_at":"2026-09-01T12:00:00Z", + "storage_key":"private/object" + }`}, + }}, + }}) + require.NoError(t, err) + require.Len(t, deliveries, 1) + require.Error(t, deliveries[0].DecodeError) + require.Contains(t, string(deliveries[0].payload), `"storage_key":"private/object"`) +} + +func TestBrokerBoundsMalformedPayloadForFailedStream(t *testing.T) { + payload := make([]byte, failedPayloadMaxBytes+1) + for index := range payload { + payload[index] = 'x' + } + broker := &Broker{config: Config{Stream: "attachment-jobs"}} + deliveries, err := broker.deliveries([]redis.XStream{{ + Stream: "attachment-jobs", + Messages: []redis.XMessage{{ + ID: "1-0", + Values: map[string]any{payloadField: payload}, + }}, + }}) + require.NoError(t, err) + require.Len(t, deliveries, 1) + require.Error(t, deliveries[0].DecodeError) + require.Len(t, deliveries[0].payload, failedPayloadMaxBytes) +} + +func TestPublishRejectsInvalidPayloadBeforeRedisCall(t *testing.T) { + broker := &Broker{} + _, err := broker.PublishConfirmed(t.Context(), []byte(`{"schema_version":2}`)) + require.Error(t, err) +} + +func TestConfigValidation(t *testing.T) { + config := Config{ + Stream: "attachment-jobs", + Group: "attachment-workers", + Consumer: "worker-1", + FailedStream: "attachment-jobs-failed", + FailedMaxLen: 1000, + Block: time.Second, + ClaimMinIdle: time.Minute, + ReadCount: 10, + } + require.NoError(t, config.validate()) + + config.ClaimMinIdle = 0 + require.Error(t, config.validate()) +} diff --git a/internal/broker/redisstream/envelope.go b/internal/broker/redisstream/envelope.go new file mode 100644 index 0000000..44ffb4a --- /dev/null +++ b/internal/broker/redisstream/envelope.go @@ -0,0 +1,147 @@ +package redisstream + +import ( + "bytes" + "encoding/json" + "errors" + "fmt" + "io" + "regexp" + "time" + "unicode/utf8" + + "github.com/google/uuid" +) + +const ( + SchemaVersion = 1 + JobType = "attachment.process.v1" +) + +var ( + errInvalidEnvelope = errors.New("invalid job envelope") + traceparentPattern = regexp.MustCompile(`^00-[0-9a-f]{32}-[0-9a-f]{16}-[0-9a-f]{2}$`) + uuidPattern = regexp.MustCompile(`^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$`) +) + +type Envelope struct { + SchemaVersion int `json:"schema_version"` + JobID uuid.UUID `json:"job_id"` + Type string `json:"type"` + AttachmentID uuid.UUID `json:"attachment_id"` + PipelineVersion int `json:"pipeline_version"` + Attempt int `json:"attempt"` + CreatedAt time.Time `json:"created_at"` + Traceparent string `json:"traceparent,omitempty"` + Tracestate string `json:"tracestate,omitempty"` +} + +func (e Envelope) Validate() error { + switch { + case e.SchemaVersion != SchemaVersion: + return fmt.Errorf("%w: unsupported schema_version", errInvalidEnvelope) + case e.Type != JobType: + return fmt.Errorf("%w: unsupported type", errInvalidEnvelope) + case e.JobID == uuid.Nil: + return fmt.Errorf("%w: job_id is required", errInvalidEnvelope) + case e.AttachmentID == uuid.Nil: + return fmt.Errorf("%w: attachment_id is required", errInvalidEnvelope) + case e.PipelineVersion < 1: + return fmt.Errorf("%w: pipeline_version must be positive", errInvalidEnvelope) + case e.Attempt < 1: + return fmt.Errorf("%w: attempt must be positive", errInvalidEnvelope) + case e.CreatedAt.IsZero(): + return fmt.Errorf("%w: created_at is required", errInvalidEnvelope) + case e.Traceparent != "" && !traceparentPattern.MatchString(e.Traceparent): + return fmt.Errorf("%w: invalid traceparent", errInvalidEnvelope) + case utf8.RuneCountInString(e.Tracestate) > 512: + return fmt.Errorf("%w: tracestate is too long", errInvalidEnvelope) + default: + return nil + } +} + +func EncodeEnvelope(envelope Envelope) ([]byte, error) { + if err := envelope.Validate(); err != nil { + return nil, err + } + + payload, err := json.Marshal(envelope) + if err != nil { + return nil, fmt.Errorf("encode job envelope: %w", err) + } + + return payload, nil +} + +func DecodeEnvelope(payload []byte) (Envelope, error) { + var wire envelopeJSON + decoder := json.NewDecoder(bytes.NewReader(payload)) + decoder.DisallowUnknownFields() + if err := decoder.Decode(&wire); err != nil { + return Envelope{}, fmt.Errorf("%w: decode: %v", errInvalidEnvelope, err) + } + if err := ensureJSONEnd(decoder); err != nil { + return Envelope{}, err + } + + jobID, err := parseUUID(wire.JobID, "job_id") + if err != nil { + return Envelope{}, err + } + attachmentID, err := parseUUID(wire.AttachmentID, "attachment_id") + if err != nil { + return Envelope{}, err + } + createdAt, err := time.Parse(time.RFC3339, wire.CreatedAt) + if err != nil { + return Envelope{}, fmt.Errorf("%w: invalid created_at", errInvalidEnvelope) + } + envelope := Envelope{ + SchemaVersion: wire.SchemaVersion, + JobID: jobID, + Type: wire.Type, + AttachmentID: attachmentID, + PipelineVersion: wire.PipelineVersion, + Attempt: wire.Attempt, + CreatedAt: createdAt, + Traceparent: wire.Traceparent, + Tracestate: wire.Tracestate, + } + if err := envelope.Validate(); err != nil { + return Envelope{}, err + } + + return envelope, nil +} + +type envelopeJSON struct { + SchemaVersion int `json:"schema_version"` + JobID string `json:"job_id"` + Type string `json:"type"` + AttachmentID string `json:"attachment_id"` + PipelineVersion int `json:"pipeline_version"` + Attempt int `json:"attempt"` + CreatedAt string `json:"created_at"` + Traceparent string `json:"traceparent,omitempty"` + Tracestate string `json:"tracestate,omitempty"` +} + +func parseUUID(value, field string) (uuid.UUID, error) { + if !uuidPattern.MatchString(value) { + return uuid.Nil, fmt.Errorf("%w: invalid %s", errInvalidEnvelope, field) + } + parsed, err := uuid.Parse(value) + if err != nil { + return uuid.Nil, fmt.Errorf("%w: invalid %s", errInvalidEnvelope, field) + } + return parsed, nil +} + +func ensureJSONEnd(decoder *json.Decoder) error { + var value any + if err := decoder.Decode(&value); !errors.Is(err, io.EOF) { + return fmt.Errorf("%w: trailing JSON value", errInvalidEnvelope) + } + return nil +} diff --git a/internal/broker/redisstream/envelope_test.go b/internal/broker/redisstream/envelope_test.go new file mode 100644 index 0000000..f24b68c --- /dev/null +++ b/internal/broker/redisstream/envelope_test.go @@ -0,0 +1,74 @@ +package redisstream + +import ( + "testing" + "time" + + "github.com/google/uuid" + "github.com/stretchr/testify/require" +) + +func TestEnvelopeRoundTrip(t *testing.T) { + envelope := validEnvelope() + payload, err := EncodeEnvelope(envelope) + require.NoError(t, err) + + decoded, err := DecodeEnvelope(payload) + require.NoError(t, err) + require.Equal(t, envelope, decoded) +} + +func TestDecodeEnvelopeRejectsUnknownField(t *testing.T) { + payload := []byte(`{ + "schema_version":1, + "job_id":"11111111-1111-4111-8111-111111111111", + "type":"attachment.process.v1", + "attachment_id":"22222222-2222-4222-8222-222222222222", + "pipeline_version":1, + "attempt":1, + "created_at":"2026-09-01T12:00:00Z", + "storage_key":"must-not-enter-the-broker" + }`) + + _, err := DecodeEnvelope(payload) + require.Error(t, err) +} + +func TestEnvelopeValidation(t *testing.T) { + tests := []struct { + name string + mutate func(*Envelope) + }{ + {name: "schema", mutate: func(e *Envelope) { e.SchemaVersion = 2 }}, + {name: "type", mutate: func(e *Envelope) { e.Type = "attachment.delete.v1" }}, + {name: "job id", mutate: func(e *Envelope) { e.JobID = uuid.Nil }}, + {name: "attachment id", mutate: func(e *Envelope) { e.AttachmentID = uuid.Nil }}, + {name: "pipeline", mutate: func(e *Envelope) { e.PipelineVersion = 0 }}, + {name: "attempt", mutate: func(e *Envelope) { e.Attempt = 0 }}, + {name: "created at", mutate: func(e *Envelope) { e.CreatedAt = time.Time{} }}, + {name: "traceparent", mutate: func(e *Envelope) { e.Traceparent = "invalid" }}, + {name: "tracestate length", mutate: func(e *Envelope) { e.Tracestate = string(make([]byte, 513)) }}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + envelope := validEnvelope() + tt.mutate(&envelope) + require.Error(t, envelope.Validate()) + }) + } +} + +func validEnvelope() Envelope { + return Envelope{ + SchemaVersion: SchemaVersion, + JobID: uuid.MustParse("11111111-1111-4111-8111-111111111111"), + Type: JobType, + AttachmentID: uuid.MustParse("22222222-2222-4222-8222-222222222222"), + PipelineVersion: 1, + Attempt: 1, + CreatedAt: time.Date(2026, 9, 1, 12, 0, 0, 0, time.UTC), + Traceparent: "00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01", + Tracestate: "vendor=value", + } +} diff --git a/internal/config/config.go b/internal/config/config.go new file mode 100644 index 0000000..977abc5 --- /dev/null +++ b/internal/config/config.go @@ -0,0 +1,416 @@ +package config + +import ( + "errors" + "fmt" + "net" + "strconv" + "strings" + "time" + + "github.com/spf13/viper" +) + +type ServerConfig struct { + Address string + DatabaseURL string + CacheURL string + SessionTTL time.Duration + IdempotencyTTL time.Duration + DBMaxConns int32 + DBMinConns int32 + DBMaxConnectionAge time.Duration + DBMaxConnectionIdle time.Duration + DBHealthCheckPeriod time.Duration + DependencyTimeout time.Duration + RequestTimeout time.Duration + HTTPReadHeaderTimeout time.Duration + HTTPReadTimeout time.Duration + HTTPWriteTimeout time.Duration + HTTPIdleTimeout time.Duration + HTTPShutdownTimeout time.Duration + PasswordMemoryKiB uint32 + PasswordIterations uint32 + PasswordParallelism uint8 + PasswordSaltLength uint32 + PasswordKeyLength uint32 + SwaggerPort string + S3Endpoint string + S3Region string + S3Bucket string + S3AccessKey string + S3SecretKey string + S3UseTLS bool + S3Timeout time.Duration + AttachmentMaxBytes int64 + AttachmentMaxPixels int64 + AttachmentBodyMemory int64 + AttachmentDecodeMemory int64 + PipelineVersion int +} + +type MigrationConfig struct { + DatabaseURL string + DBMaxConns int32 + DBMinConns int32 + DBMaxConnectionAge time.Duration + DBMaxConnectionIdle time.Duration + DBHealthCheckPeriod time.Duration + DependencyTimeout time.Duration + MigrationTimeout time.Duration +} + +type ReconciliationConfig struct { + DatabaseURL string + DBMaxConns int32 + DBMinConns int32 + DBMaxConnectionAge time.Duration + DBMaxConnectionIdle time.Duration + DBHealthCheckPeriod time.Duration + DependencyTimeout time.Duration + S3Endpoint string + S3Region string + S3Bucket string + S3AccessKey string + S3SecretKey string + S3UseTLS bool + S3Timeout time.Duration +} + +type databaseConfig struct { + url string + maxConns int + minConns int + maxConnectionAge time.Duration + maxConnectionIdle time.Duration + healthCheckPeriod time.Duration + dependencyTimeout time.Duration +} + +type objectStorageConfig struct { + endpoint string + region string + bucket string + accessKey string + secretKey string + useTLS bool + timeout time.Duration +} + +func LoadServer() (ServerConfig, error) { + values, err := loadValues() + if err != nil { + return ServerConfig{}, err + } + + host, err := requiredString(values, "server.host") + if err != nil { + return ServerConfig{}, err + } + port, err := requiredPort(values, "server.port") + if err != nil { + return ServerConfig{}, err + } + database, err := loadDatabaseConfig(values) + if err != nil { + return ServerConfig{}, err + } + cacheURL, err := requiredString(values, "cache.url") + if err != nil { + return ServerConfig{}, err + } + store, err := requiredString(values, "server.session_store") + if err != nil { + return ServerConfig{}, err + } + if store != "valkey" { + return ServerConfig{}, errors.New("server.session_store must be valkey") + } + + sessionTTL, err := duration(values, "server.session_ttl", time.Second, 30*24*time.Hour) + if err != nil { + return ServerConfig{}, err + } + idempotencyTTL, err := duration(values, "server.idempotency_ttl", 24*time.Hour, 30*24*time.Hour) + if err != nil { + return ServerConfig{}, err + } + requestTimeout, err := duration(values, "server.request_timeout", time.Second, 5*time.Minute) + if err != nil { + return ServerConfig{}, err + } + readHeaderTimeout, err := duration(values, "server.http.read_header_timeout", time.Second, 5*time.Minute) + if err != nil { + return ServerConfig{}, err + } + readTimeout, err := duration(values, "server.http.read_timeout", time.Second, 5*time.Minute) + if err != nil { + return ServerConfig{}, err + } + writeTimeout, err := duration(values, "server.http.write_timeout", time.Second, 5*time.Minute) + if err != nil { + return ServerConfig{}, err + } + idleTimeout, err := duration(values, "server.http.idle_timeout", time.Second, time.Hour) + if err != nil { + return ServerConfig{}, err + } + shutdownTimeout, err := duration(values, "server.http.shutdown_timeout", time.Second, 5*time.Minute) + if err != nil { + return ServerConfig{}, err + } + passwordMemory, err := integer(values, "server.password.memory_kib", 8*1024, 1024*1024) + if err != nil { + return ServerConfig{}, err + } + passwordIterations, err := integer(values, "server.password.iterations", 1, 20) + if err != nil { + return ServerConfig{}, err + } + passwordParallelism, err := integer(values, "server.password.parallelism", 1, 255) + if err != nil { + return ServerConfig{}, err + } + passwordSaltLength, err := integer(values, "server.password.salt_length", 8, 1024) + if err != nil { + return ServerConfig{}, err + } + passwordKeyLength, err := integer(values, "server.password.key_length", 16, 1024) + if err != nil { + return ServerConfig{}, err + } + swaggerPort, err := requiredPort(values, "server.swagger_port") + if err != nil { + return ServerConfig{}, err + } + storage, err := loadObjectStorageConfig(values) + if err != nil { + return ServerConfig{}, err + } + attachmentMaxBytes, err := integer(values, "attachments.max_bytes", 1, 10*1024*1024) + if err != nil { + return ServerConfig{}, err + } + attachmentMaxPixels, err := integer(values, "attachments.max_pixels", 1, 40_000_000) + if err != nil { + return ServerConfig{}, err + } + attachmentBodyMemory, err := integer(values, "attachments.body_memory_bytes", 1, 1024*1024*1024) + if err != nil { + return ServerConfig{}, err + } + if int64(attachmentBodyMemory) < int64(attachmentMaxBytes) { + return ServerConfig{}, fmt.Errorf("attachments.body_memory_bytes: must be at least %d", attachmentMaxBytes) + } + attachmentDecodeMemory, err := integer(values, "attachments.decode_memory_bytes", 1, 1024*1024*1024) + if err != nil { + return ServerConfig{}, err + } + minimumDecodeMemory := int64(attachmentMaxPixels) * 8 + if int64(attachmentDecodeMemory) < minimumDecodeMemory { + return ServerConfig{}, fmt.Errorf("attachments.decode_memory_bytes: must be at least %d", minimumDecodeMemory) + } + pipelineVersion, err := integer(values, "pipeline.version", 1, 1000) + if err != nil { + return ServerConfig{}, err + } + + return ServerConfig{ + Address: net.JoinHostPort(host, port), DatabaseURL: database.url, CacheURL: cacheURL, + SessionTTL: sessionTTL, IdempotencyTTL: idempotencyTTL, + DBMaxConns: int32(database.maxConns), DBMinConns: int32(database.minConns), + DBMaxConnectionAge: database.maxConnectionAge, DBMaxConnectionIdle: database.maxConnectionIdle, + DBHealthCheckPeriod: database.healthCheckPeriod, DependencyTimeout: database.dependencyTimeout, + RequestTimeout: requestTimeout, + HTTPReadHeaderTimeout: readHeaderTimeout, HTTPReadTimeout: readTimeout, + HTTPWriteTimeout: writeTimeout, HTTPIdleTimeout: idleTimeout, + HTTPShutdownTimeout: shutdownTimeout, PasswordMemoryKiB: uint32(passwordMemory), + PasswordIterations: uint32(passwordIterations), PasswordParallelism: uint8(passwordParallelism), + PasswordSaltLength: uint32(passwordSaltLength), PasswordKeyLength: uint32(passwordKeyLength), + SwaggerPort: swaggerPort, S3Endpoint: storage.endpoint, S3Region: storage.region, S3Bucket: storage.bucket, + S3AccessKey: storage.accessKey, S3SecretKey: storage.secretKey, + S3UseTLS: storage.useTLS, S3Timeout: storage.timeout, + AttachmentMaxBytes: int64(attachmentMaxBytes), AttachmentMaxPixels: int64(attachmentMaxPixels), + AttachmentBodyMemory: int64(attachmentBodyMemory), + AttachmentDecodeMemory: int64(attachmentDecodeMemory), + PipelineVersion: pipelineVersion, + }, nil +} + +func LoadMigration() (MigrationConfig, error) { + values, err := loadValues() + if err != nil { + return MigrationConfig{}, err + } + database, err := loadDatabaseConfig(values) + if err != nil { + return MigrationConfig{}, err + } + migrationTimeout, err := duration(values, "migration.timeout", time.Second, 10*time.Minute) + if err != nil { + return MigrationConfig{}, err + } + return MigrationConfig{ + DatabaseURL: database.url, DBMaxConns: int32(database.maxConns), DBMinConns: int32(database.minConns), + DBMaxConnectionAge: database.maxConnectionAge, DBMaxConnectionIdle: database.maxConnectionIdle, + DBHealthCheckPeriod: database.healthCheckPeriod, DependencyTimeout: database.dependencyTimeout, + MigrationTimeout: migrationTimeout, + }, nil +} + +func LoadReconciliation() (ReconciliationConfig, error) { + values, err := loadValues() + if err != nil { + return ReconciliationConfig{}, err + } + database, err := loadDatabaseConfig(values) + if err != nil { + return ReconciliationConfig{}, err + } + storage, err := loadObjectStorageConfig(values) + if err != nil { + return ReconciliationConfig{}, err + } + return ReconciliationConfig{ + DatabaseURL: database.url, DBMaxConns: int32(database.maxConns), DBMinConns: int32(database.minConns), + DBMaxConnectionAge: database.maxConnectionAge, DBMaxConnectionIdle: database.maxConnectionIdle, + DBHealthCheckPeriod: database.healthCheckPeriod, DependencyTimeout: database.dependencyTimeout, + S3Endpoint: storage.endpoint, S3Region: storage.region, S3Bucket: storage.bucket, + S3AccessKey: storage.accessKey, S3SecretKey: storage.secretKey, + S3UseTLS: storage.useTLS, S3Timeout: storage.timeout, + }, nil +} + +func loadObjectStorageConfig(values *viper.Viper) (objectStorageConfig, error) { + endpoint, err := requiredString(values, "object_storage.endpoint") + if err != nil { + return objectStorageConfig{}, err + } + region, err := requiredString(values, "object_storage.region") + if err != nil { + return objectStorageConfig{}, err + } + bucket, err := requiredString(values, "object_storage.bucket") + if err != nil { + return objectStorageConfig{}, err + } + accessKey, err := requiredString(values, "object_storage.access_key") + if err != nil { + return objectStorageConfig{}, err + } + secretKey, err := requiredString(values, "object_storage.secret_key") + if err != nil { + return objectStorageConfig{}, err + } + useTLS, err := boolean(values, "object_storage.use_tls") + if err != nil { + return objectStorageConfig{}, err + } + timeout, err := duration(values, "object_storage.timeout", time.Second, 5*time.Minute) + if err != nil { + return objectStorageConfig{}, err + } + return objectStorageConfig{ + endpoint: endpoint, region: region, bucket: bucket, accessKey: accessKey, + secretKey: secretKey, useTLS: useTLS, timeout: timeout, + }, nil +} + +func loadDatabaseConfig(values *viper.Viper) (databaseConfig, error) { + databaseURL, err := requiredString(values, "database.url") + if err != nil { + return databaseConfig{}, err + } + maxConns, err := integer(values, "database.max_connections", 1, 100) + if err != nil { + return databaseConfig{}, err + } + minConns, err := integer(values, "database.min_connections", 0, maxConns) + if err != nil { + return databaseConfig{}, err + } + maxConnectionAge, err := duration(values, "database.max_connection_age", time.Second, 24*time.Hour) + if err != nil { + return databaseConfig{}, err + } + maxConnectionIdle, err := duration(values, "database.max_connection_idle", time.Second, 24*time.Hour) + if err != nil { + return databaseConfig{}, err + } + healthCheckPeriod, err := duration(values, "database.health_check_period", time.Second, time.Hour) + if err != nil { + return databaseConfig{}, err + } + dependencyTimeout, err := duration(values, "database.dependency_timeout", time.Second, time.Minute) + if err != nil { + return databaseConfig{}, err + } + return databaseConfig{ + url: databaseURL, maxConns: maxConns, minConns: minConns, + maxConnectionAge: maxConnectionAge, maxConnectionIdle: maxConnectionIdle, + healthCheckPeriod: healthCheckPeriod, dependencyTimeout: dependencyTimeout, + }, nil +} + +func requiredString(values *viper.Viper, name string) (string, error) { + value := strings.TrimSpace(values.GetString(name)) + if value == "" { + return "", fmt.Errorf("%s is required", name) + } + return value, nil +} + +func requiredPort(values *viper.Viper, name string) (string, error) { + value, err := requiredString(values, name) + if err != nil { + return "", err + } + if _, err := integerValue(value, 1, 65535); err != nil { + return "", fmt.Errorf("%s: %w", name, err) + } + return value, nil +} + +func duration(values *viper.Viper, name string, minimum, maximum time.Duration) (time.Duration, error) { + value, err := requiredString(values, name) + if err != nil { + return 0, err + } + parsed, err := time.ParseDuration(value) + if err != nil || parsed < minimum || parsed > maximum { + return 0, fmt.Errorf("%s: must be between %s and %s", name, minimum, maximum) + } + return parsed, nil +} + +func integer(values *viper.Viper, name string, minimum, maximum int) (int, error) { + value, err := requiredString(values, name) + if err != nil { + return 0, err + } + parsed, err := integerValue(value, minimum, maximum) + if err != nil { + return 0, fmt.Errorf("%s: %w", name, err) + } + return parsed, nil +} + +func integerValue(value string, minimum, maximum int) (int, error) { + parsed, err := strconv.Atoi(value) + if err != nil || parsed < minimum || parsed > maximum { + return 0, fmt.Errorf("must be between %d and %d", minimum, maximum) + } + return parsed, nil +} + +func boolean(values *viper.Viper, name string) (bool, error) { + value, err := requiredString(values, name) + if err != nil { + return false, err + } + parsed, err := strconv.ParseBool(value) + if err != nil { + return false, fmt.Errorf("%s: must be true or false", name) + } + return parsed, nil +} diff --git a/internal/config/config_test.go b/internal/config/config_test.go new file mode 100644 index 0000000..80d2216 --- /dev/null +++ b/internal/config/config_test.go @@ -0,0 +1,131 @@ +package config + +import ( + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +func TestLoadReconciliationDoesNotRequireServerSettings(t *testing.T) { + setConfigEnvironment(t, validTestConfig) + t.Setenv("APP_HOST", "") + t.Setenv("CACHE_URL", "") + t.Setenv("SWAGGER_PORT", "") + + settings, err := LoadReconciliation() + require.NoError(t, err) + require.Equal(t, "attachments", settings.S3Bucket) +} + +func TestLoadWorkerUsesYAMLForRuntimeSettings(t *testing.T) { + setConfigEnvironment(t, validTestConfig) + t.Setenv("WORKER_CONCURRENCY", "64") + + settings, err := LoadWorker() + require.NoError(t, err) + require.Equal(t, 2, settings.Concurrency) + require.Equal(t, 5*time.Second, settings.HeartbeatInterval) +} + +func TestLoadWorkerRejectsHeartbeatAtLeaseBoundary(t *testing.T) { + content := strings.Replace(validTestConfig, "heartbeat_interval: 5s", "heartbeat_interval: 30s", 1) + setConfigEnvironment(t, content) + + _, err := LoadWorker() + require.ErrorContains(t, err, "heartbeat_interval must be less than worker.lease") +} + +func TestLoadValuesRejectsUnknownVersion(t *testing.T) { + setConfigEnvironment(t, "version: 2\n") + + _, err := loadValues() + require.ErrorContains(t, err, "version: must be 1") +} + +func setConfigEnvironment(t *testing.T, content string) { + t.Helper() + path := filepath.Join(t.TempDir(), "messenger.yaml") + require.NoError(t, os.WriteFile(path, []byte(content), 0o600)) + t.Setenv("MESSENGER_CONFIG_PATH", path) + for name, value := range map[string]string{ + "APP_HOST": "0.0.0.0", "APP_PORT": "8080", "SWAGGER_PORT": "8081", + "DATABASE_URL": "postgres://user:password@postgres:5432/messenger", + "CACHE_URL": "redis://valkey:6379/0", "BROKER_URL": "redis://redis-broker:6379/0", + "S3_ENDPOINT": "s3:9000", "S3_REGION": "us-east-1", "S3_BUCKET": "attachments", + "S3_ACCESS_KEY": "application", "S3_SECRET_KEY": "application-secret", + } { + t.Setenv(name, value) + } +} + +const validTestConfig = `version: 1 +server: + session_store: valkey + session_ttl: 1h + idempotency_ttl: 24h + request_timeout: 5s + http: + read_header_timeout: 5s + read_timeout: 10s + write_timeout: 10s + idle_timeout: 1m + shutdown_timeout: 5s + password: + memory_kib: 65536 + iterations: 3 + parallelism: 2 + salt_length: 16 + key_length: 32 +database: + min_connections: 0 + max_connections: 4 + max_connection_age: 5m + max_connection_idle: 1m + health_check_period: 30s + dependency_timeout: 5s +migration: + timeout: 30s +object_storage: + use_tls: false + timeout: 5s +attachments: + max_bytes: 10485760 + max_pixels: 40000000 + body_memory_bytes: 67108864 + decode_memory_bytes: 402653184 +pipeline: + version: 1 +broker: + stream: attachment-jobs + consumer_group: attachment-workers + failed_stream: attachment-jobs-failed + failed_stream_max_length: 10000 +outbox: + batch_size: 32 + lease: 15s + poll_interval: 1s + recovery_interval: 10s + retry_delay: 2s + retention: 168h + cleanup_interval: 1m + cleanup_batch_size: 1000 +worker: + concurrency: 2 + prefetch: 4 + lease: 30s + heartbeat_interval: 5s + processing_timeout: 20s + read_block: 2s + reclaim_interval: 5s + max_attempts: 5 + retry: + base: 1s + max: 30s + memory_bytes: 738197504 +monitoring: + metrics_address: 0.0.0.0:9091 +` diff --git a/internal/config/loader.go b/internal/config/loader.go new file mode 100644 index 0000000..6cd115c --- /dev/null +++ b/internal/config/loader.go @@ -0,0 +1,47 @@ +package config + +import ( + "errors" + "fmt" + "os" + "strings" + + "github.com/spf13/viper" +) + +const supportedConfigVersion = 1 + +var environmentKeys = map[string]string{ + "server.host": "APP_HOST", + "server.port": "APP_PORT", + "server.swagger_port": "SWAGGER_PORT", + "database.url": "DATABASE_URL", + "cache.url": "CACHE_URL", + "object_storage.endpoint": "S3_ENDPOINT", + "object_storage.region": "S3_REGION", + "object_storage.bucket": "S3_BUCKET", + "object_storage.access_key": "S3_ACCESS_KEY", + "object_storage.secret_key": "S3_SECRET_KEY", + "broker.url": "BROKER_URL", +} + +func loadValues() (*viper.Viper, error) { + path := strings.TrimSpace(os.Getenv("MESSENGER_CONFIG_PATH")) + if path == "" { + return nil, errors.New("MESSENGER_CONFIG_PATH is required") + } + values := viper.New() + values.SetConfigFile(path) + if err := values.ReadInConfig(); err != nil { + return nil, fmt.Errorf("read messenger config: %w", err) + } + if version := values.GetInt("version"); version != supportedConfigVersion { + return nil, fmt.Errorf("version: must be %d", supportedConfigVersion) + } + for key, name := range environmentKeys { + if err := values.BindEnv(key, name); err != nil { + return nil, fmt.Errorf("bind %s: %w", name, err) + } + } + return values, nil +} diff --git a/internal/config/pipeline.go b/internal/config/pipeline.go new file mode 100644 index 0000000..76b4f5d --- /dev/null +++ b/internal/config/pipeline.go @@ -0,0 +1,307 @@ +package config + +import ( + "errors" + "fmt" + "net" + "time" + + "github.com/spf13/viper" +) + +type DispatcherConfig struct { + DatabaseURL string + DBMaxConns int32 + DBMinConns int32 + DBMaxConnectionAge time.Duration + DBMaxConnectionIdle time.Duration + DBHealthCheckPeriod time.Duration + DependencyTimeout time.Duration + BrokerURL string + Stream string + ConsumerGroup string + FailedStream string + FailedStreamMaxLength int64 + BatchSize int + LeaseDuration time.Duration + PollInterval time.Duration + RecoveryInterval time.Duration + RetryDelay time.Duration + Retention time.Duration + CleanupInterval time.Duration + CleanupBatchSize int + MetricsAddress string +} + +type WorkerConfig struct { + DatabaseURL string + DBMaxConns int32 + DBMinConns int32 + DBMaxConnectionAge time.Duration + DBMaxConnectionIdle time.Duration + DBHealthCheckPeriod time.Duration + DependencyTimeout time.Duration + S3Endpoint string + S3Region string + S3Bucket string + S3AccessKey string + S3SecretKey string + S3UseTLS bool + S3Timeout time.Duration + BrokerURL string + Stream string + ConsumerGroup string + FailedStream string + FailedStreamMaxLength int64 + Concurrency int + Prefetch int + LeaseDuration time.Duration + HeartbeatInterval time.Duration + ProcessingTimeout time.Duration + ReadBlock time.Duration + ReclaimInterval time.Duration + MaxAttempts int + RetryBase time.Duration + RetryMax time.Duration + AttachmentMaxBytes int64 + AttachmentMaxPixels int64 + MemoryBytes int64 + MetricsAddress string +} + +type JobAdminConfig struct { + DatabaseURL string + DBMaxConns int32 + DBMinConns int32 + DBMaxConnectionAge time.Duration + DBMaxConnectionIdle time.Duration + DBHealthCheckPeriod time.Duration + DependencyTimeout time.Duration +} + +type brokerConfig struct { + url string + stream string + consumerGroup string + failedStream string + failedStreamMaxLength int +} + +func LoadJobAdmin() (JobAdminConfig, error) { + values, err := loadValues() + if err != nil { + return JobAdminConfig{}, err + } + database, err := loadDatabaseConfig(values) + if err != nil { + return JobAdminConfig{}, err + } + return JobAdminConfig{ + DatabaseURL: database.url, DBMaxConns: int32(database.maxConns), DBMinConns: int32(database.minConns), + DBMaxConnectionAge: database.maxConnectionAge, DBMaxConnectionIdle: database.maxConnectionIdle, + DBHealthCheckPeriod: database.healthCheckPeriod, DependencyTimeout: database.dependencyTimeout, + }, nil +} + +func LoadDispatcher() (DispatcherConfig, error) { + values, err := loadValues() + if err != nil { + return DispatcherConfig{}, err + } + database, err := loadDatabaseConfig(values) + if err != nil { + return DispatcherConfig{}, err + } + broker, err := loadBrokerConfig(values) + if err != nil { + return DispatcherConfig{}, err + } + batchSize, err := integer(values, "outbox.batch_size", 1, 1000) + if err != nil { + return DispatcherConfig{}, err + } + leaseDuration, err := duration(values, "outbox.lease", time.Second, 10*time.Minute) + if err != nil { + return DispatcherConfig{}, err + } + pollInterval, err := duration(values, "outbox.poll_interval", time.Millisecond, time.Minute) + if err != nil { + return DispatcherConfig{}, err + } + recoveryInterval, err := duration(values, "outbox.recovery_interval", time.Second, time.Hour) + if err != nil { + return DispatcherConfig{}, err + } + retryDelay, err := duration(values, "outbox.retry_delay", time.Second, time.Hour) + if err != nil { + return DispatcherConfig{}, err + } + retention, err := duration(values, "outbox.retention", time.Hour, 365*24*time.Hour) + if err != nil { + return DispatcherConfig{}, err + } + cleanupInterval, err := duration(values, "outbox.cleanup_interval", time.Minute, 24*time.Hour) + if err != nil { + return DispatcherConfig{}, err + } + cleanupBatchSize, err := integer(values, "outbox.cleanup_batch_size", 1, 1000) + if err != nil { + return DispatcherConfig{}, err + } + metricsAddress, err := metricsAddress(values) + if err != nil { + return DispatcherConfig{}, err + } + return DispatcherConfig{ + DatabaseURL: database.url, DBMaxConns: int32(database.maxConns), DBMinConns: int32(database.minConns), + DBMaxConnectionAge: database.maxConnectionAge, DBMaxConnectionIdle: database.maxConnectionIdle, + DBHealthCheckPeriod: database.healthCheckPeriod, DependencyTimeout: database.dependencyTimeout, + BrokerURL: broker.url, Stream: broker.stream, ConsumerGroup: broker.consumerGroup, + FailedStream: broker.failedStream, FailedStreamMaxLength: int64(broker.failedStreamMaxLength), + BatchSize: batchSize, LeaseDuration: leaseDuration, PollInterval: pollInterval, + RecoveryInterval: recoveryInterval, RetryDelay: retryDelay, + Retention: retention, CleanupInterval: cleanupInterval, CleanupBatchSize: cleanupBatchSize, + MetricsAddress: metricsAddress, + }, nil +} + +func LoadWorker() (WorkerConfig, error) { + values, err := loadValues() + if err != nil { + return WorkerConfig{}, err + } + database, err := loadDatabaseConfig(values) + if err != nil { + return WorkerConfig{}, err + } + storage, err := loadObjectStorageConfig(values) + if err != nil { + return WorkerConfig{}, err + } + broker, err := loadBrokerConfig(values) + if err != nil { + return WorkerConfig{}, err + } + concurrency, err := integer(values, "worker.concurrency", 1, 64) + if err != nil { + return WorkerConfig{}, err + } + prefetch, err := integer(values, "worker.prefetch", concurrency, 1024) + if err != nil { + return WorkerConfig{}, err + } + leaseDuration, err := duration(values, "worker.lease", time.Second, time.Hour) + if err != nil { + return WorkerConfig{}, err + } + heartbeatInterval, err := duration(values, "worker.heartbeat_interval", 100*time.Millisecond, time.Hour) + if err != nil { + return WorkerConfig{}, err + } + if heartbeatInterval >= leaseDuration { + return WorkerConfig{}, errors.New("worker.heartbeat_interval must be less than worker.lease") + } + processingTimeout, err := duration(values, "worker.processing_timeout", time.Second, time.Hour) + if err != nil { + return WorkerConfig{}, err + } + if processingTimeout >= leaseDuration { + return WorkerConfig{}, errors.New("worker.processing_timeout must be less than worker.lease") + } + readBlock, err := duration(values, "worker.read_block", 100*time.Millisecond, time.Minute) + if err != nil { + return WorkerConfig{}, err + } + reclaimInterval, err := duration(values, "worker.reclaim_interval", time.Second, time.Hour) + if err != nil { + return WorkerConfig{}, err + } + maxAttempts, err := integer(values, "worker.max_attempts", 1, 100) + if err != nil { + return WorkerConfig{}, err + } + retryBase, err := duration(values, "worker.retry.base", time.Second, time.Hour) + if err != nil { + return WorkerConfig{}, err + } + retryMax, err := duration(values, "worker.retry.max", time.Second, 24*time.Hour) + if err != nil { + return WorkerConfig{}, err + } + if retryMax < retryBase { + return WorkerConfig{}, errors.New("worker.retry.max must be at least worker.retry.base") + } + attachmentMaxBytes, err := integer(values, "attachments.max_bytes", 1, 10*1024*1024) + if err != nil { + return WorkerConfig{}, err + } + attachmentMaxPixels, err := integer(values, "attachments.max_pixels", 1, 40_000_000) + if err != nil { + return WorkerConfig{}, err + } + memoryBytes, err := integer(values, "worker.memory_bytes", attachmentMaxBytes, 2*1024*1024*1024) + if err != nil { + return WorkerConfig{}, err + } + metricsAddress, err := metricsAddress(values) + if err != nil { + return WorkerConfig{}, err + } + return WorkerConfig{ + DatabaseURL: database.url, DBMaxConns: int32(database.maxConns), DBMinConns: int32(database.minConns), + DBMaxConnectionAge: database.maxConnectionAge, DBMaxConnectionIdle: database.maxConnectionIdle, + DBHealthCheckPeriod: database.healthCheckPeriod, DependencyTimeout: database.dependencyTimeout, + S3Endpoint: storage.endpoint, S3Region: storage.region, S3Bucket: storage.bucket, + S3AccessKey: storage.accessKey, S3SecretKey: storage.secretKey, S3UseTLS: storage.useTLS, + S3Timeout: storage.timeout, BrokerURL: broker.url, Stream: broker.stream, + ConsumerGroup: broker.consumerGroup, FailedStream: broker.failedStream, + FailedStreamMaxLength: int64(broker.failedStreamMaxLength), + Concurrency: concurrency, Prefetch: prefetch, LeaseDuration: leaseDuration, + HeartbeatInterval: heartbeatInterval, ProcessingTimeout: processingTimeout, ReadBlock: readBlock, + ReclaimInterval: reclaimInterval, MaxAttempts: maxAttempts, RetryBase: retryBase, RetryMax: retryMax, + AttachmentMaxBytes: int64(attachmentMaxBytes), AttachmentMaxPixels: int64(attachmentMaxPixels), + MemoryBytes: int64(memoryBytes), MetricsAddress: metricsAddress, + }, nil +} + +func metricsAddress(values *viper.Viper) (string, error) { + address, err := requiredString(values, "monitoring.metrics_address") + if err != nil { + return "", err + } + _, port, err := net.SplitHostPort(address) + if err != nil { + return "", fmt.Errorf("monitoring.metrics_address: %w", err) + } + if _, err := integerValue(port, 1, 65535); err != nil { + return "", fmt.Errorf("monitoring.metrics_address: %w", err) + } + return address, nil +} + +func loadBrokerConfig(values *viper.Viper) (brokerConfig, error) { + url, err := requiredString(values, "broker.url") + if err != nil { + return brokerConfig{}, err + } + stream, err := requiredString(values, "broker.stream") + if err != nil { + return brokerConfig{}, err + } + consumerGroup, err := requiredString(values, "broker.consumer_group") + if err != nil { + return brokerConfig{}, err + } + failedStream, err := requiredString(values, "broker.failed_stream") + if err != nil { + return brokerConfig{}, err + } + failedStreamMaxLength, err := integer(values, "broker.failed_stream_max_length", 100, 10_000_000) + if err != nil { + return brokerConfig{}, err + } + return brokerConfig{ + url: url, stream: stream, consumerGroup: consumerGroup, + failedStream: failedStream, failedStreamMaxLength: failedStreamMaxLength, + }, nil +} diff --git a/internal/database/database.go b/internal/database/database.go new file mode 100644 index 0000000..c631b06 --- /dev/null +++ b/internal/database/database.go @@ -0,0 +1,94 @@ +package database + +import ( + "context" + "errors" + "fmt" + "time" + + "github.com/jackc/pgx/v5/pgxpool" + + "gitea.privcloud.tech/dizi/mtusi2026lab6/migrations" +) + +var ( + errURLRequired = errors.New("database URL is required") + errMinConnections = errors.New("minimum database connections must not be negative") + errMaxConnections = errors.New("maximum database connections must be positive") + errConnectionRange = errors.New("minimum database connections exceed maximum") + errMaxConnectionAge = errors.New("maximum database connection age must be positive") + errMaxConnectionIdle = errors.New("maximum database connection idle time must be positive") + errHealthCheckPeriod = errors.New("database health check period must be positive") + errConnectionTimeout = errors.New("database connection timeout must be positive") +) + +type Config struct { + URL string + MinConnections int32 + MaxConnections int32 + MaxConnectionAge time.Duration + MaxConnectionIdle time.Duration + HealthCheckPeriod time.Duration + ConnectTimeout time.Duration +} + +func Open(ctx context.Context, config Config) (*pgxpool.Pool, error) { + if err := validateConfig(config); err != nil { + return nil, err + } + poolConfig, err := pgxpool.ParseConfig(config.URL) + if err != nil { + return nil, fmt.Errorf("parse database configuration: %w", err) + } + poolConfig.ConnConfig.Tracer = queryTracer{} + poolConfig.MinConns = config.MinConnections + poolConfig.MaxConns = config.MaxConnections + poolConfig.MaxConnLifetime = config.MaxConnectionAge + poolConfig.MaxConnIdleTime = config.MaxConnectionIdle + poolConfig.HealthCheckPeriod = config.HealthCheckPeriod + + connectCtx, cancel := context.WithTimeout(ctx, config.ConnectTimeout) + defer cancel() + pool, err := pgxpool.NewWithConfig(connectCtx, poolConfig) + if err != nil { + return nil, fmt.Errorf("create database pool: %w", err) + } + if err := pool.Ping(connectCtx); err != nil { + pool.Close() + return nil, fmt.Errorf("ping database: %w", err) + } + + return pool, nil +} + +func Migrate(ctx context.Context, pool *pgxpool.Pool, timeout time.Duration) error { + if timeout <= 0 { + return errors.New("database migration timeout must be positive") + } + migrationCtx, cancel := context.WithTimeout(ctx, timeout) + defer cancel() + return applyMigrations(migrationCtx, pool, migrations.FS) +} + +func validateConfig(config Config) error { + switch { + case config.URL == "": + return errURLRequired + case config.MinConnections < 0: + return errMinConnections + case config.MaxConnections < 1: + return errMaxConnections + case config.MinConnections > config.MaxConnections: + return errConnectionRange + case config.MaxConnectionAge <= 0: + return errMaxConnectionAge + case config.MaxConnectionIdle <= 0: + return errMaxConnectionIdle + case config.HealthCheckPeriod <= 0: + return errHealthCheckPeriod + case config.ConnectTimeout <= 0: + return errConnectionTimeout + default: + return nil + } +} diff --git a/internal/database/migrations.go b/internal/database/migrations.go new file mode 100644 index 0000000..095c93d --- /dev/null +++ b/internal/database/migrations.go @@ -0,0 +1,150 @@ +package database + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "io/fs" + "path/filepath" + "regexp" + "sort" + "strconv" + "time" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" +) + +const ( + migrationLockID int64 = 0x4d54555349324c42 + migrationUnlockTimeout = 5 * time.Second +) + +var migrationNamePattern = regexp.MustCompile(`^(\d+)_([a-z0-9_]+)\.sql$`) + +type migration struct { + version int64 + name string + checksum string + sql string +} + +func applyMigrations(ctx context.Context, pool *pgxpool.Pool, migrationFS fs.FS) (returnErr error) { + migrations, err := readMigrations(migrationFS) + if err != nil { + return err + } + connection, err := pool.Acquire(ctx) + if err != nil { + return fmt.Errorf("acquire connection for migrations: %w", err) + } + defer connection.Release() + if _, err := connection.Exec(ctx, "SELECT pg_advisory_lock($1)", migrationLockID); err != nil { + return fmt.Errorf("lock migrations: %w", err) + } + defer func() { + unlockCtx, cancel := context.WithTimeout(context.Background(), migrationUnlockTimeout) + defer cancel() + if _, err := connection.Exec(unlockCtx, "SELECT pg_advisory_unlock($1)", migrationLockID); err != nil { + _ = connection.Conn().Close(unlockCtx) + if returnErr == nil { + returnErr = fmt.Errorf("unlock migrations: %w", err) + } + } + }() + + if _, err := connection.Exec(ctx, ` +CREATE TABLE IF NOT EXISTS schema_migrations ( + version BIGINT PRIMARY KEY, + name TEXT NOT NULL, + checksum TEXT NOT NULL, + applied_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp() +)`); err != nil { + return fmt.Errorf("create migration history: %w", err) + } + for _, migration := range migrations { + if err := applyMigration(ctx, connection, migration); err != nil { + return err + } + } + return nil +} + +func readMigrations(migrationFS fs.FS) ([]migration, error) { + entries, err := fs.ReadDir(migrationFS, ".") + if err != nil { + return nil, fmt.Errorf("read embedded migrations: %w", err) + } + migrations := make([]migration, 0, len(entries)) + versions := make(map[int64]string) + for _, entry := range entries { + if entry.IsDir() || filepath.Ext(entry.Name()) != ".sql" { + continue + } + matches := migrationNamePattern.FindStringSubmatch(entry.Name()) + if matches == nil { + return nil, fmt.Errorf("invalid migration filename %q", entry.Name()) + } + version, err := strconv.ParseInt(matches[1], 10, 64) + if err != nil || version < 1 { + return nil, fmt.Errorf("invalid migration version in %q", entry.Name()) + } + if previous, exists := versions[version]; exists { + return nil, fmt.Errorf("migration version %d is used by %q and %q", version, previous, entry.Name()) + } + data, err := fs.ReadFile(migrationFS, entry.Name()) + if err != nil { + return nil, fmt.Errorf("read migration %q: %w", entry.Name(), err) + } + hash := sha256.Sum256(data) + versions[version] = entry.Name() + migrations = append(migrations, migration{ + version: version, + name: entry.Name(), + checksum: hex.EncodeToString(hash[:]), + sql: string(data), + }) + } + if len(migrations) == 0 { + return nil, errors.New("no database migrations found") + } + sort.Slice(migrations, func(i, j int) bool { + return migrations[i].version < migrations[j].version + }) + return migrations, nil +} + +func applyMigration(ctx context.Context, connection *pgxpool.Conn, migration migration) error { + var checksum string + err := connection.QueryRow(ctx, "SELECT checksum FROM schema_migrations WHERE version = $1", migration.version).Scan(&checksum) + switch { + case err == nil && checksum != migration.checksum: + return fmt.Errorf("migration %q checksum does not match applied version", migration.name) + case err == nil: + return nil + case !errors.Is(err, pgx.ErrNoRows): + return fmt.Errorf("read migration %q state: %w", migration.name, err) + } + + transaction, err := connection.Begin(ctx) + if err != nil { + return fmt.Errorf("begin migration %q: %w", migration.name, err) + } + defer func() { + _ = transaction.Rollback(context.WithoutCancel(ctx)) + }() + if _, err := transaction.Exec(ctx, migration.sql, pgx.QueryExecModeSimpleProtocol); err != nil { + return fmt.Errorf("apply migration %q: %w", migration.name, err) + } + if _, err := transaction.Exec(ctx, ` +INSERT INTO schema_migrations (version, name, checksum) +VALUES ($1, $2, $3)`, migration.version, migration.name, migration.checksum); err != nil { + return fmt.Errorf("record migration %q: %w", migration.name, err) + } + if err := transaction.Commit(ctx); err != nil { + return fmt.Errorf("commit migration %q: %w", migration.name, err) + } + return nil +} diff --git a/internal/database/telemetry.go b/internal/database/telemetry.go new file mode 100644 index 0000000..e2c34f8 --- /dev/null +++ b/internal/database/telemetry.go @@ -0,0 +1,28 @@ +package database + +import ( + "context" + + "github.com/jackc/pgx/v5" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/codes" + "go.opentelemetry.io/otel/trace" +) + +type queryTracer struct{} + +func (queryTracer) TraceQueryStart(ctx context.Context, _ *pgx.Conn, _ pgx.TraceQueryStartData) context.Context { + ctx, _ = otel.Tracer("postgres").Start(ctx, "postgres query", trace.WithSpanKind(trace.SpanKindClient), + trace.WithAttributes(attribute.String("db.system", "postgresql"))) + return ctx +} + +func (queryTracer) TraceQueryEnd(ctx context.Context, _ *pgx.Conn, data pgx.TraceQueryEndData) { + span := trace.SpanFromContext(ctx) + if data.Err != nil { + span.RecordError(data.Err) + span.SetStatus(codes.Error, "query failed") + } + span.End() +} diff --git a/internal/logger/logger.go b/internal/logger/logger.go new file mode 100644 index 0000000..ab81366 --- /dev/null +++ b/internal/logger/logger.go @@ -0,0 +1,10 @@ +package logger + +import ( + "log/slog" + "os" +) + +func New() *slog.Logger { + return slog.New(slog.NewJSONHandler(os.Stdout, nil)) +} diff --git a/internal/messenger/attachment/pipeline/metrics.go b/internal/messenger/attachment/pipeline/metrics.go new file mode 100644 index 0000000..3bc4ef7 --- /dev/null +++ b/internal/messenger/attachment/pipeline/metrics.go @@ -0,0 +1,97 @@ +package pipeline + +import ( + "context" + "net/http" + "time" + + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/metric" +) + +var workerDurationBuckets = []float64{0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 15, 20, 30, 60} + +type Metrics struct { + meter metric.Meter + workerInFlight metric.Int64UpDownCounter + workerJobs metric.Int64Counter + workerDuration metric.Float64Histogram + workerRetries metric.Int64Counter + workerReclaimed metric.Int64Counter + outboxPublished metric.Int64Counter + outboxPublishFailure metric.Int64Counter + outboxRecovered metric.Int64Counter + handler http.Handler +} + +func NewMetrics(meter metric.Meter, handler http.Handler, workerCapacity int) *Metrics { + if meter == nil { + meter = otel.Meter("messenger-pipeline") + } + if handler == nil { + handler = http.NotFoundHandler() + } + workerInFlight, _ := meter.Int64UpDownCounter("messenger_attachment_worker_jobs_in_flight", metric.WithDescription("Attachment jobs currently processed by this worker.")) + workerJobs, _ := meter.Int64Counter("messenger_attachment_worker_jobs", metric.WithDescription("Completed attachment job deliveries by result.")) + workerDuration, _ := meter.Float64Histogram("messenger_attachment_worker_job_duration", + metric.WithDescription("Attachment job delivery processing duration."), metric.WithUnit("s"), + metric.WithExplicitBucketBoundaries(workerDurationBuckets...), + ) + workerRetries, _ := meter.Int64Counter("messenger_attachment_worker_retries", metric.WithDescription("Attachment jobs scheduled for another attempt.")) + workerReclaimed, _ := meter.Int64Counter("messenger_attachment_worker_reclaimed_deliveries", metric.WithDescription("Redis Stream deliveries reclaimed after their idle timeout.")) + outboxPublished, _ := meter.Int64Counter("messenger_outbox_published", metric.WithDescription("Outbox events confirmed by Redis Streams.")) + outboxPublishFailure, _ := meter.Int64Counter("messenger_outbox_publish_failures", metric.WithDescription("Outbox events whose Redis Streams publish failed.")) + outboxRecovered, _ := meter.Int64Counter("messenger_outbox_recovered_leases", metric.WithDescription("Expired outbox leases returned for publication.")) + if workerCapacity > 0 { + _, _ = meter.Int64ObservableGauge("messenger_attachment_worker_capacity", + metric.WithDescription("Maximum concurrent attachment jobs for this worker."), + metric.WithInt64Callback(func(_ context.Context, observer metric.Int64Observer) error { + observer.Observe(int64(workerCapacity)) + return nil + }), + ) + } + return &Metrics{ + meter: meter, workerInFlight: workerInFlight, workerJobs: workerJobs, + workerDuration: workerDuration, + workerRetries: workerRetries, workerReclaimed: workerReclaimed, + outboxPublished: outboxPublished, outboxPublishFailure: outboxPublishFailure, + outboxRecovered: outboxRecovered, handler: handler, + } +} + +func (m *Metrics) Handler() http.Handler { + return m.handler +} + +func (m *Metrics) ObserveStarted(ctx context.Context) { + m.workerInFlight.Add(ctx, 1) +} + +func (m *Metrics) ObserveFinished(ctx context.Context, result string, duration time.Duration) { + attributes := metric.WithAttributes(attribute.String("job_type", "attachment.process"), attribute.String("result", result)) + m.workerInFlight.Add(ctx, -1) + m.workerJobs.Add(ctx, 1, attributes) + m.workerDuration.Record(ctx, duration.Seconds(), attributes) +} + +func (m *Metrics) ObserveRetry(ctx context.Context) { + m.workerRetries.Add(ctx, 1) +} + +func (m *Metrics) ObserveReclaimed(ctx context.Context, count int) { + m.workerReclaimed.Add(ctx, int64(count)) +} + +func (m *Metrics) ObservePublished(ctx context.Context) { + m.outboxPublished.Add(ctx, 1) +} + +func (m *Metrics) ObservePublishFailure(ctx context.Context) { + m.outboxPublishFailure.Add(ctx, 1) +} + +func (m *Metrics) ObserveRecovered(ctx context.Context, count int) { + m.outboxRecovered.Add(ctx, int64(count)) +} diff --git a/internal/messenger/attachment/service.go b/internal/messenger/attachment/service.go new file mode 100644 index 0000000..38e5bdc --- /dev/null +++ b/internal/messenger/attachment/service.go @@ -0,0 +1,237 @@ +package attachment + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "image" + _ "image/jpeg" + _ "image/png" + "io" + "log/slog" + "math" + "mime" + "net/http" + "time" + + "github.com/google/uuid" + "golang.org/x/sync/semaphore" + + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/telemetry" +) + +type Repository interface { + ChatForUser(context.Context, string, string) (model.Chat, error) + CreatePendingAttachment(context.Context, model.Attachment) (model.Attachment, error) + QueueAttachmentWithJobAndOutbox(context.Context, string, string, model.AttachmentJob, string, string) (model.Attachment, error) + DeletePendingAttachment(context.Context, string, string) error + AttachmentForUser(context.Context, string, string) (model.Attachment, error) + AttachmentVariantForUser(context.Context, string, string, string) (model.Attachment, model.Variant, error) +} + +type ObjectStore interface { + Put(context.Context, string, string, []byte) error + Get(context.Context, string) (io.ReadCloser, error) + Delete(context.Context, string) error +} + +type Config struct { + MaxBytes int64 + MaxPixels int64 + BodyMemoryBytes int64 + DecodeMemoryBytes int64 + CleanupTimeout time.Duration + PipelineVersion int +} + +type Upload struct { + ChatID string + OwnerID string + Filename string + ContentType string + ContentLength int64 + Body io.Reader + TraceContext telemetry.MessageCarrier +} + +type Content struct { + Attachment model.Attachment + Variant model.Variant + Body io.ReadCloser +} + +type Service struct { + repository Repository + objects ObjectStore + config Config + bodyMemory *semaphore.Weighted + decodeMemory *semaphore.Weighted + logger *slog.Logger +} + +func NewService(logger *slog.Logger, repository Repository, objects ObjectStore, config Config) (*Service, error) { + if config.MaxBytes < 1 || config.MaxPixels < 1 || config.MaxPixels > math.MaxInt64/8 || config.CleanupTimeout <= 0 || config.PipelineVersion < 1 { + return nil, errors.New("invalid attachment service configuration") + } + if config.BodyMemoryBytes < config.MaxBytes { + return nil, errors.New("attachment body memory must fit one maximum upload") + } + if config.DecodeMemoryBytes < config.MaxPixels*8 { + return nil, errors.New("attachment decode memory must fit one maximum image") + } + return &Service{ + repository: repository, objects: objects, config: config, + bodyMemory: semaphore.NewWeighted(config.BodyMemoryBytes), + decodeMemory: semaphore.NewWeighted(config.DecodeMemoryBytes), logger: logger, + }, nil +} + +func (s *Service) Upload(ctx context.Context, input Upload) (model.Attachment, error) { + if _, err := s.repository.ChatForUser(ctx, input.ChatID, input.OwnerID); err != nil { + return model.Attachment{}, mapDependencyError(err) + } + data, contentType, release, err := s.readImage(input) + if err != nil { + return model.Attachment{}, err + } + defer release() + + attachmentID := uuid.NewString() + stagingKey := "staging/" + attachmentID + "/" + uuid.NewString() + digest := sha256.Sum256(data) + pending, err := s.repository.CreatePendingAttachment(ctx, model.Attachment{ + ID: attachmentID, ChatID: input.ChatID, OwnerID: input.OwnerID, Filename: input.Filename, + ContentType: contentType, SizeBytes: int64(len(data)), SHA256: hex.EncodeToString(digest[:]), + ObjectKey: stagingKey, StagingKey: stagingKey, Status: model.AttachmentPending, + PipelineVersion: s.config.PipelineVersion, + }) + if err != nil { + return model.Attachment{}, mapDependencyError(err) + } + if err := s.objects.Put(ctx, stagingKey, contentType, data); err != nil { + s.logger.ErrorContext(ctx, "attachment staging write failed", "attachment_id", pending.ID, "error", err) + s.deletePending(ctx, pending.ID, stagingKey) + return model.Attachment{}, model.ErrUnavailable + } + createdAt := time.Now().UTC() + queued, err := s.repository.QueueAttachmentWithJobAndOutbox(ctx, pending.ID, stagingKey, model.AttachmentJob{ + ID: uuid.NewString(), AttachmentID: pending.ID, PipelineVersion: s.config.PipelineVersion, + Attempt: 1, Status: model.AttachmentJobQueued, AvailableAt: createdAt, CreatedAt: createdAt, + }, input.TraceContext.Traceparent, input.TraceContext.Tracestate) + if err != nil { + s.logger.ErrorContext(ctx, "attachment queue transaction failed", "attachment_id", pending.ID, "error", err) + return model.Attachment{}, mapDependencyError(err) + } + return queued, nil +} + +func (s *Service) deletePending(ctx context.Context, attachmentID, stagingKey string) { + cleanupCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), s.config.CleanupTimeout) + defer cancel() + if err := s.repository.DeletePendingAttachment(cleanupCtx, attachmentID, stagingKey); err != nil { + s.logger.ErrorContext(cleanupCtx, "pending attachment cleanup failed", "attachment_id", attachmentID, "error", err) + } +} + +func (s *Service) Metadata(ctx context.Context, attachmentID, userID string) (model.Attachment, error) { + attachment, err := s.repository.AttachmentForUser(ctx, attachmentID, userID) + if err != nil { + return model.Attachment{}, mapDependencyError(err) + } + return attachment, nil +} + +func (s *Service) Content(ctx context.Context, attachmentID, userID, variantName string) (Content, error) { + attachment, variant, err := s.repository.AttachmentVariantForUser(ctx, attachmentID, userID, variantName) + if err != nil { + return Content{}, mapDependencyError(err) + } + body, err := s.objects.Get(ctx, variant.ObjectKey) + if err != nil { + return Content{}, model.ErrUnavailable + } + return Content{Attachment: attachment, Variant: variant, Body: body}, nil +} + +func (s *Service) readImage(input Upload) ([]byte, string, func(), error) { + mediaType, parameters, err := mime.ParseMediaType(input.ContentType) + if err != nil || len(parameters) != 0 || mediaType != "image/png" && mediaType != "image/jpeg" { + return nil, "", nil, model.ErrUnsupportedMedia + } + if input.ContentLength > s.config.MaxBytes { + return nil, "", nil, model.ErrPayloadTooLarge + } + reservation := s.config.MaxBytes + if !s.bodyMemory.TryAcquire(reservation) { + return nil, "", nil, model.ErrCapacity + } + releaseBody := func() { s.bodyMemory.Release(reservation) } + data, err := io.ReadAll(io.LimitReader(input.Body, s.config.MaxBytes+1)) + if err != nil { + releaseBody() + return nil, "", nil, model.ErrInvalidInput + } + if int64(len(data)) > s.config.MaxBytes { + releaseBody() + return nil, "", nil, model.ErrPayloadTooLarge + } + if len(data) == 0 { + releaseBody() + return nil, "", nil, model.ErrUnsupportedMedia + } + detected := http.DetectContentType(data) + if detected != mediaType { + releaseBody() + return nil, "", nil, model.ErrUnsupportedMedia + } + imageConfig, format, err := image.DecodeConfig(bytes.NewReader(data)) + if err != nil || imageContentType(format) != mediaType { + releaseBody() + return nil, "", nil, model.ErrUnsupportedMedia + } + if imageConfig.Width < 1 || imageConfig.Height < 1 || int64(imageConfig.Width) > s.config.MaxPixels/int64(imageConfig.Height) { + releaseBody() + return nil, "", nil, model.ErrPayloadTooLarge + } + pixels := int64(imageConfig.Width) * int64(imageConfig.Height) + decodeBytes := pixels * 8 + if !s.decodeMemory.TryAcquire(decodeBytes) { + releaseBody() + return nil, "", nil, model.ErrCapacity + } + if _, format, err := image.Decode(bytes.NewReader(data)); err != nil || imageContentType(format) != mediaType { + s.decodeMemory.Release(decodeBytes) + releaseBody() + return nil, "", nil, model.ErrUnsupportedMedia + } + release := func() { + s.decodeMemory.Release(decodeBytes) + releaseBody() + } + return data, mediaType, release, nil +} + +func imageContentType(format string) string { + switch format { + case "png": + return "image/png" + case "jpeg": + return "image/jpeg" + default: + return "" + } +} + +func mapDependencyError(err error) error { + switch { + case errors.Is(err, model.ErrNotFound): + return model.ErrNotFound + case errors.Is(err, model.ErrConflict): + return model.ErrConflict + default: + return model.ErrUnavailable + } +} diff --git a/internal/messenger/attachment/service_test.go b/internal/messenger/attachment/service_test.go new file mode 100644 index 0000000..296f283 --- /dev/null +++ b/internal/messenger/attachment/service_test.go @@ -0,0 +1,220 @@ +package attachment + +import ( + "bytes" + "context" + "errors" + "image" + "image/color" + "image/png" + "io" + "log/slog" + "math" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model" +) + +func TestUploadLifecycle(t *testing.T) { + data := pngFixture(t, 2, 2) + events := make([]string, 0, 3) + repository := &repositoryStub{events: &events} + objects := &objectStub{events: &events} + service := newTestService(t, repository, objects, 1024, 4) + + result, err := service.Upload(context.Background(), upload(data, "image/png")) + require.NoError(t, err) + require.Equal(t, model.AttachmentQueued, result.Status) + require.Equal(t, []string{"pending", "put", "queued"}, events) +} + +func TestUploadValidation(t *testing.T) { + data := pngFixture(t, 2, 2) + tests := []struct { + name string + contentType string + data []byte + maxBytes int64 + maxPixels int64 + want error + }{ + {name: "spoofed MIME", contentType: "image/jpeg", data: data, maxBytes: 1024, maxPixels: 4, want: model.ErrUnsupportedMedia}, + {name: "too many pixels", contentType: "image/png", data: data, maxBytes: 1024, maxPixels: 3, want: model.ErrPayloadTooLarge}, + {name: "too many bytes", contentType: "image/png", data: data, maxBytes: int64(len(data) - 1), maxPixels: 4, want: model.ErrPayloadTooLarge}, + {name: "damaged", contentType: "image/png", data: data[:16], maxBytes: 1024, maxPixels: 4, want: model.ErrUnsupportedMedia}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + service := newTestService(t, &repositoryStub{}, &objectStub{}, test.maxBytes, test.maxPixels) + input := upload(test.data, test.contentType) + _, err := service.Upload(context.Background(), input) + require.ErrorIs(t, err, test.want) + }) + } +} + +func TestUploadUsesSeparateBodyAndDecodeBudgets(t *testing.T) { + data := pngFixture(t, 2, 2) + service := newTestService(t, &repositoryStub{}, &objectStub{}, 1024, 4) + require.True(t, service.bodyMemory.TryAcquire(1024)) + _, err := service.Upload(context.Background(), upload(data, "image/png")) + require.ErrorIs(t, err, model.ErrCapacity) + service.bodyMemory.Release(1024) + + require.True(t, service.decodeMemory.TryAcquire(32)) + _, err = service.Upload(context.Background(), upload(data, "image/png")) + require.ErrorIs(t, err, model.ErrCapacity) + service.decodeMemory.Release(32) +} + +func TestUploadReleasesBodyMemoryAfterOversizedDimensions(t *testing.T) { + oversized := pngFixture(t, 3, 2) + valid := pngFixture(t, 2, 2) + service := newTestService(t, &repositoryStub{}, &objectStub{}, 1024, 4) + + _, err := service.Upload(context.Background(), upload(oversized, "image/png")) + require.ErrorIs(t, err, model.ErrPayloadTooLarge) + _, err = service.Upload(context.Background(), upload(valid, "image/png")) + require.NoError(t, err) +} + +func TestNewServiceRejectsDecodeBudgetOverflow(t *testing.T) { + _, err := NewService( + slog.New(slog.NewTextHandler(io.Discard, nil)), &repositoryStub{}, &objectStub{}, + Config{ + MaxBytes: 1024, MaxPixels: math.MaxInt64/8 + 1, BodyMemoryBytes: 1024, + DecodeMemoryBytes: math.MaxInt64, CleanupTimeout: time.Second, PipelineVersion: 1, + }, + ) + require.Error(t, err) +} + +func TestUploadCleansPendingMetadataWhenObjectWriteFails(t *testing.T) { + data := pngFixture(t, 2, 2) + repository := &repositoryStub{} + objects := &objectStub{putErr: errors.New("s3 unavailable")} + service := newTestService(t, repository, objects, 1024, 4) + + _, err := service.Upload(context.Background(), upload(data, "image/png")) + require.ErrorIs(t, err, model.ErrUnavailable) + require.True(t, repository.pendingDeleted) +} + +func TestUploadKeepsStagingAfterQueueFailure(t *testing.T) { + data := pngFixture(t, 2, 2) + repository := &repositoryStub{queueErr: errors.New("connection lost")} + objects := &objectStub{} + service := newTestService(t, repository, objects, 1024, 4) + + _, err := service.Upload(context.Background(), upload(data, "image/png")) + require.ErrorIs(t, err, model.ErrUnavailable) + require.False(t, objects.deleted) + require.False(t, repository.pendingDeleted) +} + +func newTestService(t *testing.T, repository Repository, objects ObjectStore, maxBytes, maxPixels int64) *Service { + t.Helper() + service, err := NewService( + slog.New(slog.NewTextHandler(io.Discard, nil)), repository, objects, + Config{ + MaxBytes: maxBytes, MaxPixels: maxPixels, BodyMemoryBytes: maxBytes, + DecodeMemoryBytes: maxPixels * 8, CleanupTimeout: time.Second, PipelineVersion: 1, + }, + ) + require.NoError(t, err) + return service +} + +func upload(data []byte, contentType string) Upload { + return Upload{ + ChatID: "chat", OwnerID: "owner", Filename: "photo.png", ContentType: contentType, + ContentLength: int64(len(data)), Body: bytes.NewReader(data), + } +} + +type repositoryStub struct { + events *[]string + queueErr error + pendingDeleted bool + pending model.Attachment +} + +func (*repositoryStub) ChatForUser(context.Context, string, string) (model.Chat, error) { + return model.Chat{}, nil +} + +func (s *repositoryStub) CreatePendingAttachment(_ context.Context, attachment model.Attachment) (model.Attachment, error) { + attachment.Status = model.AttachmentPending + s.pending = attachment + s.record("pending") + return attachment, nil +} + +func (s *repositoryStub) QueueAttachmentWithJobAndOutbox( + context.Context, + string, + string, + model.AttachmentJob, + string, + string, +) (model.Attachment, error) { + if s.queueErr != nil { + return model.Attachment{}, s.queueErr + } + s.pending.Status = model.AttachmentQueued + s.record("queued") + return s.pending, nil +} + +func (s *repositoryStub) DeletePendingAttachment(context.Context, string, string) error { + s.pendingDeleted = true + return nil +} + +func (s *repositoryStub) AttachmentForUser(context.Context, string, string) (model.Attachment, error) { + return s.pending, nil +} + +func (s *repositoryStub) AttachmentVariantForUser(context.Context, string, string, string) (model.Attachment, model.Variant, error) { + return s.pending, model.Variant{}, nil +} + +func (s *repositoryStub) record(event string) { + if s.events != nil { + *s.events = append(*s.events, event) + } +} + +type objectStub struct { + events *[]string + putErr error + deleted bool +} + +func (s *objectStub) Put(context.Context, string, string, []byte) error { + if s.events != nil { + *s.events = append(*s.events, "put") + } + return s.putErr +} + +func (*objectStub) Get(context.Context, string) (io.ReadCloser, error) { + return io.NopCloser(bytes.NewReader(nil)), nil +} + +func (s *objectStub) Delete(context.Context, string) error { + s.deleted = true + return nil +} + +func pngFixture(t *testing.T, width, height int) []byte { + t.Helper() + var data bytes.Buffer + value := image.NewRGBA(image.Rect(0, 0, width, height)) + value.Set(0, 0, color.White) + require.NoError(t, png.Encode(&data, value)) + return data.Bytes() +} diff --git a/internal/messenger/attachment/transform/exif.go b/internal/messenger/attachment/transform/exif.go new file mode 100644 index 0000000..53e532b --- /dev/null +++ b/internal/messenger/attachment/transform/exif.go @@ -0,0 +1,169 @@ +package transform + +import ( + "context" + "encoding/binary" + "errors" + "image" + "image/color" +) + +var errInvalidEXIF = errors.New("invalid EXIF metadata") + +func jpegOrientation(ctx context.Context, data []byte) (int, error) { + if len(data) < 2 || data[0] != 0xff || data[1] != 0xd8 { + return 0, errInvalidEXIF + } + for offset := 2; offset < len(data); { + if err := contextError(ctx); err != nil { + return 0, err + } + if data[offset] != 0xff { + return 0, errInvalidEXIF + } + for offset < len(data) && data[offset] == 0xff { + offset++ + } + if offset >= len(data) { + return 0, errInvalidEXIF + } + marker := data[offset] + offset++ + if marker == 0xd9 || marker == 0xda { + return 1, nil + } + if marker == 0x01 || marker >= 0xd0 && marker <= 0xd7 { + continue + } + if offset+2 > len(data) { + return 0, errInvalidEXIF + } + length := int(binary.BigEndian.Uint16(data[offset : offset+2])) + if length < 2 || offset+length > len(data) { + return 0, errInvalidEXIF + } + payload := data[offset+2 : offset+length] + offset += length + if marker != 0xe1 || len(payload) < 6 || string(payload[:6]) != "Exif\x00\x00" { + continue + } + return tiffOrientation(ctx, payload[6:]) + } + return 1, nil +} + +func tiffOrientation(ctx context.Context, data []byte) (int, error) { + if len(data) < 8 { + return 0, errInvalidEXIF + } + var order binary.ByteOrder + switch string(data[:2]) { + case "II": + order = binary.LittleEndian + case "MM": + order = binary.BigEndian + default: + return 0, errInvalidEXIF + } + if order.Uint16(data[2:4]) != 42 { + return 0, errInvalidEXIF + } + ifdOffset := uint64(order.Uint32(data[4:8])) + if ifdOffset > uint64(len(data)-2) { + return 0, errInvalidEXIF + } + count := uint64(order.Uint16(data[ifdOffset : ifdOffset+2])) + entriesStart := ifdOffset + 2 + if count > (uint64(len(data))-entriesStart)/12 { + return 0, errInvalidEXIF + } + for index := uint64(0); index < count; index++ { + if err := contextError(ctx); err != nil { + return 0, err + } + entry := data[entriesStart+index*12 : entriesStart+(index+1)*12] + if order.Uint16(entry[:2]) != 0x0112 { + continue + } + if order.Uint16(entry[2:4]) != 3 || order.Uint32(entry[4:8]) != 1 { + return 0, errInvalidEXIF + } + orientation := int(order.Uint16(entry[8:10])) + if orientation < 1 || orientation > 8 { + return 0, errInvalidEXIF + } + return orientation, nil + } + return 1, nil +} + +func applyOrientation(ctx context.Context, source image.Image, orientation int) (*image.NRGBA, error) { + bounds := source.Bounds() + width, height := bounds.Dx(), bounds.Dy() + outputWidth, outputHeight := width, height + if orientation >= 5 { + outputWidth, outputHeight = height, width + } + output := image.NewNRGBA(image.Rect(0, 0, outputWidth, outputHeight)) + for y := 0; y < outputHeight; y++ { + if err := contextError(ctx); err != nil { + return nil, err + } + for x := 0; x < outputWidth; x++ { + if x&255 == 0 { + if err := contextError(ctx); err != nil { + return nil, err + } + } + sourceX, sourceY := orientedSourcePoint(x, y, width, height, orientation) + output.Set(x, y, source.At(bounds.Min.X+sourceX, bounds.Min.Y+sourceY)) + } + } + return output, nil +} + +func orientedSourcePoint(x, y, width, height, orientation int) (int, int) { + switch orientation { + case 2: + return width - 1 - x, y + case 3: + return width - 1 - x, height - 1 - y + case 4: + return x, height - 1 - y + case 5: + return y, x + case 6: + return y, height - 1 - x + case 7: + return width - 1 - y, height - 1 - x + case 8: + return width - 1 - y, x + default: + return x, y + } +} + +func flattenWhite(ctx context.Context, source image.Image) (*image.NRGBA, error) { + bounds := source.Bounds() + output := image.NewNRGBA(image.Rect(0, 0, bounds.Dx(), bounds.Dy())) + for y := 0; y < bounds.Dy(); y++ { + if err := contextError(ctx); err != nil { + return nil, err + } + for x := 0; x < bounds.Dx(); x++ { + if x&255 == 0 { + if err := contextError(ctx); err != nil { + return nil, err + } + } + r, g, b, a := source.At(bounds.Min.X+x, bounds.Min.Y+y).RGBA() + output.SetNRGBA(x, y, color.NRGBA{ + R: uint8((r + 0xffff - a) / 0x101), + G: uint8((g + 0xffff - a) / 0x101), + B: uint8((b + 0xffff - a) / 0x101), + A: 0xff, + }) + } + } + return output, nil +} diff --git a/internal/messenger/attachment/transform/processor.go b/internal/messenger/attachment/transform/processor.go new file mode 100644 index 0000000..8a6dd08 --- /dev/null +++ b/internal/messenger/attachment/transform/processor.go @@ -0,0 +1,272 @@ +package transform + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "image" + "image/jpeg" + _ "image/png" + "io" + "math" + "net/http" + + "github.com/deepteams/webp" +) + +const ( + // DefaultJPEGQuality is used for thumbnail and crop variants. + DefaultJPEGQuality = 85 + // DefaultWebPQuality keeps the optimized variant visually close to the source while reducing its size. + DefaultWebPQuality = 82 + // DefaultWebPMethod selects the encoder's balanced quality and CPU mode. + DefaultWebPMethod = 4 +) + +var ( + ErrInvalidConfig = errors.New("invalid image transform configuration") + ErrUnsupportedMedia = errors.New("unsupported image media type") + ErrPayloadTooLarge = errors.New("image exceeds configured limits") + ErrInvalidImage = errors.New("invalid image data") +) + +type Config struct { + MaxBytes int64 + MaxPixels int64 + JPEGQuality int + WebPQuality float32 + WebPMethod int +} + +type Variant struct { + Bytes []byte + ContentType string + Width int + Height int + SHA256 string +} + +type Result struct { + Original Variant + Thumbnail Variant + Crop Variant + Optimized Variant + Orientation int +} + +type Processor struct { + config Config +} + +func DefaultConfig(maxBytes, maxPixels int64) Config { + return Config{ + MaxBytes: maxBytes, MaxPixels: maxPixels, + JPEGQuality: DefaultJPEGQuality, WebPQuality: DefaultWebPQuality, WebPMethod: DefaultWebPMethod, + } +} + +func NewProcessor(config Config) (*Processor, error) { + if config.MaxBytes < 1 || config.MaxPixels < 1 || config.JPEGQuality < 1 || config.JPEGQuality > 100 || + config.WebPQuality < 1 || config.WebPQuality > 100 || config.WebPMethod < 0 || config.WebPMethod > 6 { + return nil, ErrInvalidConfig + } + return &Processor{config: config}, nil +} + +func (p *Processor) Process(ctx context.Context, data []byte, declaredContentType string) (Result, error) { + if err := contextError(ctx); err != nil { + return Result{}, err + } + img, format, width, height, orientation, err := p.decode(ctx, data, declaredContentType) + if err != nil { + return Result{}, err + } + if err := contextError(ctx); err != nil { + return Result{}, err + } + + oriented, err := applyOrientation(ctx, img, orientation) + if err != nil { + return Result{}, err + } + opaque, err := flattenWhite(ctx, oriented) + if err != nil { + return Result{}, err + } + result := Result{ + Original: variant(data, format, width, height), + Orientation: orientation, + } + + thumbnail, err := fitContext(ctx, opaque, 256, 256, false) + if err != nil { + return Result{}, err + } + result.Thumbnail, err = encodeJPEG(thumbnail, p.config.JPEGQuality) + if err != nil { + return Result{}, err + } + if err := contextError(ctx); err != nil { + return Result{}, err + } + + crop := centeredSquare(opaque) + crop, err = resizeContext(ctx, crop, 128, 128) + if err != nil { + return Result{}, err + } + result.Crop, err = encodeJPEG(crop, p.config.JPEGQuality) + if err != nil { + return Result{}, err + } + if err := contextError(ctx); err != nil { + return Result{}, err + } + + optimized, err := fitContext(ctx, opaque, 1280, 1280, false) + if err != nil { + return Result{}, err + } + result.Optimized, err = encodeWebP(optimized, p.config.WebPQuality, p.config.WebPMethod) + if err != nil { + return Result{}, err + } + if err := contextError(ctx); err != nil { + return Result{}, err + } + return result, nil +} + +func (p *Processor) decode( + ctx context.Context, + data []byte, + declaredContentType string, +) (image.Image, string, int, int, int, error) { + if declaredContentType != "image/jpeg" && declaredContentType != "image/png" { + return nil, "", 0, 0, 0, ErrUnsupportedMedia + } + if len(data) == 0 { + return nil, "", 0, 0, 0, ErrInvalidImage + } + if int64(len(data)) > p.config.MaxBytes { + return nil, "", 0, 0, 0, ErrPayloadTooLarge + } + if http.DetectContentType(data) != declaredContentType { + return nil, "", 0, 0, 0, ErrUnsupportedMedia + } + config, format, err := image.DecodeConfig(contextReader{ctx: ctx, reader: bytes.NewReader(data)}) + if contextErr := contextError(ctx); contextErr != nil { + return nil, "", 0, 0, 0, contextErr + } + if err != nil || contentType(format) != declaredContentType || config.Width < 1 || config.Height < 1 { + return nil, "", 0, 0, 0, ErrInvalidImage + } + if int64(config.Width) > p.config.MaxPixels/int64(config.Height) { + return nil, "", 0, 0, 0, ErrPayloadTooLarge + } + img, decodedFormat, err := image.Decode(contextReader{ctx: ctx, reader: bytes.NewReader(data)}) + if contextErr := contextError(ctx); contextErr != nil { + return nil, "", 0, 0, 0, contextErr + } + if err != nil || decodedFormat != format { + return nil, "", 0, 0, 0, ErrInvalidImage + } + orientation := 1 + if format == "jpeg" { + orientation, err = jpegOrientation(ctx, data) + if err != nil { + if contextErr := contextError(ctx); contextErr != nil { + return nil, "", 0, 0, 0, contextErr + } + return nil, "", 0, 0, 0, ErrInvalidImage + } + } + return img, declaredContentType, config.Width, config.Height, orientation, nil +} + +type contextReader struct { + ctx context.Context + reader io.Reader +} + +func (r contextReader) Read(buffer []byte) (int, error) { + if err := contextError(r.ctx); err != nil { + return 0, err + } + return r.reader.Read(buffer) +} + +func contextError(ctx context.Context) error { + select { + case <-ctx.Done(): + return ctx.Err() + default: + return nil + } +} + +func contentType(format string) string { + switch format { + case "jpeg": + return "image/jpeg" + case "png": + return "image/png" + default: + return "" + } +} + +func encodeJPEG(img image.Image, quality int) (Variant, error) { + var output bytes.Buffer + if err := jpeg.Encode(&output, img, &jpeg.Options{Quality: quality}); err != nil { + return Variant{}, err + } + bounds := img.Bounds() + return variant(output.Bytes(), "image/jpeg", bounds.Dx(), bounds.Dy()), nil +} + +func encodeWebP(img image.Image, quality float32, method int) (Variant, error) { + var output bytes.Buffer + options := &webp.EncoderOptions{Quality: quality, Method: method} + if err := webp.Encode(&output, img, options); err != nil { + return Variant{}, err + } + bounds := img.Bounds() + return variant(output.Bytes(), "image/webp", bounds.Dx(), bounds.Dy()), nil +} + +func variant(data []byte, contentType string, width, height int) Variant { + digest := sha256.Sum256(data) + return Variant{ + Bytes: data, ContentType: contentType, Width: width, Height: height, + SHA256: hex.EncodeToString(digest[:]), + } +} + +// EstimateMemoryWeight returns a conservative reservation for the input, decoded source, +// orientation and alpha copies, variant working buffer, and encoded variant buffers. +func EstimateMemoryWeight(inputBytes int64, width, height int) (int64, error) { + if inputBytes < 0 || width < 1 || height < 1 || int64(width) > math.MaxInt64/int64(height) { + return 0, ErrInvalidImage + } + pixels := int64(width) * int64(height) + optimizedPixels := fittedPixels(width, height, 1280, 1280) + thumbnailPixels := fittedPixels(width, height, 256, 256) + const cropPixels int64 = 128 * 128 + if pixels > (math.MaxInt64-inputBytes)/16 { + return 0, ErrPayloadTooLarge + } + weight := inputBytes + pixels*16 + variants := (optimizedPixels + thumbnailPixels + cropPixels) * 4 + if variants > math.MaxInt64-weight { + return 0, ErrPayloadTooLarge + } + return weight + variants, nil +} + +func fittedPixels(width, height, maxWidth, maxHeight int) int64 { + newWidth, newHeight := fitDimensions(width, height, maxWidth, maxHeight, false) + return int64(newWidth) * int64(newHeight) +} diff --git a/internal/messenger/attachment/transform/processor_test.go b/internal/messenger/attachment/transform/processor_test.go new file mode 100644 index 0000000..7b595dd --- /dev/null +++ b/internal/messenger/attachment/transform/processor_test.go @@ -0,0 +1,261 @@ +package transform + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/binary" + "encoding/hex" + "errors" + "image" + "image/color" + "image/jpeg" + "image/png" + "testing" + + "github.com/deepteams/webp" +) + +func TestProcessProducesExpectedVariants(t *testing.T) { + source := image.NewNRGBA(image.Rect(0, 0, 320, 200)) + fillGradient(source) + data := encodePNG(t, source) + processor := newTestProcessor(t, int64(len(data))+1, 320*200) + + result, err := processor.Process(context.Background(), data, "image/png") + if err != nil { + t.Fatalf("process image: %v", err) + } + + assertVariant(t, result.Original, "image/png", 320, 200) + assertVariant(t, result.Thumbnail, "image/jpeg", 256, 160) + assertVariant(t, result.Crop, "image/jpeg", 128, 128) + assertVariant(t, result.Optimized, "image/webp", 320, 200) + if !bytes.Equal(result.Original.Bytes, data) { + t.Fatal("original bytes changed") + } + digest := sha256.Sum256(data) + if result.Original.SHA256 != hex.EncodeToString(digest[:]) { + t.Fatalf("unexpected original digest: %s", result.Original.SHA256) + } + if _, err := webp.Decode(bytes.NewReader(result.Optimized.Bytes)); err != nil { + t.Fatalf("decode optimized WebP: %v", err) + } +} + +func TestProcessDoesNotUpscaleFitVariants(t *testing.T) { + source := image.NewNRGBA(image.Rect(0, 0, 64, 32)) + fillGradient(source) + data := encodePNG(t, source) + processor := newTestProcessor(t, int64(len(data))+1, 64*32) + + result, err := processor.Process(context.Background(), data, "image/png") + if err != nil { + t.Fatalf("process image: %v", err) + } + + assertVariant(t, result.Thumbnail, "image/jpeg", 64, 32) + assertVariant(t, result.Optimized, "image/webp", 64, 32) + assertVariant(t, result.Crop, "image/jpeg", 128, 128) +} + +func TestFlattenWhiteCompositesAlpha(t *testing.T) { + source := image.NewNRGBA(image.Rect(0, 0, 2, 1)) + source.SetNRGBA(0, 0, color.NRGBA{R: 10, G: 20, B: 30, A: 0}) + source.SetNRGBA(1, 0, color.NRGBA{R: 100, G: 50, B: 0, A: 128}) + + result, err := flattenWhite(context.Background(), source) + if err != nil { + t.Fatalf("flatten image: %v", err) + } + if got := result.NRGBAAt(0, 0); got != (color.NRGBA{R: 255, G: 255, B: 255, A: 255}) { + t.Fatalf("transparent pixel: got %#v", got) + } + if got := result.NRGBAAt(1, 0); got != (color.NRGBA{R: 177, G: 152, B: 127, A: 255}) { + t.Fatalf("semi-transparent pixel: got %#v", got) + } +} + +func TestProcessAppliesEXIFOrientation(t *testing.T) { + source := image.NewNRGBA(image.Rect(0, 0, 3, 2)) + fillGradient(source) + data := injectOrientation(t, encodeJPEGForTest(t, source), 6) + processor := newTestProcessor(t, int64(len(data))+1, 3*2) + + result, err := processor.Process(context.Background(), data, "image/jpeg") + if err != nil { + t.Fatalf("process image: %v", err) + } + + if result.Orientation != 6 { + t.Fatalf("orientation: got %d, want 6", result.Orientation) + } + assertVariant(t, result.Original, "image/jpeg", 3, 2) + assertVariant(t, result.Thumbnail, "image/jpeg", 2, 3) + assertVariant(t, result.Optimized, "image/webp", 2, 3) +} + +func TestProcessRejectsInvalidInputs(t *testing.T) { + valid := encodePNG(t, image.NewNRGBA(image.Rect(0, 0, 11, 10))) + + tests := []struct { + name string + data []byte + contentType string + maxPixels int64 + want error + }{ + {name: "MIME mismatch", data: valid, contentType: "image/jpeg", maxPixels: 200, want: ErrUnsupportedMedia}, + {name: "unsupported MIME", data: valid, contentType: "image/gif", maxPixels: 200, want: ErrUnsupportedMedia}, + {name: "pixel limit", data: valid, contentType: "image/png", maxPixels: 100, want: ErrPayloadTooLarge}, + {name: "corrupted", data: valid[:len(valid)-12], contentType: "image/png", maxPixels: 200, want: ErrInvalidImage}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + processor := newTestProcessor(t, int64(len(valid))+1, test.maxPixels) + _, err := processor.Process(context.Background(), test.data, test.contentType) + if !errors.Is(err, test.want) { + t.Fatalf("error: got %v, want %v", err, test.want) + } + }) + } +} + +func TestProcessIsDeterministic(t *testing.T) { + source := image.NewNRGBA(image.Rect(0, 0, 48, 31)) + fillGradient(source) + data := encodePNG(t, source) + processor := newTestProcessor(t, int64(len(data))+1, 48*31) + + first, err := processor.Process(context.Background(), data, "image/png") + if err != nil { + t.Fatalf("first process: %v", err) + } + second, err := processor.Process(context.Background(), data, "image/png") + if err != nil { + t.Fatalf("second process: %v", err) + } + + for name, variants := range map[string][2]Variant{ + "thumbnail": {first.Thumbnail, second.Thumbnail}, + "crop": {first.Crop, second.Crop}, + "optimized": {first.Optimized, second.Optimized}, + } { + if !bytes.Equal(variants[0].Bytes, variants[1].Bytes) { + t.Fatalf("%s bytes differ", name) + } + } +} + +func TestProcessHonorsCanceledContext(t *testing.T) { + processor := newTestProcessor(t, 1024, 100) + ctx, cancel := context.WithCancel(context.Background()) + cancel() + _, err := processor.Process(ctx, []byte("unused"), "image/png") + if !errors.Is(err, context.Canceled) { + t.Fatalf("error: got %v, want %v", err, context.Canceled) + } +} + +func TestResizeStopsAfterCancellation(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + source := cancelingImage{ + Image: image.NewNRGBA(image.Rect(0, 0, 256, 256)), + cancel: cancel, + } + _, err := resizeContext(ctx, source, 128, 128) + if !errors.Is(err, context.Canceled) { + t.Fatalf("error: got %v, want %v", err, context.Canceled) + } +} + +type cancelingImage struct { + image.Image + cancel context.CancelFunc +} + +func (i cancelingImage) At(x, y int) color.Color { + i.cancel() + return i.Image.At(x, y) +} + +func TestEstimateMemoryWeight(t *testing.T) { + weight, err := EstimateMemoryWeight(1000, 320, 200) + if err != nil { + t.Fatalf("estimate memory: %v", err) + } + want := int64(1000 + 320*200*16 + (320*200+256*160+128*128)*4) + if weight != want { + t.Fatalf("weight: got %d, want %d", weight, want) + } +} + +func newTestProcessor(t *testing.T, maxBytes, maxPixels int64) *Processor { + t.Helper() + processor, err := NewProcessor(DefaultConfig(maxBytes, maxPixels)) + if err != nil { + t.Fatalf("new processor: %v", err) + } + return processor +} + +func assertVariant(t *testing.T, variant Variant, contentType string, width, height int) { + t.Helper() + if variant.ContentType != contentType || variant.Width != width || variant.Height != height { + t.Fatalf("variant: got type=%s dimensions=%dx%d, want type=%s dimensions=%dx%d", + variant.ContentType, variant.Width, variant.Height, contentType, width, height) + } + if len(variant.Bytes) == 0 || variant.SHA256 == "" { + t.Fatal("variant has no bytes or digest") + } +} + +func fillGradient(img *image.NRGBA) { + for y := 0; y < img.Bounds().Dy(); y++ { + for x := 0; x < img.Bounds().Dx(); x++ { + img.SetNRGBA(x, y, color.NRGBA{R: uint8(x), G: uint8(y), B: uint8(x + y), A: 255}) + } + } +} + +func encodePNG(t *testing.T, img image.Image) []byte { + t.Helper() + var output bytes.Buffer + if err := png.Encode(&output, img); err != nil { + t.Fatalf("encode PNG: %v", err) + } + return output.Bytes() +} + +func encodeJPEGForTest(t *testing.T, img image.Image) []byte { + t.Helper() + var output bytes.Buffer + if err := jpeg.Encode(&output, img, &jpeg.Options{Quality: 100}); err != nil { + t.Fatalf("encode JPEG: %v", err) + } + return output.Bytes() +} + +func injectOrientation(t *testing.T, data []byte, orientation uint16) []byte { + t.Helper() + if len(data) < 2 || data[0] != 0xff || data[1] != 0xd8 { + t.Fatal("not a JPEG") + } + tiff := make([]byte, 26) + copy(tiff[:2], "II") + binary.LittleEndian.PutUint16(tiff[2:4], 42) + binary.LittleEndian.PutUint32(tiff[4:8], 8) + binary.LittleEndian.PutUint16(tiff[8:10], 1) + binary.LittleEndian.PutUint16(tiff[10:12], 0x0112) + binary.LittleEndian.PutUint16(tiff[12:14], 3) + binary.LittleEndian.PutUint32(tiff[14:18], 1) + binary.LittleEndian.PutUint16(tiff[18:20], orientation) + payload := append([]byte("Exif\x00\x00"), tiff...) + segment := []byte{0xff, 0xe1, 0, 0} + binary.BigEndian.PutUint16(segment[2:4], uint16(len(payload)+2)) + result := make([]byte, 0, len(data)+len(segment)+len(payload)) + result = append(result, data[:2]...) + result = append(result, segment...) + result = append(result, payload...) + return append(result, data[2:]...) +} diff --git a/internal/messenger/attachment/transform/resize.go b/internal/messenger/attachment/transform/resize.go new file mode 100644 index 0000000..2d86bb8 --- /dev/null +++ b/internal/messenger/attachment/transform/resize.go @@ -0,0 +1,128 @@ +package transform + +import ( + "context" + "image" + "image/color" +) + +func fitContext( + ctx context.Context, + source image.Image, + maxWidth int, + maxHeight int, + upscale bool, +) (*image.NRGBA, error) { + bounds := source.Bounds() + width, height := fitDimensions(bounds.Dx(), bounds.Dy(), maxWidth, maxHeight, upscale) + return resizeContext(ctx, source, width, height) +} + +func fitDimensions(width, height, maxWidth, maxHeight int, upscale bool) (int, int) { + if !upscale && width <= maxWidth && height <= maxHeight { + return width, height + } + if int64(width)*int64(maxHeight) > int64(height)*int64(maxWidth) { + newHeight := max(1, int(int64(height)*int64(maxWidth)/int64(width))) + return maxWidth, newHeight + } + newWidth := max(1, int(int64(width)*int64(maxHeight)/int64(height))) + return newWidth, maxHeight +} + +func centeredSquare(source image.Image) image.Image { + bounds := source.Bounds() + size := min(bounds.Dx(), bounds.Dy()) + left := bounds.Min.X + (bounds.Dx()-size)/2 + top := bounds.Min.Y + (bounds.Dy()-size)/2 + return croppedImage{source: source, bounds: image.Rect(0, 0, size, size), offset: image.Pt(left, top)} +} + +type croppedImage struct { + source image.Image + bounds image.Rectangle + offset image.Point +} + +func (i croppedImage) ColorModel() color.Model { return i.source.ColorModel() } +func (i croppedImage) Bounds() image.Rectangle { return i.bounds } +func (i croppedImage) At(x, y int) color.Color { + return i.source.At(i.offset.X+x, i.offset.Y+y) +} + +func resizeContext(ctx context.Context, source image.Image, width, height int) (*image.NRGBA, error) { + bounds := source.Bounds() + if bounds.Dx() == width && bounds.Dy() == height { + return copyNRGBAContext(ctx, source) + } + output := image.NewNRGBA(image.Rect(0, 0, width, height)) + for y := 0; y < height; y++ { + if err := contextError(ctx); err != nil { + return nil, err + } + y0, y1, fy := sampleCoordinate(y, height, bounds.Dy()) + for x := 0; x < width; x++ { + if x&255 == 0 { + if err := contextError(ctx); err != nil { + return nil, err + } + } + x0, x1, fx := sampleCoordinate(x, width, bounds.Dx()) + c00 := color.NRGBAModel.Convert(source.At(bounds.Min.X+x0, bounds.Min.Y+y0)).(color.NRGBA) + c10 := color.NRGBAModel.Convert(source.At(bounds.Min.X+x1, bounds.Min.Y+y0)).(color.NRGBA) + c01 := color.NRGBAModel.Convert(source.At(bounds.Min.X+x0, bounds.Min.Y+y1)).(color.NRGBA) + c11 := color.NRGBAModel.Convert(source.At(bounds.Min.X+x1, bounds.Min.Y+y1)).(color.NRGBA) + output.SetNRGBA(x, y, interpolate(c00, c10, c01, c11, fx, fy)) + } + } + return output, nil +} + +func sampleCoordinate(destination, destinationSize, sourceSize int) (int, int, uint32) { + coordinate := ((int64(2*destination+1)*int64(sourceSize))<<15)/int64(destinationSize) - 1<<15 + if coordinate <= 0 { + return 0, min(1, sourceSize-1), 0 + } + maximum := int64(sourceSize-1) << 16 + if coordinate >= maximum { + return sourceSize - 1, sourceSize - 1, 0 + } + first := int(coordinate >> 16) + return first, first + 1, uint32(coordinate & 0xffff) +} + +func interpolate(c00, c10, c01, c11 color.NRGBA, fx, fy uint32) color.NRGBA { + return color.NRGBA{ + R: interpolateChannel(c00.R, c10.R, c01.R, c11.R, fx, fy), + G: interpolateChannel(c00.G, c10.G, c01.G, c11.G, fx, fy), + B: interpolateChannel(c00.B, c10.B, c01.B, c11.B, fx, fy), + A: interpolateChannel(c00.A, c10.A, c01.A, c11.A, fx, fy), + } +} + +func interpolateChannel(c00, c10, c01, c11 uint8, fx, fy uint32) uint8 { + const one uint64 = 1 << 16 + top := uint64(c00)*(one-uint64(fx)) + uint64(c10)*uint64(fx) + bottom := uint64(c01)*(one-uint64(fx)) + uint64(c11)*uint64(fx) + value := top*(one-uint64(fy)) + bottom*uint64(fy) + return uint8((value + 1<<31) >> 32) +} + +func copyNRGBAContext(ctx context.Context, source image.Image) (*image.NRGBA, error) { + bounds := source.Bounds() + output := image.NewNRGBA(image.Rect(0, 0, bounds.Dx(), bounds.Dy())) + for y := 0; y < bounds.Dy(); y++ { + if err := contextError(ctx); err != nil { + return nil, err + } + for x := 0; x < bounds.Dx(); x++ { + if x&255 == 0 { + if err := contextError(ctx); err != nil { + return nil, err + } + } + output.Set(x, y, source.At(bounds.Min.X+x, bounds.Min.Y+y)) + } + } + return output, nil +} diff --git a/internal/messenger/attachment/worker/worker.go b/internal/messenger/attachment/worker/worker.go new file mode 100644 index 0000000..8b88c84 --- /dev/null +++ b/internal/messenger/attachment/worker/worker.go @@ -0,0 +1,540 @@ +package worker + +import ( + "bytes" + "context" + "errors" + "fmt" + "image" + _ "image/jpeg" + _ "image/png" + "io" + "log/slog" + "math" + "path" + "strings" + "sync" + "time" + + "golang.org/x/sync/semaphore" + + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/codes" + "go.opentelemetry.io/otel/trace" + + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/broker/redisstream" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/messenger/attachment/transform" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/objectstore" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/telemetry" +) + +type Repository interface { + AttachmentJob(context.Context, string) (model.AttachmentJob, error) + ClaimAttachmentJob(context.Context, string, string, int, int, string, time.Duration) (model.AttachmentJobLease, error) + RenewAttachmentJobLease(context.Context, model.AttachmentJobLease, time.Duration) (bool, error) + CompleteAttachmentJob(context.Context, model.AttachmentJobLease, []model.Variant) (model.JobMutationResult, error) + RetryAttachmentJob(context.Context, model.AttachmentJobLease, string, time.Time, string, string) (model.JobMutationResult, error) + FailAttachmentJob(context.Context, model.AttachmentJobLease, string) (model.JobMutationResult, error) + RecoverExpiredAttachmentJobs(context.Context, int, int, time.Time) (model.RecoveryResult, error) +} + +type Broker interface { + Read(context.Context) ([]redisstream.Delivery, error) + Reclaim(context.Context, string) ([]redisstream.Delivery, string, error) +} + +type ObjectStore interface { + Get(context.Context, string) (io.ReadCloser, error) + Put(context.Context, string, string, []byte) error + Delete(context.Context, string) error +} + +type ImageProcessor interface { + Process(context.Context, []byte, string) (transform.Result, error) +} + +type Metrics interface { + ObserveStarted(context.Context) + ObserveFinished(context.Context, string, time.Duration) + ObserveRetry(context.Context) + ObserveReclaimed(context.Context, int) +} + +type Config struct { + WorkerID string + Concurrency int + LeaseDuration time.Duration + HeartbeatInterval time.Duration + ProcessingTimeout time.Duration + ReclaimInterval time.Duration + MaxAttempts int + RetryBase time.Duration + RetryMax time.Duration + MaxBytes int64 + MaxPixels int64 + MemoryBytes int64 +} + +type FaultPoint string + +const ( + FaultNone FaultPoint = "" + FaultBeforeSideEffect FaultPoint = "before_side_effect" + FaultAfterSideEffect FaultPoint = "after_side_effect" + FaultAfterCommit FaultPoint = "after_commit" +) + +var ErrInjectedFailure = errors.New("injected worker failure") + +type Worker struct { + logger *slog.Logger + repository Repository + broker Broker + objects ObjectStore + processor ImageProcessor + metrics Metrics + config Config + memory *semaphore.Weighted + now func() time.Time +} + +func New( + logger *slog.Logger, + repository Repository, + broker Broker, + objects ObjectStore, + processor ImageProcessor, + metrics Metrics, + config Config, +) (*Worker, error) { + if logger == nil || repository == nil || broker == nil || objects == nil || processor == nil { + return nil, errors.New("worker dependencies are required") + } + if err := validateConfig(config); err != nil { + return nil, err + } + if metrics == nil { + metrics = nopMetrics{} + } + return &Worker{ + logger: logger, repository: repository, broker: broker, objects: objects, + processor: processor, metrics: metrics, config: config, + memory: semaphore.NewWeighted(config.MemoryBytes), now: time.Now, + }, nil +} + +func validateConfig(config Config) error { + if config.WorkerID == "" || config.Concurrency < 1 || config.LeaseDuration <= 0 || + config.HeartbeatInterval <= 0 || config.HeartbeatInterval >= config.LeaseDuration || + config.ProcessingTimeout <= 0 || config.ProcessingTimeout >= config.LeaseDuration || + config.ReclaimInterval <= 0 || config.MaxAttempts < 1 || config.RetryBase <= 0 || + config.RetryMax < config.RetryBase || config.MaxBytes < 1 || config.MaxPixels < 1 || config.MemoryBytes < 1 { + return errors.New("invalid worker configuration") + } + const maximumVariantPixels int64 = 1280*1280 + 256*256 + 128*128 + if config.MaxPixels > (math.MaxInt64-config.MaxBytes-maximumVariantPixels*4)/16 || + config.MaxBytes+config.MaxPixels*16+maximumVariantPixels*4 > config.MemoryBytes { + return errors.New("worker memory must fit one maximum image") + } + return nil +} + +func (w *Worker) Run(ctx context.Context) error { + lastReclaim := time.Time{} + reclaimCursor := "0-0" + for ctx.Err() == nil { + deliveries, err := w.broker.Read(ctx) + if err != nil { + if ctx.Err() != nil { + return nil + } + return err + } + w.processBatch(ctx, deliveries) + if w.now().Sub(lastReclaim) < w.config.ReclaimInterval { + continue + } + recovery, err := w.repository.RecoverExpiredAttachmentJobs( + ctx, w.config.Concurrency*4, w.config.MaxAttempts, w.now().Add(w.config.RetryBase), + ) + if err != nil { + return fmt.Errorf("recover expired attachment jobs: %w", err) + } + if recovery.Requeued > 0 || recovery.Failed > 0 { + w.logger.Info("expired attachment jobs recovered", "requeued", recovery.Requeued, "failed", recovery.Failed) + } + deliveries, next, err := w.broker.Reclaim(ctx, reclaimCursor) + if err != nil { + if ctx.Err() != nil { + return nil + } + return err + } + w.metrics.ObserveReclaimed(ctx, len(deliveries)) + w.processBatch(ctx, deliveries) + reclaimCursor = next + if reclaimCursor == "" { + reclaimCursor = "0-0" + } + lastReclaim = w.now() + } + return nil +} + +func (w *Worker) processBatch(ctx context.Context, deliveries []redisstream.Delivery) { + limit := make(chan struct{}, w.config.Concurrency) + var group sync.WaitGroup + for _, delivery := range deliveries { + if ctx.Err() != nil { + break + } + limit <- struct{}{} + group.Add(1) + go func(delivery redisstream.Delivery) { + defer group.Done() + defer func() { <-limit }() + _ = w.ProcessDelivery(ctx, delivery, FaultNone) + }(delivery) + } + group.Wait() +} + +func (w *Worker) ProcessDelivery(ctx context.Context, delivery redisstream.Delivery, fault FaultPoint) (resultErr error) { + started := w.now() + result := "transient_error" + if delivery.DecodeError != nil { + w.metrics.ObserveStarted(ctx) + defer func() { w.metrics.ObserveFinished(ctx, result, w.now().Sub(started)) }() + result = "invalid_job" + return delivery.ToFailed(ctx, redisstream.FailedDelivery{Code: "invalid_job", FailedAt: w.now()}) + } + envelope := delivery.Envelope + ctx = telemetry.ExtractMessage(ctx, telemetry.MessageCarrier{ + Traceparent: envelope.Traceparent, Tracestate: envelope.Tracestate, + }) + ctx, span := otel.Tracer("messenger-worker").Start(ctx, "attachment.process", trace.WithSpanKind(trace.SpanKindConsumer), + trace.WithAttributes( + attribute.String("messaging.system", "redis"), attribute.String("messaging.destination.name", "attachment-jobs"), + attribute.String("messaging.message.id", delivery.ID), attribute.String("job.id", envelope.JobID.String()), + attribute.String("attachment.id", envelope.AttachmentID.String()), attribute.Int("job.attempt", envelope.Attempt), + attribute.Int("pipeline.version", envelope.PipelineVersion), + )) + w.metrics.ObserveStarted(ctx) + defer func() { + w.metrics.ObserveFinished(ctx, result, w.now().Sub(started)) + span.SetAttributes(attribute.String("job.outcome", result)) + if resultErr != nil { + span.RecordError(resultErr) + span.SetStatus(codes.Error, result) + if ctx.Err() == nil { + w.logger.ErrorContext(ctx, "attachment job failed", "event", "attachment_job_failed", "message_id", delivery.ID, "job_id", envelope.JobID.String(), "job_type", envelope.Type, "attempt", envelope.Attempt, "result", result, "error", resultErr) + } + } else { + w.logger.InfoContext(ctx, "attachment job completed", "event", "attachment_job_completed", "message_id", delivery.ID, "job_id", envelope.JobID.String(), "job_type", envelope.Type, "attempt", envelope.Attempt, "result", result) + } + span.End() + }() + + lease, err := w.repository.ClaimAttachmentJob( + ctx, envelope.JobID.String(), envelope.AttachmentID.String(), envelope.PipelineVersion, envelope.Attempt, + w.config.WorkerID, w.config.LeaseDuration, + ) + if err != nil { + result, err = w.resolveClaim(ctx, delivery, envelope, err) + return err + } + if fault == FaultBeforeSideEffect { + result = "injected_failure" + return ErrInjectedFailure + } + + processingCtx, cancelProcessing := context.WithTimeout(ctx, w.config.ProcessingTimeout) + heartbeatCtx, cancelHeartbeat := context.WithCancel(ctx) + heartbeatDone := make(chan error, 1) + go func() { heartbeatDone <- w.maintainLease(heartbeatCtx, cancelProcessing, lease) }() + variants, err := w.process(processingCtx, lease) + cancelProcessing() + cancelHeartbeat() + heartbeatErr := <-heartbeatDone + if heartbeatErr != nil { + return heartbeatErr + } + if err != nil { + result, err = w.resolveProcessingFailure(ctx, delivery, lease, err) + return err + } + if fault == FaultAfterSideEffect { + result = "injected_failure" + return ErrInjectedFailure + } + mutation, err := w.repository.CompleteAttachmentJob(ctx, lease, variants) + if err != nil { + return fmt.Errorf("complete attachment job: %w", err) + } + if mutation == model.JobMutationApplied && strings.HasPrefix(lease.Attachment.StagingKey, "staging/") { + if err := w.objects.Delete(context.WithoutCancel(ctx), lease.Attachment.StagingKey); err != nil { + w.logger.WarnContext(ctx, "staging cleanup failed", "attachment_id", lease.Attachment.ID, "error", err) + } + } + if mutation != model.JobMutationApplied && mutation != model.JobMutationAlreadyComplete && mutation != model.JobMutationStale { + return fmt.Errorf("unexpected completion result %q", mutation) + } + if fault == FaultAfterCommit { + result = "injected_failure" + return ErrInjectedFailure + } + if err := delivery.Ack(ctx); err != nil { + return err + } + result = "ready" + return nil +} + +func (w *Worker) maintainLease( + ctx context.Context, + cancelProcessing context.CancelFunc, + lease model.AttachmentJobLease, +) error { + ticker := time.NewTicker(w.config.HeartbeatInterval) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return nil + case <-ticker.C: + renewed, err := w.repository.RenewAttachmentJobLease(ctx, lease, w.config.LeaseDuration) + if err != nil { + if ctx.Err() != nil { + return nil + } + cancelProcessing() + return fmt.Errorf("renew attachment job lease: %w", err) + } + if !renewed { + cancelProcessing() + return model.ErrStaleLease + } + } + } +} + +func (w *Worker) resolveClaim( + ctx context.Context, + delivery redisstream.Delivery, + envelope redisstream.Envelope, + claimErr error, +) (string, error) { + switch { + case errors.Is(claimErr, model.ErrNotFound), errors.Is(claimErr, model.ErrInvalidInput): + return "invalid_job", delivery.ToFailed(ctx, redisstream.FailedDelivery{Code: "invalid_job", FailedAt: w.now()}) + case errors.Is(claimErr, model.ErrJobComplete), errors.Is(claimErr, model.ErrStaleAttempt), errors.Is(claimErr, model.ErrStalePipeline): + return "duplicate", delivery.Ack(ctx) + case errors.Is(claimErr, model.ErrJobBusy): + return "duplicate", delivery.Ack(ctx) + case errors.Is(claimErr, model.ErrJobNotDue): + return "not_due", nil + case errors.Is(claimErr, model.ErrConflict): + job, err := w.repository.AttachmentJob(ctx, envelope.JobID.String()) + if err != nil { + return "transient_error", fmt.Errorf("inspect conflicting job: %w", err) + } + switch job.Status { + case model.AttachmentJobReady: + return "duplicate", delivery.Ack(ctx) + case model.AttachmentJobFailed: + code := "processing_failed" + if job.ErrorCode != nil { + code = *job.ErrorCode + } + return "duplicate_failed", delivery.ToFailed(ctx, redisstream.FailedDelivery{Code: code, FailedAt: w.now()}) + default: + return "transient_error", fmt.Errorf("claim attachment job: %w", claimErr) + } + default: + return "transient_error", fmt.Errorf("claim attachment job: %w", claimErr) + } +} + +func (w *Worker) process(ctx context.Context, lease model.AttachmentJobLease) ([]model.Variant, error) { + body, err := w.objects.Get(ctx, lease.Attachment.StagingKey) + if err != nil { + return nil, err + } + defer func() { _ = body.Close() }() + data, err := io.ReadAll(io.LimitReader(contextReader{ctx: ctx, reader: body}, w.config.MaxBytes+1)) + if err != nil { + return nil, err + } + if int64(len(data)) > w.config.MaxBytes { + return nil, transform.ErrPayloadTooLarge + } + imageConfig, _, err := image.DecodeConfig(contextReader{ctx: ctx, reader: bytes.NewReader(data)}) + if err != nil { + if ctx.Err() != nil { + return nil, ctx.Err() + } + return nil, transform.ErrInvalidImage + } + weight, err := transform.EstimateMemoryWeight(int64(len(data)), imageConfig.Width, imageConfig.Height) + if err != nil { + return nil, err + } + if weight > w.config.MemoryBytes { + return nil, errWorkerCapacity + } + if err := w.memory.Acquire(ctx, weight); err != nil { + return nil, err + } + defer w.memory.Release(weight) + + processed, err := w.processor.Process(ctx, data, lease.Attachment.ContentType) + if err != nil { + return nil, err + } + variants := []struct { + name string + value transform.Variant + }{ + {name: "original", value: processed.Original}, + {name: "thumbnail", value: processed.Thumbnail}, + {name: "crop", value: processed.Crop}, + {name: "optimized", value: processed.Optimized}, + } + result := make([]model.Variant, 0, len(variants)) + for _, item := range variants { + objectKey := variantObjectKey(lease.Attachment.ID, lease.Job.PipelineVersion, item.name, item.value) + if err := w.objects.Put(ctx, objectKey, item.value.ContentType, item.value.Bytes); err != nil { + return nil, err + } + result = append(result, model.Variant{ + Name: item.name, ContentType: item.value.ContentType, SizeBytes: int64(len(item.value.Bytes)), + SHA256: item.value.SHA256, Width: item.value.Width, Height: item.value.Height, + ObjectKey: objectKey, PipelineVersion: lease.Job.PipelineVersion, + }) + } + return result, nil +} + +type contextReader struct { + ctx context.Context + reader io.Reader +} + +func (r contextReader) Read(buffer []byte) (int, error) { + select { + case <-r.ctx.Done(): + return 0, r.ctx.Err() + default: + return r.reader.Read(buffer) + } +} + +var errWorkerCapacity = errors.New("image exceeds worker memory budget") + +func variantObjectKey(attachmentID string, pipelineVersion int, name string, value transform.Variant) string { + extension := ".bin" + switch value.ContentType { + case "image/png": + extension = ".png" + case "image/jpeg": + extension = ".jpg" + case "image/webp": + extension = ".webp" + } + return path.Join( + "attachments", attachmentID, "pipelines", fmt.Sprintf("%d", pipelineVersion), + name, value.SHA256+extension, + ) +} + +func (w *Worker) resolveProcessingFailure( + ctx context.Context, + delivery redisstream.Delivery, + lease model.AttachmentJobLease, + processingErr error, +) (string, error) { + code, permanent := processingError(processingErr) + if permanent || lease.Job.Attempt >= w.config.MaxAttempts { + if !permanent { + code = "retry_exhausted" + } + mutation, err := w.repository.FailAttachmentJob(ctx, lease, code) + if err != nil { + return "transient_error", fmt.Errorf("fail attachment job: %w", err) + } + if mutation == model.JobMutationStale || mutation == model.JobMutationAlreadyComplete { + return "duplicate", delivery.Ack(ctx) + } + if mutation != model.JobMutationApplied { + return "transient_error", fmt.Errorf("unexpected failure result %q", mutation) + } + return "failed", delivery.ToFailed(ctx, redisstream.FailedDelivery{Code: code, FailedAt: w.now()}) + } + nextAttempt := w.now().Add(w.retryDelay(lease.Job.ID, lease.Job.Attempt)) + traceContext := telemetry.ContextCarrier(ctx) + mutation, err := w.repository.RetryAttachmentJob( + ctx, lease, code, nextAttempt, traceContext.Traceparent, traceContext.Tracestate, + ) + if err != nil { + return "transient_error", fmt.Errorf("retry attachment job: %w", err) + } + if mutation == model.JobMutationStale || mutation == model.JobMutationAlreadyComplete { + return "duplicate", delivery.Ack(ctx) + } + if mutation != model.JobMutationApplied { + return "transient_error", fmt.Errorf("unexpected retry result %q", mutation) + } + w.metrics.ObserveRetry(ctx) + return "retry", delivery.Ack(ctx) +} + +func processingError(err error) (string, bool) { + switch { + case errors.Is(err, transform.ErrInvalidImage), errors.Is(err, transform.ErrUnsupportedMedia): + return "invalid_image", true + case errors.Is(err, transform.ErrPayloadTooLarge): + return "image_too_large", true + case errors.Is(err, objectstore.ErrNotFound): + return "source_missing", true + case errors.Is(err, errWorkerCapacity): + return "worker_capacity", true + default: + return "processing_failed", false + } +} + +func (w *Worker) retryDelay(jobID string, attempt int) time.Duration { + delay := w.config.RetryBase + for step := 1; step < attempt && delay < w.config.RetryMax; step++ { + if delay > w.config.RetryMax/2 { + delay = w.config.RetryMax + break + } + delay *= 2 + } + if delay > w.config.RetryMax { + delay = w.config.RetryMax + } + var hash uint64 + for index := range jobID { + hash = hash*131 + uint64(jobID[index]) + } + jitterRange := min(delay/4, w.config.RetryMax-delay) + if jitterRange <= 0 { + return delay + } + jitter := time.Duration(hash % uint64(jitterRange)) + if delay > time.Duration(math.MaxInt64)-jitter { + return delay + } + return delay + jitter +} + +type nopMetrics struct{} + +func (nopMetrics) ObserveStarted(context.Context) {} +func (nopMetrics) ObserveFinished(context.Context, string, time.Duration) {} +func (nopMetrics) ObserveRetry(context.Context) {} +func (nopMetrics) ObserveReclaimed(context.Context, int) {} diff --git a/internal/messenger/attachment/worker/worker_test.go b/internal/messenger/attachment/worker/worker_test.go new file mode 100644 index 0000000..c2ca384 --- /dev/null +++ b/internal/messenger/attachment/worker/worker_test.go @@ -0,0 +1,109 @@ +package worker + +import ( + "context" + "errors" + "sync/atomic" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/messenger/attachment/transform" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/objectstore" +) + +func TestConfigRequiresEnoughMemoryForMaximumImage(t *testing.T) { + config := validConfig() + require.NoError(t, validateConfig(config)) + + config.MemoryBytes = 384 * 1024 * 1024 + require.ErrorContains(t, validateConfig(config), "memory must fit") +} + +func TestRetryDelayIsBounded(t *testing.T) { + worker := &Worker{config: Config{RetryBase: time.Second, RetryMax: 4 * time.Second}} + for attempt := 1; attempt <= 10; attempt++ { + delay := worker.retryDelay("11111111-1111-4111-8111-111111111111", attempt) + require.GreaterOrEqual(t, delay, time.Second) + require.LessOrEqual(t, delay, 4*time.Second) + } +} + +func TestProcessingErrorClassification(t *testing.T) { + tests := []struct { + err error + code string + permanent bool + }{ + {err: transform.ErrInvalidImage, code: "invalid_image", permanent: true}, + {err: transform.ErrPayloadTooLarge, code: "image_too_large", permanent: true}, + {err: objectstore.ErrNotFound, code: "source_missing", permanent: true}, + {err: errors.New("temporary S3 failure"), code: "processing_failed", permanent: false}, + } + for _, test := range tests { + code, permanent := processingError(test.err) + require.Equal(t, test.code, code) + require.Equal(t, test.permanent, permanent) + } +} + +func TestLeaseHeartbeatRenewsUntilCanceled(t *testing.T) { + var renewals atomic.Int32 + repository := &repositoryStub{renew: func(context.Context, model.AttachmentJobLease, time.Duration) (bool, error) { + renewals.Add(1) + return true, nil + }} + config := validConfig() + config.HeartbeatInterval = 5 * time.Millisecond + worker := &Worker{repository: repository, config: config} + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan error, 1) + go func() { done <- worker.maintainLease(ctx, cancel, testLease()) }() + require.Eventually(t, func() bool { return renewals.Load() >= 2 }, time.Second, time.Millisecond) + cancel() + require.NoError(t, <-done) +} + +func TestLeaseHeartbeatCancelsProcessingWhenLeaseIsLost(t *testing.T) { + repository := &repositoryStub{renew: func(context.Context, model.AttachmentJobLease, time.Duration) (bool, error) { + return false, nil + }} + config := validConfig() + config.HeartbeatInterval = time.Millisecond + worker := &Worker{repository: repository, config: config} + ctx, cancel := context.WithCancel(context.Background()) + err := worker.maintainLease(ctx, cancel, testLease()) + require.ErrorIs(t, err, model.ErrStaleLease) + require.ErrorIs(t, ctx.Err(), context.Canceled) +} + +type repositoryStub struct { + Repository + renew func(context.Context, model.AttachmentJobLease, time.Duration) (bool, error) +} + +func (s *repositoryStub) RenewAttachmentJobLease( + ctx context.Context, + lease model.AttachmentJobLease, + duration time.Duration, +) (bool, error) { + return s.renew(ctx, lease, duration) +} + +func testLease() model.AttachmentJobLease { + return model.AttachmentJobLease{Job: model.AttachmentJob{ + ID: "11111111-1111-4111-8111-111111111111", LeaseToken: "lease-token", LeaseOwner: "worker-1", + }} +} + +func validConfig() Config { + return Config{ + WorkerID: "worker-1", Concurrency: 2, LeaseDuration: 30 * time.Second, + HeartbeatInterval: 5 * time.Second, ProcessingTimeout: 20 * time.Second, + ReclaimInterval: 5 * time.Second, + MaxAttempts: 5, RetryBase: time.Second, RetryMax: 30 * time.Second, + MaxBytes: 10 * 1024 * 1024, MaxPixels: 40_000_000, MemoryBytes: 704 * 1024 * 1024, + } +} diff --git a/internal/messenger/httpapi/attachments.go b/internal/messenger/httpapi/attachments.go new file mode 100644 index 0000000..ad8d1df --- /dev/null +++ b/internal/messenger/httpapi/attachments.go @@ -0,0 +1,186 @@ +package httpapi + +import ( + "errors" + "io" + "mime" + "net/http" + "net/url" + "strconv" + "strings" + "time" + "unicode" + "unicode/utf8" + + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/codes" + "go.opentelemetry.io/otel/metric" + "go.opentelemetry.io/otel/trace" + + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/messenger/attachment" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/telemetry" +) + +func (s *Server) uploadAttachment(w http.ResponseWriter, r *http.Request) *apiError { + started := time.Now() + result := "client_error" + defer func() { + attributes := metric.WithAttributes(attribute.String("result", result)) + s.metrics.attachmentUploads.Add(r.Context(), 1, attributes) + s.metrics.attachmentDuration.Record(r.Context(), time.Since(started).Seconds(), attributes) + }() + + if r.Method != http.MethodPost { + return methodNotAllowed(w, http.MethodPost) + } + user, _, authErr := s.authenticate(r) + if authErr != nil { + return authErr + } + chatID, valid := canonicalID(strings.Split(r.URL.Path, "/")[4]) + if !valid { + return invalidRequest() + } + query, err := url.ParseQuery(r.URL.RawQuery) + if err != nil { + return invalidRequest() + } + filename, valid := attachmentFilename(query) + if !valid { + return invalidRequest() + } + uploaded, err := s.attachments.Upload(r.Context(), attachment.Upload{ + ChatID: chatID, OwnerID: user.ID, Filename: filename, + ContentType: r.Header.Get("Content-Type"), ContentLength: r.ContentLength, Body: r.Body, + TraceContext: telemetry.ContextCarrier(r.Context()), + }) + if err != nil { + if errors.Is(err, model.ErrCapacity) { + result = "capacity_rejected" + w.Header().Set("Retry-After", "1") + } else if errors.Is(err, model.ErrUnavailable) { + result = "dependency_error" + } + return attachmentError(err) + } + result = "success" + s.metrics.attachmentBytes.Add(r.Context(), uploaded.SizeBytes) + writeJSON(w, http.StatusAccepted, uploaded) + return nil +} + +func attachmentFilename(query url.Values) (string, bool) { + if len(query) != 1 || len(query["filename"]) != 1 { + return "", false + } + filename := query["filename"][0] + length := utf8.RuneCountInString(filename) + if !utf8.ValidString(filename) || strings.TrimSpace(filename) == "" || length < 1 || length > 255 { + return "", false + } + for _, char := range filename { + if char == '/' || char == '\\' || char == 0 || unicode.IsControl(char) { + return "", false + } + } + return filename, true +} + +func (s *Server) attachmentMetadata(w http.ResponseWriter, r *http.Request) *apiError { + if r.Method != http.MethodGet { + return methodNotAllowed(w, http.MethodGet) + } + user, _, authErr := s.authenticate(r) + if authErr != nil { + return authErr + } + attachmentID, apiErr := attachmentIDFromRequest(r) + if apiErr != nil { + return apiErr + } + value, err := s.attachments.Metadata(r.Context(), attachmentID, user.ID) + if err != nil { + return attachmentError(err) + } + writeJSON(w, http.StatusOK, value) + return nil +} + +func (s *Server) attachmentContent(w http.ResponseWriter, r *http.Request) *apiError { + if r.Method != http.MethodGet { + return methodNotAllowed(w, http.MethodGet) + } + user, _, authErr := s.authenticate(r) + if authErr != nil { + return authErr + } + query, err := url.ParseQuery(r.URL.RawQuery) + variant, valid := attachmentVariant(query) + if err != nil || !valid { + return invalidRequest() + } + attachmentID, apiErr := attachmentIDFromRequest(r) + if apiErr != nil { + return apiErr + } + content, err := s.attachments.Content(r.Context(), attachmentID, user.ID, variant) + if err != nil { + return attachmentError(err) + } + defer func() { _ = content.Body.Close() }() + w.Header().Set("Content-Type", content.Variant.ContentType) + w.Header().Set("Content-Length", strconv.FormatInt(content.Variant.SizeBytes, 10)) + w.Header().Set("Content-Disposition", mime.FormatMediaType("inline", map[string]string{"filename": content.Attachment.Filename})) + w.WriteHeader(http.StatusOK) + if _, err := io.Copy(w, content.Body); err != nil { + span := trace.SpanFromContext(r.Context()) + span.RecordError(err) + span.SetStatus(codes.Error, "response body write failed") + s.logger.WarnContext(r.Context(), "attachment download interrupted", "event", "attachment_download_interrupted", "attachment_id", content.Attachment.ID, "error", err) + } + return nil +} + +func attachmentVariant(query url.Values) (string, bool) { + if len(query) == 0 { + return "original", true + } + if len(query) != 1 || len(query["variant"]) != 1 { + return "", false + } + switch query["variant"][0] { + case "original", "thumbnail", "crop", "optimized": + return query["variant"][0], true + default: + return "", false + } +} + +func attachmentIDFromRequest(r *http.Request) (string, *apiError) { + parts := strings.Split(strings.Trim(r.URL.Path, "/"), "/") + attachmentID, valid := canonicalID(parts[3]) + if !valid { + return "", invalidRequest() + } + return attachmentID, nil +} + +func attachmentError(err error) *apiError { + switch { + case errors.Is(err, model.ErrNotFound): + return notFound() + case errors.Is(err, model.ErrInvalidInput): + return invalidRequest() + case errors.Is(err, model.ErrUnsupportedMedia): + return &apiError{http.StatusUnsupportedMediaType, errorCodeUnsupportedMediaType, "unsupported image type"} + case errors.Is(err, model.ErrPayloadTooLarge): + return &apiError{http.StatusRequestEntityTooLarge, errorCodePayloadTooLarge, "image is too large"} + case errors.Is(err, model.ErrCapacity): + return &apiError{http.StatusTooManyRequests, errorCodeRateLimited, "image capacity is temporarily exhausted"} + case errors.Is(err, model.ErrConflict): + return &apiError{http.StatusConflict, errorCodeConflict, "attachment is not ready"} + default: + return unavailable() + } +} diff --git a/internal/messenger/httpapi/auth.go b/internal/messenger/httpapi/auth.go new file mode 100644 index 0000000..29419a1 --- /dev/null +++ b/internal/messenger/httpapi/auth.go @@ -0,0 +1,111 @@ +package httpapi + +import ( + "errors" + "io" + "net/http" + "strings" + + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model" +) + +func (s *Server) createSession(w http.ResponseWriter, r *http.Request) *apiError { + if r.Method != http.MethodPost { + return methodNotAllowed(w, http.MethodPost) + } + body, err := io.ReadAll(io.LimitReader(r.Body, 1)) + if err != nil || len(body) != 0 { + return invalidRequest() + } + username, passwordValue, ok := singleBasicAuth(r) + if !ok { + return unauthorized() + } + credentials, err := s.userRepository.UserByUsername(r.Context(), username) + if errors.Is(err, model.ErrNotFound) { + if _, verifyErr := s.passwords.Verify(passwordValue, s.passwords.DummyHash()); verifyErr != nil { + return internalError() + } + return unauthorized() + } + if err != nil { + return unavailable() + } + verified, err := s.passwords.Verify(passwordValue, credentials.PasswordHash) + if err != nil { + return internalError() + } + if !verified { + return unauthorized() + } + token, expiresAt, err := s.sessions.Create(r.Context(), credentials.ID, s.sessionTTL) + if err != nil { + return unavailable() + } + writeJSON(w, http.StatusCreated, struct { + Token string `json:"token"` + TokenType string `json:"token_type"` + ExpiresAt string `json:"expires_at"` + User User `json:"user"` + }{token, "Bearer", model.FormatTime(expiresAt), credentials.User}) + return nil +} + +func singleBasicAuth(r *http.Request) (string, string, bool) { + if len(r.Header.Values("Authorization")) != 1 { + return "", "", false + } + username, passwordValue, ok := r.BasicAuth() + return username, passwordValue, ok && username != "" +} + +func (s *Server) revokeSession(w http.ResponseWriter, r *http.Request) *apiError { + if r.Method != http.MethodDelete { + return methodNotAllowed(w, http.MethodDelete) + } + _, token, authErr := s.authenticate(r) + if authErr != nil { + return authErr + } + if err := s.sessions.Delete(r.Context(), token); errors.Is(err, model.ErrSessionNotFound) { + return unauthorized() + } else if err != nil { + return unavailable() + } + w.WriteHeader(http.StatusNoContent) + return nil +} + +func (s *Server) authenticate(r *http.Request) (User, string, *apiError) { + values := r.Header.Values("Authorization") + if len(values) != 1 { + return User{}, "", unauthorized() + } + scheme, token, found := strings.Cut(values[0], " ") + if !found || !strings.EqualFold(scheme, "Bearer") || token == "" || strings.ContainsAny(token, " \t\r\n") { + return User{}, "", unauthorized() + } + userID, err := s.sessions.UserID(r.Context(), token) + if errors.Is(err, model.ErrSessionNotFound) || errors.Is(err, model.ErrInvalidToken) { + return User{}, "", unauthorized() + } + if err != nil { + return User{}, "", unavailable() + } + user, err := s.userRepository.UserByID(r.Context(), userID) + if errors.Is(err, model.ErrNotFound) { + _ = s.sessions.Delete(r.Context(), token) + return User{}, "", unauthorized() + } + if err != nil { + return User{}, "", unavailable() + } + return user, token, nil +} + +func authenticationChallenge(r *http.Request) string { + if r.URL.Path == "/api/v1/auth/sessions" { + return `Basic realm="messenger", charset="UTF-8"` + } + return `Bearer realm="messenger"` +} diff --git a/internal/messenger/httpapi/errors.go b/internal/messenger/httpapi/errors.go new file mode 100644 index 0000000..95ffd1c --- /dev/null +++ b/internal/messenger/httpapi/errors.go @@ -0,0 +1,61 @@ +package httpapi + +import ( + "encoding/json" + "net/http" +) + +const ( + errorCodeInvalidRequest = "invalid_request" + errorCodeUnauthorized = "unauthorized" + errorCodeNotFound = "not_found" + errorCodeInternal = "internal_error" + errorCodeUnsupportedMediaType = "unsupported_media_type" + errorCodePayloadTooLarge = "payload_too_large" + errorCodeUnavailable = "unavailable" + errorCodeConflict = "conflict" + errorCodeRateLimited = "rate_limited" +) + +type apiError struct { + status int + code string + message string +} + +func methodNotAllowed(w http.ResponseWriter, allowed string) *apiError { + w.Header().Set("Allow", allowed) + return &apiError{http.StatusMethodNotAllowed, errorCodeInvalidRequest, "method not allowed"} +} + +func invalidRequest() *apiError { + return &apiError{http.StatusBadRequest, errorCodeInvalidRequest, "invalid request"} +} + +func unauthorized() *apiError { + return &apiError{http.StatusUnauthorized, errorCodeUnauthorized, "authentication required"} +} + +func notFound() *apiError { + return &apiError{http.StatusNotFound, errorCodeNotFound, "resource not found"} +} + +func internalError() *apiError { + return &apiError{http.StatusInternalServerError, errorCodeInternal, "internal server error"} +} + +func unavailable() *apiError { + return &apiError{http.StatusServiceUnavailable, errorCodeUnavailable, "service unavailable"} +} + +func writeError(w http.ResponseWriter, requestID string, err *apiError) { + writeJSON(w, err.status, map[string]any{"error": map[string]string{ + "code": err.code, "message": err.message, "request_id": requestID, + }}) +} + +func writeJSON(w http.ResponseWriter, status int, value any) { + w.Header().Set("Content-Type", "application/json; charset=utf-8") + w.WriteHeader(status) + _ = json.NewEncoder(w).Encode(value) +} diff --git a/internal/messenger/httpapi/handlers.go b/internal/messenger/httpapi/handlers.go new file mode 100644 index 0000000..9084d13 --- /dev/null +++ b/internal/messenger/httpapi/handlers.go @@ -0,0 +1,300 @@ +package httpapi + +import ( + "context" + "errors" + "net/http" + "regexp" + "strings" + "time" + "unicode/utf8" + + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model" +) + +var usernamePattern = regexp.MustCompile(`^[a-z0-9_]{3,32}$`) + +func (s *Server) live(w http.ResponseWriter, r *http.Request) *apiError { + if r.Method != http.MethodGet { + return methodNotAllowed(w, http.MethodGet) + } + writeJSON(w, http.StatusOK, map[string]string{"status": "ok"}) + return nil +} + +func (s *Server) readiness(w http.ResponseWriter, r *http.Request) *apiError { + if r.Method != http.MethodGet { + return methodNotAllowed(w, http.MethodGet) + } + if !s.ready.Load() { + return unavailable() + } + ctx, cancel := context.WithTimeout(r.Context(), 2*time.Second) + defer cancel() + if err := s.database.Ping(ctx); err != nil { + return unavailable() + } + if err := s.sessions.Ping(ctx); err != nil { + return unavailable() + } + if err := s.objects.Ping(ctx); err != nil { + return unavailable() + } + writeJSON(w, http.StatusOK, map[string]string{"status": "ok"}) + return nil +} + +func (s *Server) users(w http.ResponseWriter, r *http.Request) *apiError { + if r.Method != http.MethodPost { + return methodNotAllowed(w, http.MethodPost) + } + var input struct { + Username string `json:"username"` + DisplayName string `json:"display_name"` + Password string `json:"password"` + } + if err := readJSON(w, r, &input); err != nil { + return err + } + if !usernamePattern.MatchString(input.Username) || !validText(input.DisplayName, 100) || !validPassword(input.Password) { + return invalidRequest() + } + passwordHash, err := s.passwords.Hash(input.Password) + if err != nil { + return internalError() + } + user, err := s.userRepository.CreateUser(r.Context(), input.Username, input.DisplayName, passwordHash) + if errors.Is(err, model.ErrConflict) { + return &apiError{http.StatusConflict, errorCodeConflict, "username already exists"} + } + if err != nil { + return unavailable() + } + writeJSON(w, http.StatusCreated, user) + return nil +} + +func validPassword(value string) bool { + length := utf8.RuneCountInString(value) + return utf8.ValidString(value) && strings.TrimSpace(value) != "" && length >= 12 && length <= 128 +} + +func (s *Server) me(w http.ResponseWriter, r *http.Request) *apiError { + if r.Method != http.MethodGet { + return methodNotAllowed(w, http.MethodGet) + } + user, _, err := s.authenticate(r) + if err != nil { + return err + } + writeJSON(w, http.StatusOK, user) + return nil +} + +func (s *Server) chats(w http.ResponseWriter, r *http.Request) *apiError { + if r.Method != http.MethodPost && r.Method != http.MethodGet { + return methodNotAllowed(w, http.MethodGet+", "+http.MethodPost) + } + user, _, authErr := s.authenticate(r) + if authErr != nil { + return authErr + } + if r.Method == http.MethodGet { + return s.listChats(w, r, user) + } + var input struct { + Title string `json:"title"` + MemberIDs []string `json:"member_ids"` + } + if err := readJSON(w, r, &input); err != nil { + return err + } + if !validText(input.Title, 100) || len(input.MemberIDs) < 1 || len(input.MemberIDs) > 100 { + return invalidRequest() + } + members := make(map[string]struct{}, len(input.MemberIDs)) + for i, memberID := range input.MemberIDs { + canonical, valid := canonicalID(memberID) + if !valid { + return invalidRequest() + } + if _, duplicate := members[canonical]; duplicate { + return invalidRequest() + } + members[canonical] = struct{}{} + input.MemberIDs[i] = canonical + } + if _, included := members[user.ID]; !included { + return invalidRequest() + } + chat, err := s.chatRepository.CreateChat(r.Context(), input.Title, input.MemberIDs, user.ID) + if errors.Is(err, model.ErrInvalidMembers) { + return invalidRequest() + } + if err != nil { + return unavailable() + } + writeJSON(w, http.StatusCreated, chat) + return nil +} + +func (s *Server) listChats(w http.ResponseWriter, r *http.Request, user User) *apiError { + limit, cursor, err := pageParams(r, "chat", user.ID) + if err != nil { + return err + } + items, hasNext, listErr := s.chatRepository.ListChats(r.Context(), user.ID, cursorPosition(cursor), limit) + if errors.Is(listErr, model.ErrInvalidCursor) { + return invalidRequest() + } + if listErr != nil { + return unavailable() + } + if items == nil { + items = []Chat{} + } + var next *string + if hasNext { + last := items[len(items)-1] + value := encodeCursor("chat", user.ID, last.CreatedAt, last.ID) + next = &value + } + writeJSON(w, http.StatusOK, struct { + Items []Chat `json:"items"` + NextCursor *string `json:"next_cursor"` + }{items, next}) + return nil +} + +func (s *Server) chat(w http.ResponseWriter, r *http.Request) *apiError { + if r.Method != http.MethodGet { + return methodNotAllowed(w, http.MethodGet) + } + user, _, authErr := s.authenticate(r) + if authErr != nil { + return authErr + } + chatID, valid := canonicalID(strings.Split(r.URL.Path, "/")[4]) + if !valid { + return invalidRequest() + } + chat, err := s.chatRepository.ChatForUser(r.Context(), chatID, user.ID) + if errors.Is(err, model.ErrNotFound) { + return notFound() + } + if err != nil { + return unavailable() + } + writeJSON(w, http.StatusOK, chat) + return nil +} + +func (s *Server) messages(w http.ResponseWriter, r *http.Request) *apiError { + if r.Method != http.MethodPost && r.Method != http.MethodGet { + return methodNotAllowed(w, http.MethodGet+", "+http.MethodPost) + } + user, _, authErr := s.authenticate(r) + if authErr != nil { + return authErr + } + chatID, valid := canonicalID(strings.Split(r.URL.Path, "/")[4]) + if !valid { + return invalidRequest() + } + if r.Method == http.MethodGet { + return s.listMessages(w, r, user, chatID) + } + key, valid := idempotencyKey(r) + if !valid { + return invalidRequest() + } + var input createMessageRequest + if err := readJSON(w, r, &input); err != nil { + return err + } + text, attachmentIDs, valid := input.values() + if !valid || !validMessage(text, attachmentIDs) || !canonicalAttachmentIDs(attachmentIDs) { + return invalidRequest() + } + message, replay, err := s.messageRepository.CreateMessage( + r.Context(), chatID, user.ID, text, attachmentIDs, key, s.idempotencyTTL, + ) + if errors.Is(err, model.ErrNotFound) { + return notFound() + } + if errors.Is(err, model.ErrIdempotencyConflict) { + return &apiError{http.StatusConflict, errorCodeConflict, "idempotency key conflicts with previous request"} + } + if err != nil { + return unavailable() + } + if replay { + s.metrics.replays.Add(r.Context(), 1) + } + writeJSON(w, http.StatusCreated, message) + return nil +} + +func validMessage(text string, attachmentIDs []string) bool { + if !utf8.ValidString(text) || utf8.RuneCountInString(text) > 4000 { + return false + } + return len(attachmentIDs) > 0 || strings.TrimSpace(text) != "" +} + +func canonicalAttachmentIDs(attachmentIDs []string) bool { + if len(attachmentIDs) > 10 { + return false + } + seen := make(map[string]struct{}, len(attachmentIDs)) + for index, attachmentID := range attachmentIDs { + canonical, valid := canonicalID(attachmentID) + if !valid { + return false + } + if _, duplicate := seen[canonical]; duplicate { + return false + } + seen[canonical] = struct{}{} + attachmentIDs[index] = canonical + } + return true +} + +func (s *Server) listMessages(w http.ResponseWriter, r *http.Request, user User, chatID string) *apiError { + limit, cursor, err := pageParams(r, "message", chatID) + if err != nil { + return err + } + items, hasNext, listErr := s.messageRepository.ListMessages(r.Context(), chatID, user.ID, cursorPosition(cursor), limit) + if errors.Is(listErr, model.ErrNotFound) { + return notFound() + } + if errors.Is(listErr, model.ErrInvalidCursor) { + return invalidRequest() + } + if listErr != nil { + return unavailable() + } + if items == nil { + items = []Message{} + } + var next *string + if hasNext { + last := items[len(items)-1] + value := encodeCursor("message", chatID, last.CreatedAt, last.ID) + next = &value + } + writeJSON(w, http.StatusOK, struct { + Items []Message `json:"items"` + NextCursor *string `json:"next_cursor"` + }{items, next}) + return nil +} + +func cursorPosition(cursor *pageCursor) *model.Position { + if cursor == nil { + return nil + } + return &model.Position{CreatedAt: cursor.Time, ID: cursor.ID} +} diff --git a/internal/messenger/httpapi/messenger_interface.go b/internal/messenger/httpapi/messenger_interface.go new file mode 100644 index 0000000..9844c25 --- /dev/null +++ b/internal/messenger/httpapi/messenger_interface.go @@ -0,0 +1,49 @@ +package httpapi + +import ( + "context" + "time" + + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/messenger/attachment" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model" +) + +type UserRepository interface { + CreateUser(context.Context, string, string, string) (model.User, error) + UserByID(context.Context, string) (model.User, error) + UserByUsername(context.Context, string) (model.UserCredentials, error) +} + +type ChatRepository interface { + CreateChat(context.Context, string, []string, string) (model.Chat, error) + ChatForUser(context.Context, string, string) (model.Chat, error) + ListChats(context.Context, string, *model.Position, int) ([]model.Chat, bool, error) +} + +type MessageRepository interface { + CreateMessage(context.Context, string, string, string, []string, string, time.Duration) (model.Message, bool, error) + ListMessages(context.Context, string, string, *model.Position, int) ([]model.Message, bool, error) +} + +type AttachmentService interface { + Upload(context.Context, attachment.Upload) (model.Attachment, error) + Metadata(context.Context, string, string) (model.Attachment, error) + Content(context.Context, string, string, string) (attachment.Content, error) +} + +type HealthChecker interface { + Ping(context.Context) error +} + +type SessionStore interface { + HealthChecker + Create(context.Context, string, time.Duration) (string, time.Time, error) + UserID(context.Context, string) (string, error) + Delete(context.Context, string) error +} + +type PasswordService interface { + Hash(string) (string, error) + Verify(string, string) (bool, error) + DummyHash() string +} diff --git a/internal/messenger/httpapi/metrics.go b/internal/messenger/httpapi/metrics.go new file mode 100644 index 0000000..5047ebd --- /dev/null +++ b/internal/messenger/httpapi/metrics.go @@ -0,0 +1,90 @@ +package httpapi + +import ( + "context" + "net/http" + "strconv" + "strings" + "time" + + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/metric" +) + +var ( + requestDurationBuckets = []float64{0.001, 0.0025, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 20, 30} + uploadDurationBuckets = []float64{0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 20, 30, 60} +) + +type metrics struct { + requests metric.Int64Counter + duration metric.Float64Histogram + inFlight metric.Int64UpDownCounter + replays metric.Int64Counter + attachmentUploads metric.Int64Counter + attachmentDuration metric.Float64Histogram + attachmentBytes metric.Int64Counter + handler http.Handler +} + +func newMetrics(meter metric.Meter, handler http.Handler) *metrics { + if meter == nil { + meter = otel.Meter("messenger-api") + } + if handler == nil { + handler = http.NotFoundHandler() + } + requests, _ := meter.Int64Counter("messenger_http_requests", metric.WithDescription("Completed HTTP requests.")) + duration, _ := meter.Float64Histogram("messenger_http_request_duration", + metric.WithDescription("HTTP request duration."), metric.WithUnit("s"), + metric.WithExplicitBucketBoundaries(requestDurationBuckets...), + ) + inFlight, _ := meter.Int64UpDownCounter("messenger_http_requests_in_flight", metric.WithDescription("HTTP requests currently being served.")) + replays, _ := meter.Int64Counter("messenger_http_idempotency_replays", metric.WithDescription("Message requests served from an idempotency record.")) + attachmentUploads, _ := meter.Int64Counter("messenger_attachment_uploads", metric.WithDescription("Completed attachment uploads by result.")) + attachmentDuration, _ := meter.Float64Histogram("messenger_attachment_upload_duration", + metric.WithDescription("Attachment upload duration."), metric.WithUnit("s"), + metric.WithExplicitBucketBoundaries(uploadDurationBuckets...), + ) + attachmentBytes, _ := meter.Int64Counter("messenger_attachment_upload_bytes", metric.WithDescription("Bytes accepted by successful attachment uploads."), metric.WithUnit("By")) + return &metrics{ + requests: requests, duration: duration, inFlight: inFlight, replays: replays, + attachmentUploads: attachmentUploads, attachmentDuration: attachmentDuration, + attachmentBytes: attachmentBytes, handler: handler, + } +} + +func (m *metrics) observe(ctx context.Context, r *http.Request, status int, started time.Time) { + attributes := metric.WithAttributes( + attribute.String("method", r.Method), attribute.String("route", routeTemplate(r.URL.Path)), + attribute.String("status_class", strconv.Itoa(status/100)+"xx"), + ) + m.requests.Add(ctx, 1, attributes) + m.duration.Record(ctx, time.Since(started).Seconds(), attributes) +} + +func routeTemplate(path string) string { + switch path { + case "/health/live", "/health/ready", "/metrics", "/api/v1/users", "/api/v1/users/me", + "/api/v1/auth/sessions", "/api/v1/auth/sessions/current", "/api/v1/chats": + return path + } + parts := strings.Split(strings.Trim(path, "/"), "/") + if len(parts) == 4 && parts[0] == "api" && parts[1] == "v1" && parts[2] == "chats" { + return "/api/v1/chats/{chat_id}" + } + if len(parts) == 5 && parts[0] == "api" && parts[1] == "v1" && parts[2] == "chats" && parts[4] == "messages" { + return "/api/v1/chats/{chat_id}/messages" + } + if len(parts) == 5 && parts[0] == "api" && parts[1] == "v1" && parts[2] == "chats" && parts[4] == "attachments" { + return "/api/v1/chats/{chat_id}/attachments" + } + if len(parts) == 4 && parts[0] == "api" && parts[1] == "v1" && parts[2] == "attachments" { + return "/api/v1/attachments/{attachment_id}" + } + if len(parts) == 5 && parts[0] == "api" && parts[1] == "v1" && parts[2] == "attachments" && parts[4] == "content" { + return "/api/v1/attachments/{attachment_id}/content" + } + return "unknown" +} diff --git a/internal/messenger/httpapi/model.go b/internal/messenger/httpapi/model.go new file mode 100644 index 0000000..7f3f6dc --- /dev/null +++ b/internal/messenger/httpapi/model.go @@ -0,0 +1,30 @@ +package httpapi + +import ( + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model" + "github.com/google/uuid" +) + +type User = model.User +type Chat = model.Chat +type Message = model.Message +type Attachment = model.Attachment + +func newID() (string, error) { + id, err := uuid.NewRandom() + if err != nil { + return "", err + } + return id.String(), nil +} + +func canonicalID(value string) (string, bool) { + if len(value) != 36 { + return "", false + } + id, err := uuid.Parse(value) + if err != nil { + return "", false + } + return id.String(), true +} diff --git a/internal/messenger/httpapi/request.go b/internal/messenger/httpapi/request.go new file mode 100644 index 0000000..63cb694 --- /dev/null +++ b/internal/messenger/httpapi/request.go @@ -0,0 +1,147 @@ +package httpapi + +import ( + "bytes" + "encoding/base64" + "encoding/json" + "errors" + "io" + "mime" + "net/http" + "net/url" + "regexp" + "strconv" + "strings" + "time" + "unicode/utf8" +) + +const maxRequestBytes = 1 << 20 + +var idempotencyKeyPattern = regexp.MustCompile(`^[A-Za-z0-9._:-]{1,128}$`) + +type createMessageRequest struct { + Text json.RawMessage `json:"text"` + AttachmentIDs json.RawMessage `json:"attachment_ids"` +} + +func (r createMessageRequest) values() (string, []string, bool) { + var text *string + if len(r.Text) == 0 || json.Unmarshal(r.Text, &text) != nil || text == nil { + return "", nil, false + } + if len(r.AttachmentIDs) == 0 { + return *text, nil, true + } + var attachmentIDs *[]string + if json.Unmarshal(r.AttachmentIDs, &attachmentIDs) != nil || attachmentIDs == nil { + return "", nil, false + } + return *text, *attachmentIDs, true +} + +func readJSON(w http.ResponseWriter, r *http.Request, target any) *apiError { + mediaType, params, err := mime.ParseMediaType(r.Header.Get("Content-Type")) + if err != nil || mediaType != "application/json" || (params["charset"] != "" && !strings.EqualFold(params["charset"], "utf-8")) { + return &apiError{http.StatusUnsupportedMediaType, errorCodeUnsupportedMediaType, "expected application/json"} + } + r.Body = http.MaxBytesReader(w, r.Body, maxRequestBytes) + data, err := io.ReadAll(r.Body) + if err != nil { + var tooLarge *http.MaxBytesError + if errors.As(err, &tooLarge) { + return &apiError{http.StatusRequestEntityTooLarge, errorCodePayloadTooLarge, "request body too large"} + } + return invalidRequest() + } + if !utf8.Valid(data) { + return invalidRequest() + } + var object map[string]json.RawMessage + if err := json.Unmarshal(data, &object); err != nil || object == nil { + return invalidRequest() + } + normalized, err := json.Marshal(object) + if err != nil { + return invalidRequest() + } + decoder := json.NewDecoder(bytes.NewReader(normalized)) + decoder.DisallowUnknownFields() + if err := decoder.Decode(target); err != nil { + return invalidRequest() + } + return nil +} + +func validText(value string, maxRunes int) bool { + return utf8.ValidString(value) && strings.TrimSpace(value) != "" && utf8.RuneCountInString(value) <= maxRunes +} + +func idempotencyKey(r *http.Request) (string, bool) { + values := r.Header.Values("Idempotency-Key") + if len(values) != 1 || !idempotencyKeyPattern.MatchString(values[0]) { + return "", false + } + return values[0], true +} + +type pageCursor struct { + Kind string `json:"kind"` + Resource string `json:"resource"` + Time time.Time `json:"time"` + ID string `json:"id"` +} + +func pageParams(r *http.Request, kind, resource string) (int, *pageCursor, *apiError) { + query, err := url.ParseQuery(r.URL.RawQuery) + if err != nil { + return 0, nil, invalidRequest() + } + for key := range query { + if key != "limit" && key != "cursor" { + return 0, nil, invalidRequest() + } + if len(query[key]) != 1 { + return 0, nil, invalidRequest() + } + } + limit := 50 + if value, present := query["limit"]; present { + parsed, err := strconv.Atoi(value[0]) + if err != nil || parsed < 1 || parsed > 100 { + return 0, nil, invalidRequest() + } + limit = parsed + } + if value, present := query["cursor"]; present { + cursor, valid := decodeCursor(value[0], kind, resource) + if !valid { + return 0, nil, invalidRequest() + } + return limit, cursor, nil + } + return limit, nil, nil +} + +func encodeCursor(kind, resource string, createdAt time.Time, id string) string { + data, _ := json.Marshal(pageCursor{kind, resource, createdAt, id}) + return base64.RawURLEncoding.EncodeToString(data) +} + +func decodeCursor(value, kind, resource string) (*pageCursor, bool) { + data, err := base64.RawURLEncoding.DecodeString(value) + if err != nil { + return nil, false + } + var cursor pageCursor + if err := json.Unmarshal(data, &cursor); err != nil || cursor.Kind != kind || cursor.Resource != resource { + return nil, false + } + if cursor.Time.IsZero() { + return nil, false + } + if _, valid := canonicalID(cursor.ID); !valid { + return nil, false + } + return &cursor, true +} diff --git a/internal/messenger/httpapi/routes.go b/internal/messenger/httpapi/routes.go new file mode 100644 index 0000000..f272767 --- /dev/null +++ b/internal/messenger/httpapi/routes.go @@ -0,0 +1,47 @@ +package httpapi + +import ( + "net/http" + "strings" +) + +type requestHandler func(http.ResponseWriter, *http.Request) *apiError + +func (s *Server) route(w http.ResponseWriter, r *http.Request) *apiError { + var handler requestHandler + switch r.URL.Path { + case "/health/live": + handler = s.live + case "/health/ready": + handler = s.readiness + case "/metrics": + s.metrics.handler.ServeHTTP(w, r) + return nil + case "/api/v1/users": + handler = s.users + case "/api/v1/users/me": + handler = s.me + case "/api/v1/auth/sessions": + handler = s.createSession + case "/api/v1/auth/sessions/current": + handler = s.revokeSession + case "/api/v1/chats": + handler = s.chats + default: + parts := strings.Split(strings.Trim(r.URL.Path, "/"), "/") + if len(parts) == 4 && parts[0] == "api" && parts[1] == "v1" && parts[2] == "chats" { + handler = s.chat + } else if len(parts) == 5 && parts[0] == "api" && parts[1] == "v1" && parts[2] == "chats" && parts[4] == "messages" { + handler = s.messages + } else if len(parts) == 5 && parts[0] == "api" && parts[1] == "v1" && parts[2] == "chats" && parts[4] == "attachments" { + handler = s.uploadAttachment + } else if len(parts) == 4 && parts[0] == "api" && parts[1] == "v1" && parts[2] == "attachments" { + handler = s.attachmentMetadata + } else if len(parts) == 5 && parts[0] == "api" && parts[1] == "v1" && parts[2] == "attachments" && parts[4] == "content" { + handler = s.attachmentContent + } else { + return notFound() + } + } + return handler(w, r) +} diff --git a/internal/messenger/httpapi/server.go b/internal/messenger/httpapi/server.go new file mode 100644 index 0000000..fcc18bf --- /dev/null +++ b/internal/messenger/httpapi/server.go @@ -0,0 +1,158 @@ +package httpapi + +import ( + "context" + "log/slog" + "net/http" + "sync/atomic" + "time" + + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/codes" + "go.opentelemetry.io/otel/metric" + "go.opentelemetry.io/otel/propagation" + semconv "go.opentelemetry.io/otel/semconv/v1.34.0" + "go.opentelemetry.io/otel/trace" + + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/telemetry" +) + +type Dependencies struct { + Users UserRepository + Chats ChatRepository + Messages MessageRepository + Attachments AttachmentService + Sessions SessionStore + Passwords PasswordService + Database HealthChecker + Objects HealthChecker + Meter metric.Meter + Metrics http.Handler +} + +type Config struct { + SessionTTL time.Duration + IdempotencyTTL time.Duration + RequestTTL time.Duration + InstanceID string + SwaggerPort string +} + +type Server struct { + userRepository UserRepository + chatRepository ChatRepository + messageRepository MessageRepository + attachments AttachmentService + sessions SessionStore + passwords PasswordService + database HealthChecker + objects HealthChecker + sessionTTL time.Duration + idempotencyTTL time.Duration + requestTTL time.Duration + instanceID string + logger *slog.Logger + swaggerPort string + metrics *metrics + ready atomic.Bool +} + +type statusWriter struct { + http.ResponseWriter + status int +} + +func (w *statusWriter) WriteHeader(status int) { + w.status = status + w.ResponseWriter.WriteHeader(status) +} + +func NewServer( + logger *slog.Logger, + dependencies Dependencies, + config Config, +) *Server { + s := &Server{ + userRepository: dependencies.Users, chatRepository: dependencies.Chats, + messageRepository: dependencies.Messages, + attachments: dependencies.Attachments, sessions: dependencies.Sessions, + passwords: dependencies.Passwords, database: dependencies.Database, objects: dependencies.Objects, + sessionTTL: config.SessionTTL, idempotencyTTL: config.IdempotencyTTL, requestTTL: config.RequestTTL, + instanceID: config.InstanceID, logger: logger, swaggerPort: config.SwaggerPort, + metrics: newMetrics(dependencies.Meter, dependencies.Metrics), + } + s.ready.Store(true) + return s +} + +func (s *Server) SetReady(ready bool) { + s.ready.Store(ready) +} + +func (s *Server) Handler() http.Handler { + return http.HandlerFunc(s.serve) +} + +func (s *Server) serve(w http.ResponseWriter, r *http.Request) { + requestContext := telemetry.Extract(r.Context(), propagation.HeaderCarrier(r.Header)) + tracer := otel.Tracer("messenger-api/http") + route := routeTemplate(r.URL.Path) + requestContext, span := tracer.Start(requestContext, r.Method+" "+route, trace.WithSpanKind(trace.SpanKindServer), + trace.WithAttributes(semconv.HTTPRequestMethodKey.String(r.Method), semconv.HTTPRoute(route))) + defer span.End() + r = r.WithContext(requestContext) + w.Header().Set("X-Instance-Id", s.instanceID) + requestID, err := newID() + if err != nil { + http.Error(w, "server unavailable", http.StatusServiceUnavailable) + return + } + span.SetAttributes(attribute.String("request.id", requestID)) + w.Header().Set("X-Request-Id", requestID) + w.Header().Set("Cache-Control", "no-store") + wrapped := &statusWriter{ResponseWriter: w, status: http.StatusOK} + started := time.Now() + s.metrics.inFlight.Add(r.Context(), 1) + defer func() { + s.metrics.inFlight.Add(r.Context(), -1) + if recovered := recover(); recovered != nil { + s.logger.ErrorContext(r.Context(), "request panic", "event", "http_request_panic", "request_id", requestID, "method", r.Method, "route", route) + if wrapped.status == http.StatusOK { + writeError(wrapped, requestID, internalError()) + } + } + s.metrics.observe(r.Context(), r, wrapped.status, started) + span.SetAttributes(semconv.HTTPResponseStatusCode(wrapped.status)) + if wrapped.status >= http.StatusInternalServerError { + span.SetStatus(codes.Error, http.StatusText(wrapped.status)) + } + s.logger.InfoContext(r.Context(), "request", "event", "http_request_completed", "request_id", requestID, "method", r.Method, "route", route, "status", wrapped.status, "duration_ms", time.Since(started).Milliseconds()) + }() + if s.allowSwaggerOrigin(wrapped, r) && r.Method == http.MethodOptions && r.Header.Get("Access-Control-Request-Method") != "" { + wrapped.Header().Set("Access-Control-Allow-Methods", "GET, POST, DELETE, OPTIONS") + wrapped.Header().Set("Access-Control-Allow-Headers", "Authorization, Content-Type, Idempotency-Key") + wrapped.WriteHeader(http.StatusNoContent) + return + } + ctx, cancel := context.WithTimeout(r.Context(), s.requestTTL) + defer cancel() + r = r.WithContext(ctx) + if apiErr := s.route(wrapped, r); apiErr != nil { + if apiErr.status == http.StatusUnauthorized { + wrapped.Header().Set("WWW-Authenticate", authenticationChallenge(r)) + } + writeError(wrapped, requestID, apiErr) + } +} + +func (s *Server) allowSwaggerOrigin(w http.ResponseWriter, r *http.Request) bool { + origin := r.Header.Get("Origin") + if origin != "http://localhost:"+s.swaggerPort && origin != "http://127.0.0.1:"+s.swaggerPort { + return false + } + w.Header().Set("Access-Control-Allow-Origin", origin) + w.Header().Set("Access-Control-Expose-Headers", "X-Request-Id, X-Instance-Id") + w.Header().Add("Vary", "Origin") + return true +} diff --git a/internal/messenger/httpapi/server_test.go b/internal/messenger/httpapi/server_test.go new file mode 100644 index 0000000..c1bf91d --- /dev/null +++ b/internal/messenger/httpapi/server_test.go @@ -0,0 +1,378 @@ +package httpapi + +import ( + "context" + "errors" + "fmt" + "io" + "log/slog" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/suite" + + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/messenger/attachment" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model" +) + +var errDependency = errors.New("dependency unavailable") + +type MessengerSuite struct { + suite.Suite + logger *slog.Logger +} + +func TestMessengerSuite(t *testing.T) { + suite.Run(t, new(MessengerSuite)) +} + +func (s *MessengerSuite) SetupTest() { + s.logger = slog.New(slog.NewTextHandler(io.Discard, nil)) +} + +type messengerStoreStub struct { + pingErr error + userByID model.User + userByIDErr error + credentials model.UserCredentials + credentialsErr error + message model.Message + messageReplay bool + messageErr error +} + +func (s messengerStoreStub) Ping(context.Context) error { return s.pingErr } +func (messengerStoreStub) CreateUser(context.Context, string, string, string) (model.User, error) { + return model.User{}, nil +} +func (s messengerStoreStub) UserByID(context.Context, string) (model.User, error) { + return s.userByID, s.userByIDErr +} +func (s messengerStoreStub) UserByUsername(context.Context, string) (model.UserCredentials, error) { + return s.credentials, s.credentialsErr +} +func (messengerStoreStub) CreateChat(context.Context, string, []string, string) (model.Chat, error) { + return model.Chat{}, nil +} +func (messengerStoreStub) ChatForUser(context.Context, string, string) (model.Chat, error) { + return model.Chat{}, nil +} +func (messengerStoreStub) ListChats(context.Context, string, *model.Position, int) ([]model.Chat, bool, error) { + return nil, false, nil +} +func (s messengerStoreStub) CreateMessage(context.Context, string, string, string, []string, string, time.Duration) (model.Message, bool, error) { + return s.message, s.messageReplay, s.messageErr +} +func (messengerStoreStub) ListMessages(context.Context, string, string, *model.Position, int) ([]model.Message, bool, error) { + return nil, false, nil +} + +type objectStoreStub struct { + pingErr error +} + +func (s objectStoreStub) Ping(context.Context) error { return s.pingErr } + +type attachmentServiceStub struct { + uploaded model.Attachment + err error +} + +func (s attachmentServiceStub) Upload(context.Context, attachment.Upload) (model.Attachment, error) { + return s.uploaded, s.err +} +func (s attachmentServiceStub) Metadata(context.Context, string, string) (model.Attachment, error) { + return s.uploaded, s.err +} +func (s attachmentServiceStub) Content(context.Context, string, string, string) (attachment.Content, error) { + return attachment.Content{ + Attachment: s.uploaded, + Variant: model.Variant{ContentType: s.uploaded.ContentType, SizeBytes: s.uploaded.SizeBytes}, + Body: io.NopCloser(strings.NewReader("content")), + }, s.err +} + +type sessionStub struct { + pingErr error + userID string + userIDErr error +} + +func (s sessionStub) Ping(context.Context) error { return s.pingErr } +func (sessionStub) Create(context.Context, string, time.Duration) (string, time.Time, error) { + return "token", time.Now().Add(time.Hour), nil +} +func (s sessionStub) UserID(context.Context, string) (string, error) { return s.userID, s.userIDErr } +func (sessionStub) Delete(context.Context, string) error { return nil } + +type passwordsStub struct { + verified bool + err error +} + +func (passwordsStub) Hash(string) (string, error) { return "hash", nil } +func (passwordsStub) DummyHash() string { return "hash" } +func (s passwordsStub) Verify(string, string) (bool, error) { + return s.verified, s.err +} + +func (s *MessengerSuite) TestReadinessReturnsUnavailableForDependencyFailure() { + t := s.T() + tests := []struct { + name string + store messengerStoreStub + sessions sessionStub + objects objectStoreStub + }{ + {name: "postgres", store: messengerStoreStub{pingErr: errDependency}}, + {name: "valkey", sessions: sessionStub{pingErr: errDependency}}, + {name: "s3", objects: objectStoreStub{pingErr: errDependency}}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + server := s.server(tt.store, tt.sessions, passwordsStub{}, tt.objects, attachmentServiceStub{}) + response := serveRequest(server, http.MethodGet, "/health/ready", "") + if response.Code != http.StatusServiceUnavailable { + t.Fatalf("status = %d, want %d", response.Code, http.StatusServiceUnavailable) + } + }) + } +} + +func (s *MessengerSuite) TestBusinessRequestReturnsUnavailableForDependencyFailure() { + t := s.T() + tests := []struct { + name string + store messengerStoreStub + sessions sessionStub + }{ + {name: "valkey", sessions: sessionStub{userIDErr: errDependency}}, + {name: "postgres", store: messengerStoreStub{userByIDErr: errDependency}, sessions: sessionStub{userID: "user-id"}}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + server := s.newTestServer(tt.store, tt.sessions, passwordsStub{}) + request := httptest.NewRequest(http.MethodGet, "/api/v1/users/me", nil) + request.Header.Set("Authorization", "Bearer valid-token") + response := httptest.NewRecorder() + server.Handler().ServeHTTP(response, request) + if response.Code != http.StatusServiceUnavailable { + t.Fatalf("status = %d, want %d", response.Code, http.StatusServiceUnavailable) + } + }) + } +} + +func (s *MessengerSuite) TestAuthenticationChallenges() { + t := s.T() + server := s.newTestServer(messengerStoreStub{}, sessionStub{}, passwordsStub{}) + bearer := serveRequest(server, http.MethodGet, "/api/v1/users/me", "") + if bearer.Code != http.StatusUnauthorized || bearer.Header().Get("WWW-Authenticate") != `Bearer realm="messenger"` { + t.Fatalf("business challenge = %d %q", bearer.Code, bearer.Header().Get("WWW-Authenticate")) + } + + store := messengerStoreStub{credentials: model.UserCredentials{PasswordHash: "hash"}} + server = s.newTestServer(store, sessionStub{}, passwordsStub{verified: false}) + request := httptest.NewRequest(http.MethodPost, "/api/v1/auth/sessions", nil) + request.SetBasicAuth("user", "wrong-password") + response := httptest.NewRecorder() + server.Handler().ServeHTTP(response, request) + if response.Code != http.StatusUnauthorized || response.Header().Get("WWW-Authenticate") != `Basic realm="messenger", charset="UTF-8"` { + t.Fatalf("session challenge = %d %q", response.Code, response.Header().Get("WWW-Authenticate")) + } +} + +func (s *MessengerSuite) TestAttachmentContentAuthenticatesBeforeVariantValidation() { + server := s.newTestServer(messengerStoreStub{}, sessionStub{}, passwordsStub{}) + response := serveRequest( + server, + http.MethodGet, + "/api/v1/attachments/00000000-0000-0000-0000-000000000001/content?variant=invalid", + "", + ) + s.Equal(http.StatusUnauthorized, response.Code) + s.Equal(`Bearer realm="messenger"`, response.Header().Get("WWW-Authenticate")) +} + +func (s *MessengerSuite) TestSessionCreationRejectsChunkedBody() { + t := s.T() + server := s.newTestServer(messengerStoreStub{}, sessionStub{}, passwordsStub{}) + request := httptest.NewRequest(http.MethodPost, "/api/v1/auth/sessions", strings.NewReader("{}")) + request.ContentLength = -1 + request.SetBasicAuth("user", "password") + response := httptest.NewRecorder() + server.Handler().ServeHTTP(response, request) + if response.Code != http.StatusBadRequest { + t.Fatalf("status = %d, want %d", response.Code, http.StatusBadRequest) + } +} + +func (s *MessengerSuite) TestRegistrationRejectsWhitespacePassword() { + t := s.T() + server := s.newTestServer(messengerStoreStub{}, sessionStub{}, passwordsStub{}) + response := serveRequest(server, http.MethodPost, "/api/v1/users", `{"username":"student","display_name":"Student","password":" "}`) + if response.Code != http.StatusBadRequest { + t.Fatalf("status = %d, want %d", response.Code, http.StatusBadRequest) + } +} + +func (s *MessengerSuite) TestInstanceHeaderIsSetOnSuccessAndError() { + t := s.T() + server := s.newTestServer(messengerStoreStub{}, sessionStub{}, passwordsStub{}) + for _, response := range []*httptest.ResponseRecorder{ + serveRequest(server, http.MethodGet, "/health/live", ""), + serveRequest(server, http.MethodGet, "/api/v1/users/me", ""), + } { + if value := response.Header().Get("X-Instance-Id"); value != "test-instance" { + t.Fatalf("X-Instance-Id = %q, want test-instance", value) + } + } +} + +func (s *MessengerSuite) TestMessageCreationValidatesIdempotencyKey() { + t := s.T() + store := messengerStoreStub{userByID: model.User{ID: "user-id"}} + server := s.newTestServer(store, sessionStub{userID: "user-id"}, passwordsStub{}) + tests := []struct { + name string + key string + }{ + {name: "missing"}, + {name: "invalid", key: "invalid key"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + request := httptest.NewRequest(http.MethodPost, "/api/v1/chats/00000000-0000-0000-0000-000000000001/messages", strings.NewReader(`{"text":"message"}`)) + request.Header.Set("Authorization", "Bearer valid-token") + request.Header.Set("Content-Type", "application/json") + if tt.key != "" { + request.Header.Set("Idempotency-Key", tt.key) + } + response := httptest.NewRecorder() + server.Handler().ServeHTTP(response, request) + if response.Code != http.StatusBadRequest { + t.Fatalf("status = %d, want %d", response.Code, http.StatusBadRequest) + } + }) + } +} + +func (s *MessengerSuite) TestMessageCreationValidatesFieldPresenceAndTypes() { + store := messengerStoreStub{userByID: model.User{ID: "user-id"}} + server := s.newTestServer(store, sessionStub{userID: "user-id"}, passwordsStub{}) + attachmentID := "00000000-0000-0000-0000-000000000002" + invalidBodies := []string{ + `{"attachment_ids":["` + attachmentID + `"]}`, + `{"text":null,"attachment_ids":["` + attachmentID + `"]}`, + `{"text":"x","attachment_ids":null}`, + `{"text":1}`, + `{"text":"x","attachment_ids":{}}`, + } + for _, body := range invalidBodies { + response := createMessageRequestResponse(server, body) + s.Equal(http.StatusBadRequest, response.Code, body) + } + + validBodies := []string{ + `{"text":"message"}`, + `{"text":"message","attachment_ids":[]}`, + `{"text":"","attachment_ids":["` + attachmentID + `"]}`, + } + for _, body := range validBodies { + response := createMessageRequestResponse(server, body) + s.Equal(http.StatusCreated, response.Code, body) + } +} + +func createMessageRequestResponse(server *Server, body string) *httptest.ResponseRecorder { + request := httptest.NewRequest( + http.MethodPost, + "/api/v1/chats/00000000-0000-0000-0000-000000000001/messages", + strings.NewReader(body), + ) + request.Header.Set("Authorization", "Bearer valid-token") + request.Header.Set("Content-Type", "application/json") + request.Header.Set("Idempotency-Key", "message-key") + response := httptest.NewRecorder() + server.Handler().ServeHTTP(response, request) + return response +} + +func (s *MessengerSuite) TestMessageCreationReturnsConflictForReusedKeyWithDifferentBody() { + t := s.T() + store := messengerStoreStub{ + userByID: model.User{ID: "user-id"}, + messageErr: fmt.Errorf("create message: %w", model.ErrIdempotencyConflict), + } + server := s.newTestServer(store, sessionStub{userID: "user-id"}, passwordsStub{}) + request := httptest.NewRequest(http.MethodPost, "/api/v1/chats/00000000-0000-0000-0000-000000000001/messages", strings.NewReader(`{"text":"different"}`)) + request.Header.Set("Authorization", "Bearer valid-token") + request.Header.Set("Content-Type", "application/json") + request.Header.Set("Idempotency-Key", "same-key") + response := httptest.NewRecorder() + server.Handler().ServeHTTP(response, request) + if response.Code != http.StatusConflict { + t.Fatalf("status = %d, want %d", response.Code, http.StatusConflict) + } +} + +func (s *MessengerSuite) TestAttachmentFilenameValidation() { + for _, filename := range []string{"", " ", "../photo.png", "dir\\photo.png", "photo\x00.png", "photo\n.png"} { + _, valid := attachmentFilename(map[string][]string{"filename": {filename}}) + s.False(valid, filename) + } + filename, valid := attachmentFilename(map[string][]string{"filename": {"фото.png"}}) + s.True(valid) + s.Equal("фото.png", filename) +} + +func (s *MessengerSuite) TestUploadRejectsMalformedQuery() { + store := messengerStoreStub{userByID: model.User{ID: "user-id"}} + server := s.newTestServer(store, sessionStub{userID: "user-id"}, passwordsStub{}) + request := httptest.NewRequest( + http.MethodPost, + "/api/v1/chats/00000000-0000-0000-0000-000000000002/attachments?filename=photo.png&junk=%GG", + strings.NewReader("image"), + ) + request.Header.Set("Authorization", "Bearer valid-token") + request.Header.Set("Content-Type", "image/png") + response := httptest.NewRecorder() + server.Handler().ServeHTTP(response, request) + s.Equal(http.StatusBadRequest, response.Code) +} + +func (s *MessengerSuite) newTestServer(store messengerStoreStub, sessions SessionStore, passwords PasswordService) *Server { + return s.server(store, sessions, passwords, objectStoreStub{}, attachmentServiceStub{}) +} + +func (s *MessengerSuite) server( + store messengerStoreStub, + sessions SessionStore, + passwords PasswordService, + objects objectStoreStub, + attachments AttachmentService, +) *Server { + return NewServer(s.logger, Dependencies{ + Users: store, Chats: store, Messages: store, Attachments: attachments, + Sessions: sessions, Passwords: passwords, Database: store, Objects: objects, + }, Config{ + SessionTTL: time.Hour, IdempotencyTTL: 24 * time.Hour, RequestTTL: time.Second, + InstanceID: "test-instance", SwaggerPort: "8081", + }) +} + +func serveRequest(server *Server, method, path, body string) *httptest.ResponseRecorder { + request := httptest.NewRequest(method, path, strings.NewReader(body)) + if body != "" { + request.Header.Set("Content-Type", "application/json") + } + response := httptest.NewRecorder() + server.Handler().ServeHTTP(response, request) + return response +} diff --git a/internal/model/errors.go b/internal/model/errors.go new file mode 100644 index 0000000..83105b2 --- /dev/null +++ b/internal/model/errors.go @@ -0,0 +1,24 @@ +package model + +import "errors" + +var ( + ErrNotFound = errors.New("not found") + ErrConflict = errors.New("conflict") + ErrInvalidMembers = errors.New("invalid members") + ErrInvalidCursor = errors.New("invalid cursor") + ErrIdempotencyConflict = errors.New("idempotency conflict") + ErrInvalidToken = errors.New("invalid token") + ErrSessionNotFound = errors.New("session not found") + ErrInvalidInput = errors.New("invalid input") + ErrUnsupportedMedia = errors.New("unsupported media") + ErrPayloadTooLarge = errors.New("payload too large") + ErrCapacity = errors.New("capacity exhausted") + ErrUnavailable = errors.New("dependency unavailable") + ErrStaleLease = errors.New("stale lease") + ErrStaleAttempt = errors.New("stale attempt") + ErrStalePipeline = errors.New("stale pipeline") + ErrJobBusy = errors.New("job is already leased") + ErrJobComplete = errors.New("job is already complete") + ErrJobNotDue = errors.New("job is not due") +) diff --git a/internal/model/model.go b/internal/model/model.go new file mode 100644 index 0000000..83f7995 --- /dev/null +++ b/internal/model/model.go @@ -0,0 +1,223 @@ +package model + +import ( + "encoding/json" + "time" +) + +const ( + AttachmentPending = "pending" + AttachmentQueued = "queued" + AttachmentProcessing = "processing" + AttachmentReady = "ready" + AttachmentFailed = "failed" + AttachmentDeleting = "deleting" + + AttachmentJobQueued = "queued" + AttachmentJobProcessing = "processing" + AttachmentJobReady = "ready" + AttachmentJobFailed = "failed" + + JobMutationApplied JobMutationResult = "applied" + JobMutationAlreadyComplete JobMutationResult = "already_complete" + JobMutationStale JobMutationResult = "stale" +) + +const timestampLayout = "2006-01-02T15:04:05.000000Z" + +type User struct { + ID string `json:"id"` + Username string `json:"username"` + DisplayName string `json:"display_name"` + CreatedAt time.Time `json:"created_at"` +} + +type UserCredentials struct { + User + PasswordHash string `json:"-"` +} + +type Chat struct { + ID string `json:"id"` + Title string `json:"title"` + MemberIDs []string `json:"member_ids"` + CreatedAt time.Time `json:"created_at"` +} + +type Message struct { + ID string `json:"id"` + ChatID string `json:"chat_id"` + SenderID string `json:"sender_id"` + Text string `json:"text"` + CreatedAt time.Time `json:"created_at"` + AttachmentIDs []string `json:"attachment_ids"` +} + +type Attachment struct { + ID string `json:"id"` + ChatID string `json:"chat_id"` + OwnerID string `json:"owner_id"` + Filename string `json:"filename"` + ContentType string `json:"content_type"` + SizeBytes int64 `json:"size_bytes"` + SHA256 string `json:"sha256"` + Status string `json:"status"` + Variants []Variant `json:"variants"` + ErrorCode *string `json:"error_code"` + CreatedAt time.Time `json:"created_at"` + UpdatedAt time.Time `json:"updated_at"` + ObjectKey string `json:"-"` + StagingKey string `json:"-"` + ReadyAt *time.Time `json:"-"` + Referenced bool `json:"-"` + PipelineVersion int `json:"-"` +} + +type StoredObject struct { + Key string + Size int64 + LastModified time.Time +} + +type Variant struct { + Name string `json:"name"` + ContentType string `json:"content_type"` + SizeBytes int64 `json:"size_bytes"` + SHA256 string `json:"sha256"` + Width int `json:"width"` + Height int `json:"height"` + ObjectKey string `json:"-"` + PipelineVersion int `json:"-"` +} + +type AttachmentJob struct { + ID string + AttachmentID string + PipelineVersion int + Attempt int + Status string + AvailableAt time.Time + LeaseToken string + LeaseOwner string + LeaseUntil *time.Time + ErrorCode *string + CreatedAt time.Time + UpdatedAt time.Time +} + +type AttachmentJobMessage struct { + SchemaVersion int `json:"schema_version"` + JobID string `json:"job_id"` + Type string `json:"type"` + AttachmentID string `json:"attachment_id"` + PipelineVersion int `json:"pipeline_version"` + Attempt int `json:"attempt"` + CreatedAt time.Time `json:"created_at"` + Traceparent string `json:"traceparent,omitempty"` + Tracestate string `json:"tracestate,omitempty"` +} + +type AttachmentJobLease struct { + Job AttachmentJob + Attachment Attachment +} + +type OutboxEvent struct { + ID string + Topic string + PartitionKey string + Payload json.RawMessage + JobID string + Attempt int + AvailableAt time.Time + PublishTries int + LeaseToken string + LeaseUntil *time.Time + LastErrorCode *string + CreatedAt time.Time +} + +type JobMutationResult string + +type RecoveryResult struct { + Requeued int + Failed int +} + +type Position struct { + CreatedAt time.Time + ID string +} + +type userJSON struct { + ID string `json:"id"` + Username string `json:"username"` + DisplayName string `json:"display_name"` + CreatedAt string `json:"created_at"` +} + +type chatJSON struct { + ID string `json:"id"` + Title string `json:"title"` + MemberIDs []string `json:"member_ids"` + CreatedAt string `json:"created_at"` +} + +type messageJSON struct { + ID string `json:"id"` + ChatID string `json:"chat_id"` + SenderID string `json:"sender_id"` + Text string `json:"text"` + CreatedAt string `json:"created_at"` + AttachmentIDs []string `json:"attachment_ids"` +} + +type attachmentJSON struct { + ID string `json:"id"` + ChatID string `json:"chat_id"` + OwnerID string `json:"owner_id"` + Filename string `json:"filename"` + ContentType string `json:"content_type"` + SizeBytes int64 `json:"size_bytes"` + SHA256 string `json:"sha256"` + Status string `json:"status"` + Variants []Variant `json:"variants"` + ErrorCode *string `json:"error_code"` + CreatedAt string `json:"created_at"` + UpdatedAt string `json:"updated_at"` +} + +func (u User) MarshalJSON() ([]byte, error) { + return json.Marshal(userJSON{u.ID, u.Username, u.DisplayName, formatTime(u.CreatedAt)}) +} + +func (c Chat) MarshalJSON() ([]byte, error) { + return json.Marshal(chatJSON{c.ID, c.Title, c.MemberIDs, formatTime(c.CreatedAt)}) +} + +func (m Message) MarshalJSON() ([]byte, error) { + attachments := m.AttachmentIDs + if attachments == nil { + attachments = []string{} + } + return json.Marshal(messageJSON{m.ID, m.ChatID, m.SenderID, m.Text, formatTime(m.CreatedAt), attachments}) +} + +func (a Attachment) MarshalJSON() ([]byte, error) { + variants := a.Variants + if variants == nil { + variants = []Variant{} + } + return json.Marshal(attachmentJSON{ + a.ID, a.ChatID, a.OwnerID, a.Filename, a.ContentType, a.SizeBytes, a.SHA256, + a.Status, variants, a.ErrorCode, formatTime(a.CreatedAt), formatTime(a.UpdatedAt), + }) +} + +func FormatTime(value time.Time) string { + return formatTime(value) +} + +func formatTime(value time.Time) string { + return value.UTC().Format(timestampLayout) +} diff --git a/internal/model/model_test.go b/internal/model/model_test.go new file mode 100644 index 0000000..d3a54d9 --- /dev/null +++ b/internal/model/model_test.go @@ -0,0 +1,24 @@ +package model + +import ( + "encoding/json" + "testing" + "time" + + "github.com/stretchr/testify/suite" +) + +type ModelSuite struct { + suite.Suite +} + +func TestModelSuite(t *testing.T) { + suite.Run(t, new(ModelSuite)) +} + +func (s *ModelSuite) TestUserTimestampHasFixedMicroseconds() { + createdAt := time.Date(2026, time.September, 23, 21, 28, 15, 423030000, time.UTC) + data, err := json.Marshal(User{ID: "id", Username: "user", DisplayName: "User", CreatedAt: createdAt}) + s.Require().NoError(err) + s.Equal(`{"id":"id","username":"user","display_name":"User","created_at":"2026-09-23T21:28:15.423030Z"}`, string(data)) +} diff --git a/internal/objectstore/store.go b/internal/objectstore/store.go new file mode 100644 index 0000000..e2d6ded --- /dev/null +++ b/internal/objectstore/store.go @@ -0,0 +1,225 @@ +package objectstore + +import ( + "bytes" + "context" + "errors" + "fmt" + "io" + "net/url" + "strings" + "time" + + "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" + + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/codes" + "go.opentelemetry.io/otel/trace" + + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model" +) + +var ErrNotFound = errors.New("object not found") + +type Config struct { + Endpoint string + Region string + Bucket string + AccessKey string + SecretKey string + UseTLS bool + Timeout time.Duration +} + +type Store struct { + client *minio.Client + bucket string + timeout time.Duration +} + +func New(config Config) (*Store, error) { + endpoint, err := endpointHost(config.Endpoint, config.UseTLS) + if err != nil { + return nil, err + } + if strings.TrimSpace(config.Bucket) == "" || config.Timeout <= 0 { + return nil, errors.New("invalid object store configuration") + } + client, err := minio.New(endpoint, &minio.Options{ + Creds: credentials.NewStaticV4(config.AccessKey, config.SecretKey, ""), + Secure: config.UseTLS, + Region: config.Region, + }) + if err != nil { + return nil, fmt.Errorf("create S3 client: %w", err) + } + return &Store{client: client, bucket: config.Bucket, timeout: config.Timeout}, nil +} + +func endpointHost(value string, useTLS bool) (string, error) { + value = strings.TrimSpace(value) + if !strings.Contains(value, "://") { + if value == "" { + return "", errors.New("S3 endpoint is required") + } + return value, nil + } + parsed, err := url.Parse(value) + if err != nil || parsed.Host == "" || parsed.Path != "" && parsed.Path != "/" { + return "", errors.New("invalid S3 endpoint") + } + if (parsed.Scheme == "https") != useTLS || parsed.Scheme != "http" && parsed.Scheme != "https" { + return "", errors.New("S3 endpoint scheme conflicts with S3_USE_TLS") + } + return parsed.Host, nil +} + +func (s *Store) Ping(ctx context.Context) error { + ctx, cancel := context.WithTimeout(ctx, s.timeout) + defer cancel() + exists, err := s.client.BucketExists(ctx, s.bucket) + if err != nil { + return fmt.Errorf("check S3 bucket: %w", err) + } + if !exists { + return errors.New("S3 bucket does not exist") + } + return nil +} + +func (s *Store) Put(ctx context.Context, key, contentType string, data []byte) error { + ctx, span := s.startSpan(ctx, "PUT") + defer span.End() + ctx, cancel := context.WithTimeout(ctx, s.timeout) + defer cancel() + _, err := s.client.PutObject( + ctx, s.bucket, key, bytes.NewReader(data), int64(len(data)), minio.PutObjectOptions{ContentType: contentType}, + ) + if err != nil { + recordSpanError(span, err) + return fmt.Errorf("put S3 object: %w", err) + } + return nil +} + +func (s *Store) Get(ctx context.Context, key string) (io.ReadCloser, error) { + ctx, span := s.startSpan(ctx, "GET") + ctx, cancel := context.WithTimeout(ctx, s.timeout) + object, err := s.client.GetObject(ctx, s.bucket, key, minio.GetObjectOptions{}) + if err != nil { + cancel() + recordSpanError(span, err) + span.End() + return nil, objectError("get S3 object", err) + } + if _, err := object.Stat(); err != nil { + cancel() + _ = object.Close() + recordSpanError(span, err) + span.End() + return nil, objectError("stat S3 object", err) + } + return &tracedReader{ReadCloser: &timeoutReader{ReadCloser: object, cancel: cancel}, span: span}, nil +} + +func (s *Store) Stat(ctx context.Context, key string) (model.StoredObject, error) { + ctx, cancel := context.WithTimeout(ctx, s.timeout) + defer cancel() + info, err := s.client.StatObject(ctx, s.bucket, key, minio.StatObjectOptions{}) + if err != nil { + return model.StoredObject{}, objectError("stat S3 object", err) + } + return model.StoredObject{Key: info.Key, Size: info.Size, LastModified: info.LastModified}, nil +} + +func (s *Store) Delete(ctx context.Context, key string) error { + ctx, span := s.startSpan(ctx, "DELETE") + defer span.End() + ctx, cancel := context.WithTimeout(ctx, s.timeout) + defer cancel() + if err := s.client.RemoveObject(ctx, s.bucket, key, minio.RemoveObjectOptions{}); err != nil { + recordSpanError(span, err) + return objectError("delete S3 object", err) + } + return nil +} + +func (s *Store) List(ctx context.Context, prefix, after string, limit int) ([]model.StoredObject, string, error) { + if limit < 1 || limit > 1000 { + return nil, "", errors.New("object list limit must be between 1 and 1000") + } + ctx, cancel := context.WithTimeout(ctx, s.timeout) + defer cancel() + objects := make([]model.StoredObject, 0, limit+1) + options := minio.ListObjectsOptions{Prefix: prefix, Recursive: true, StartAfter: after, MaxKeys: limit + 1} + for info := range s.client.ListObjects(ctx, s.bucket, options) { + if info.Err != nil { + return nil, "", fmt.Errorf("list S3 objects: %w", info.Err) + } + objects = append(objects, model.StoredObject{Key: info.Key, Size: info.Size, LastModified: info.LastModified}) + if len(objects) == limit+1 { + cancel() + break + } + } + var next string + if len(objects) > limit { + objects = objects[:limit] + next = objects[len(objects)-1].Key + } + return objects, next, nil +} + +type timeoutReader struct { + io.ReadCloser + cancel context.CancelFunc +} + +type tracedReader struct { + io.ReadCloser + span trace.Span +} + +func (r *tracedReader) Read(buffer []byte) (int, error) { + read, err := r.ReadCloser.Read(buffer) + if err != nil && !errors.Is(err, io.EOF) { + recordSpanError(r.span, err) + } + return read, err +} + +func (r *tracedReader) Close() error { + err := r.ReadCloser.Close() + if err != nil { + recordSpanError(r.span, err) + } + r.span.End() + return err +} + +func (r *timeoutReader) Close() error { + err := r.ReadCloser.Close() + r.cancel() + return err +} + +func objectError(action string, err error) error { + response := minio.ToErrorResponse(err) + if response.Code == "NoSuchKey" || response.Code == "NoSuchObject" { + return ErrNotFound + } + return fmt.Errorf("%s: %w", action, err) +} + +func (s *Store) startSpan(ctx context.Context, operation string) (context.Context, trace.Span) { + return otel.Tracer("objectstore").Start(ctx, "s3 "+operation, trace.WithSpanKind(trace.SpanKindClient), + trace.WithAttributes(attribute.String("storage.system", "s3"), attribute.String("storage.operation.name", operation), + attribute.String("s3.bucket.name", s.bucket))) +} + +func recordSpanError(span trace.Span, err error) { + span.RecordError(err) + span.SetStatus(codes.Error, "S3 operation failed") +} diff --git a/internal/outbox/dispatcher.go b/internal/outbox/dispatcher.go new file mode 100644 index 0000000..ea74f61 --- /dev/null +++ b/internal/outbox/dispatcher.go @@ -0,0 +1,233 @@ +package outbox + +import ( + "context" + "errors" + "fmt" + "time" + + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/codes" + "go.opentelemetry.io/otel/trace" + + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/telemetry" +) + +var errLeaseLost = errors.New("outbox lease lost") + +type Store interface { + ClaimOutboxEvents(context.Context, int, time.Duration) ([]model.OutboxEvent, error) + MarkOutboxPublished(context.Context, string, string) (bool, error) + ReleaseOutboxEvent(context.Context, string, string, string, time.Time) (bool, error) + RecoverExpiredOutboxEvents(context.Context, time.Time) (int, error) + DeletePublishedOutboxEvents(context.Context, time.Time, int) (int, error) +} + +type Publisher interface { + PublishConfirmed(context.Context, []byte) (string, error) +} + +type Metrics interface { + ObservePublished(context.Context) + ObservePublishFailure(context.Context) + ObserveRecovered(context.Context, int) +} + +type Config struct { + BatchSize int + LeaseDuration time.Duration + PollInterval time.Duration + RecoveryInterval time.Duration + RetryDelay time.Duration + Retention time.Duration + CleanupInterval time.Duration + CleanupBatchSize int +} + +type Dispatcher struct { + store Store + publisher Publisher + metrics Metrics + config Config + now func() time.Time +} + +func New(store Store, publisher Publisher, metrics Metrics, config Config) (*Dispatcher, error) { + if store == nil { + return nil, errors.New("outbox store is required") + } + if publisher == nil { + return nil, errors.New("outbox publisher is required") + } + if err := config.validate(); err != nil { + return nil, err + } + if metrics == nil { + metrics = nopMetrics{} + } + return &Dispatcher{store: store, publisher: publisher, metrics: metrics, config: config, now: time.Now}, nil +} + +func (c Config) validate() error { + switch { + case c.BatchSize <= 0: + return errors.New("outbox batch size must be positive") + case c.LeaseDuration <= 0: + return errors.New("outbox lease duration must be positive") + case c.PollInterval <= 0: + return errors.New("outbox poll interval must be positive") + case c.RecoveryInterval <= 0: + return errors.New("outbox recovery interval must be positive") + case c.RetryDelay <= 0: + return errors.New("outbox retry delay must be positive") + case c.Retention <= 0: + return errors.New("outbox retention must be positive") + case c.CleanupInterval <= 0: + return errors.New("outbox cleanup interval must be positive") + case c.CleanupBatchSize <= 0 || c.CleanupBatchSize > 1000: + return errors.New("outbox cleanup batch size must be between 1 and 1000") + default: + return nil + } +} + +func (d *Dispatcher) Run(ctx context.Context) error { + if ctx.Err() != nil { + return nil + } + + pollTicker := time.NewTicker(d.config.PollInterval) + recoveryTicker := time.NewTicker(d.config.RecoveryInterval) + cleanupTicker := time.NewTicker(d.config.CleanupInterval) + defer pollTicker.Stop() + defer recoveryTicker.Stop() + defer cleanupTicker.Stop() + + if err := d.RecoverExpired(ctx); err != nil { + if ctx.Err() != nil { + return nil + } + return err + } + for { + if _, err := d.DispatchOnce(ctx); err != nil { + if ctx.Err() != nil { + return nil + } + return err + } + + select { + case <-ctx.Done(): + return nil + case <-pollTicker.C: + case <-recoveryTicker.C: + if err := d.RecoverExpired(ctx); err != nil { + if ctx.Err() != nil { + return nil + } + return err + } + case <-cleanupTicker.C: + if _, err := d.CleanupPublished(ctx); err != nil { + if ctx.Err() != nil { + return nil + } + return err + } + } + } +} + +func (d *Dispatcher) CleanupPublished(ctx context.Context) (int, error) { + deleted, err := d.store.DeletePublishedOutboxEvents( + ctx, d.now().Add(-d.config.Retention), d.config.CleanupBatchSize, + ) + if err != nil { + return 0, fmt.Errorf("delete published outbox events: %w", err) + } + return deleted, nil +} + +func (d *Dispatcher) DispatchOnce(ctx context.Context) (int, error) { + events, err := d.store.ClaimOutboxEvents(ctx, d.config.BatchSize, d.config.LeaseDuration) + if err != nil { + return 0, fmt.Errorf("claim due outbox events: %w", err) + } + + published := 0 + for _, event := range events { + if err := ctx.Err(); err != nil { + return published, nil + } + publishContext := telemetry.ExtractJSONMessage(ctx, event.Payload) + publishContext, span := otel.Tracer("messenger-dispatcher").Start(publishContext, "attachment-jobs publish", + trace.WithSpanKind(trace.SpanKindProducer), trace.WithAttributes( + attribute.String("messaging.system", "redis"), + attribute.String("messaging.destination.name", "attachment-jobs"), + attribute.String("job.id", event.JobID), attribute.Int("job.attempt", event.Attempt), + )) + payload, injectErr := telemetry.InjectJSONMessage(publishContext, event.Payload) + if injectErr != nil { + span.RecordError(injectErr) + span.SetStatus(codes.Error, "inject trace context") + span.End() + return published, fmt.Errorf("inject outbox trace context: %w", injectErr) + } + _, err := d.publisher.PublishConfirmed(publishContext, payload) + if err != nil { + span.RecordError(err) + span.SetStatus(codes.Error, "publish failed") + span.End() + if ctx.Err() != nil { + return published, nil + } + d.metrics.ObservePublishFailure(publishContext) + nextAttemptAt := d.now().Add(d.config.RetryDelay) + released, releaseErr := d.store.ReleaseOutboxEvent( + publishContext, event.ID, event.LeaseToken, "broker_publish_failed", nextAttemptAt, + ) + if releaseErr != nil { + return published, fmt.Errorf("release outbox event after publish failure: %w", releaseErr) + } + if !released { + return published, fmt.Errorf("release outbox event after publish failure: %w", errLeaseLost) + } + continue + } + marked, err := d.store.MarkOutboxPublished(publishContext, event.ID, event.LeaseToken) + if err != nil { + span.RecordError(err) + span.SetStatus(codes.Error, "mark published") + span.End() + return published, fmt.Errorf("mark outbox event published: %w", err) + } + if !marked { + span.SetStatus(codes.Error, "outbox lease lost") + span.End() + return published, fmt.Errorf("mark outbox event published: %w", errLeaseLost) + } + span.SetAttributes(attribute.String("messaging.operation.type", "publish")) + span.End() + d.metrics.ObservePublished(publishContext) + published++ + } + return published, nil +} + +func (d *Dispatcher) RecoverExpired(ctx context.Context) error { + recovered, err := d.store.RecoverExpiredOutboxEvents(ctx, d.now()) + if err != nil { + return fmt.Errorf("recover expired outbox leases: %w", err) + } + d.metrics.ObserveRecovered(ctx, recovered) + return nil +} + +type nopMetrics struct{} + +func (nopMetrics) ObservePublished(context.Context) {} +func (nopMetrics) ObservePublishFailure(context.Context) {} +func (nopMetrics) ObserveRecovered(context.Context, int) {} diff --git a/internal/outbox/dispatcher_test.go b/internal/outbox/dispatcher_test.go new file mode 100644 index 0000000..b9d792c --- /dev/null +++ b/internal/outbox/dispatcher_test.go @@ -0,0 +1,207 @@ +package outbox + +import ( + "context" + "encoding/json" + "errors" + "sync" + "testing" + "time" + + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model" + "github.com/stretchr/testify/require" +) + +func TestDispatcherPublishesAndMarksEvent(t *testing.T) { + store := &storeStub{events: []model.OutboxEvent{{ID: "event-1", LeaseToken: "lease-1", Payload: testPayload()}}} + publisher := &publisherStub{messageID: "1-0"} + dispatcher := newTestDispatcher(t, store, publisher) + + published, err := dispatcher.DispatchOnce(context.Background()) + require.NoError(t, err) + require.Equal(t, 1, published) + require.Equal(t, []string{"event-1:lease-1"}, store.marked) +} + +func TestDispatcherRetriesSafelyAfterMarkFailure(t *testing.T) { + event := model.OutboxEvent{ID: "event-1", LeaseToken: "lease-1", Payload: testPayload()} + store := &storeStub{events: []model.OutboxEvent{event}, markErr: errors.New("database unavailable")} + publisher := &publisherStub{messageID: "1-0"} + dispatcher := newTestDispatcher(t, store, publisher) + + _, err := dispatcher.DispatchOnce(context.Background()) + require.Error(t, err) + require.Equal(t, 1, publisher.calls) + + store.markErr = nil + store.events[0].LeaseToken = "lease-2" + publisher.messageID = "2-0" + published, err := dispatcher.DispatchOnce(context.Background()) + require.NoError(t, err) + require.Equal(t, 1, published) + require.Equal(t, 2, publisher.calls) + require.JSONEq(t, string(event.Payload), string(publisher.payloads[0])) + require.JSONEq(t, string(publisher.payloads[0]), string(publisher.payloads[1])) + require.Equal(t, []string{"event-1:lease-2"}, store.marked) +} + +func TestDispatcherReleasesPublishFailure(t *testing.T) { + store := &storeStub{events: []model.OutboxEvent{{ID: "event-1", LeaseToken: "lease-1", Payload: testPayload()}}} + publisher := &publisherStub{err: errors.New("redis unavailable")} + dispatcher := newTestDispatcher(t, store, publisher) + dispatcher.now = func() time.Time { return time.Date(2026, 10, 22, 12, 0, 0, 0, time.UTC) } + + published, err := dispatcher.DispatchOnce(context.Background()) + require.NoError(t, err) + require.Zero(t, published) + require.Equal(t, "event-1:lease-1:broker_publish_failed", store.released) + require.Equal(t, dispatcher.now().Add(time.Second), store.nextAttemptAt) +} + +func TestDispatcherRejectsStaleLeaseAfterPublish(t *testing.T) { + store := &storeStub{ + events: []model.OutboxEvent{{ID: "event-1", LeaseToken: "stale-lease", Payload: testPayload()}}, + markLost: true, + } + dispatcher := newTestDispatcher(t, store, &publisherStub{messageID: "1-0"}) + + published, err := dispatcher.DispatchOnce(context.Background()) + require.ErrorIs(t, err, errLeaseLost) + require.Zero(t, published) +} + +func TestDispatcherCleansPublishedEventsBeforeRetentionCutoff(t *testing.T) { + store := &storeStub{deleted: 2} + dispatcher := newTestDispatcher(t, store, &publisherStub{}) + dispatcher.now = func() time.Time { return time.Date(2026, 10, 22, 12, 0, 0, 0, time.UTC) } + + deleted, err := dispatcher.CleanupPublished(context.Background()) + require.NoError(t, err) + require.Equal(t, 2, deleted) + require.Equal(t, dispatcher.now().Add(-24*time.Hour), store.publishedBefore) + require.Equal(t, 100, store.cleanupLimit) +} + +func TestDispatcherStopsAfterCancellation(t *testing.T) { + store := &storeStub{} + publisher := &publisherStub{} + dispatcher := newTestDispatcher(t, store, publisher) + ctx, cancel := context.WithCancel(context.Background()) + cancel() + + done := make(chan error, 1) + go func() { done <- dispatcher.Run(ctx) }() + + select { + case err := <-done: + require.NoError(t, err) + case <-time.After(time.Second): + t.Fatal("dispatcher did not stop after cancellation") + } +} + +func newTestDispatcher(t *testing.T, store Store, publisher Publisher) *Dispatcher { + t.Helper() + dispatcher, err := New(store, publisher, nil, Config{ + BatchSize: 10, + LeaseDuration: time.Minute, + PollInterval: time.Millisecond, + RecoveryInterval: time.Second, + RetryDelay: time.Second, + Retention: 24 * time.Hour, + CleanupInterval: time.Hour, + CleanupBatchSize: 100, + }) + require.NoError(t, err) + return dispatcher +} + +func testPayload() json.RawMessage { + return json.RawMessage(`{ + "schema_version": 1, + "job_id": "11111111-1111-4111-8111-111111111111", + "type": "attachment.process.v1", + "attachment_id": "22222222-2222-4222-8222-222222222222", + "pipeline_version": 1, + "attempt": 1, + "created_at": "2026-10-22T12:00:00Z" + }`) +} + +type storeStub struct { + mu sync.Mutex + events []model.OutboxEvent + markErr error + markLost bool + marked []string + released string + nextAttemptAt time.Time + recoveredCalls int + deleted int + publishedBefore time.Time + cleanupLimit int +} + +func (s *storeStub) ClaimOutboxEvents(context.Context, int, time.Duration) ([]model.OutboxEvent, error) { + s.mu.Lock() + defer s.mu.Unlock() + return append([]model.OutboxEvent(nil), s.events...), nil +} + +func (s *storeStub) MarkOutboxPublished(_ context.Context, id, leaseToken string) (bool, error) { + s.mu.Lock() + defer s.mu.Unlock() + if s.markErr != nil { + return false, s.markErr + } + if s.markLost { + return false, nil + } + s.marked = append(s.marked, id+":"+leaseToken) + return true, nil +} + +func (s *storeStub) ReleaseOutboxEvent( + _ context.Context, + id string, + leaseToken string, + code string, + nextAttemptAt time.Time, +) (bool, error) { + s.mu.Lock() + defer s.mu.Unlock() + s.released = id + ":" + leaseToken + ":" + code + s.nextAttemptAt = nextAttemptAt + return true, nil +} + +func (s *storeStub) RecoverExpiredOutboxEvents(context.Context, time.Time) (int, error) { + s.mu.Lock() + defer s.mu.Unlock() + s.recoveredCalls++ + return 0, nil +} + +func (s *storeStub) DeletePublishedOutboxEvents(_ context.Context, before time.Time, limit int) (int, error) { + s.mu.Lock() + defer s.mu.Unlock() + s.publishedBefore = before + s.cleanupLimit = limit + return s.deleted, nil +} + +type publisherStub struct { + mu sync.Mutex + messageID string + err error + calls int + payloads [][]byte +} + +func (p *publisherStub) PublishConfirmed(_ context.Context, payload []byte) (string, error) { + p.mu.Lock() + defer p.mu.Unlock() + p.calls++ + p.payloads = append(p.payloads, append([]byte(nil), payload...)) + return p.messageID, p.err +} diff --git a/internal/password/password.go b/internal/password/password.go new file mode 100644 index 0000000..5dd8b1b --- /dev/null +++ b/internal/password/password.go @@ -0,0 +1,133 @@ +package password + +import ( + "crypto/rand" + "crypto/subtle" + "encoding/base64" + "errors" + "fmt" + "strconv" + "strings" + + "golang.org/x/crypto/argon2" +) + +var ( + ErrInvalidConfig = errors.New("invalid password configuration") + ErrInvalidHash = errors.New("invalid password hash") +) + +type Config struct { + MemoryKiB uint32 + Iterations uint32 + Parallelism uint8 + SaltLength uint32 + KeyLength uint32 +} + +type Hasher struct { + config Config + dummyHash string +} + +func New(config Config) (*Hasher, error) { + if err := validateConfig(config); err != nil { + return nil, err + } + salt := make([]byte, config.SaltLength) + digest := argon2.IDKey([]byte("invalid-user-password"), salt, config.Iterations, config.MemoryKiB, config.Parallelism, config.KeyLength) + return &Hasher{config: config, dummyHash: encode(config, salt, digest)}, nil +} + +func validateConfig(config Config) error { + if config.MemoryKiB < 8*1024 || config.MemoryKiB > 1024*1024 || + config.Iterations < 1 || config.Iterations > 20 || + config.Parallelism < 1 || + config.SaltLength < 8 || config.SaltLength > 1024 || + config.KeyLength < 16 || config.KeyLength > 1024 { + return ErrInvalidConfig + } + return nil +} + +func (h *Hasher) Hash(value string) (string, error) { + salt := make([]byte, h.config.SaltLength) + if _, err := rand.Read(salt); err != nil { + return "", fmt.Errorf("generate password salt: %w", err) + } + digest := argon2.IDKey([]byte(value), salt, h.config.Iterations, h.config.MemoryKiB, h.config.Parallelism, h.config.KeyLength) + return encode(h.config, salt, digest), nil +} + +func (h *Hasher) Verify(value, encodedHash string) (bool, error) { + config, salt, digest, err := decode(encodedHash) + if err != nil { + return false, err + } + actual := argon2.IDKey([]byte(value), salt, config.Iterations, config.MemoryKiB, config.Parallelism, config.KeyLength) + return subtle.ConstantTimeCompare(actual, digest) == 1, nil +} + +func (h *Hasher) DummyHash() string { + return h.dummyHash +} + +func encode(config Config, salt, digest []byte) string { + return fmt.Sprintf( + "$argon2id$v=%d$m=%d,t=%d,p=%d$%s$%s", + argon2.Version, + config.MemoryKiB, + config.Iterations, + config.Parallelism, + base64.RawStdEncoding.EncodeToString(salt), + base64.RawStdEncoding.EncodeToString(digest), + ) +} + +func decode(encodedHash string) (Config, []byte, []byte, error) { + parts := strings.Split(encodedHash, "$") + if len(parts) != 6 || parts[0] != "" || parts[1] != "argon2id" || parts[2] != fmt.Sprintf("v=%d", argon2.Version) { + return Config{}, nil, nil, ErrInvalidHash + } + + parameters := strings.Split(parts[3], ",") + if len(parameters) != 3 { + return Config{}, nil, nil, ErrInvalidHash + } + memory, err := parameter(parameters[0], "m=", 32) + if err != nil { + return Config{}, nil, nil, ErrInvalidHash + } + iterations, err := parameter(parameters[1], "t=", 32) + if err != nil { + return Config{}, nil, nil, ErrInvalidHash + } + parallelism, err := parameter(parameters[2], "p=", 8) + if err != nil { + return Config{}, nil, nil, ErrInvalidHash + } + + salt, err := base64.RawStdEncoding.Strict().DecodeString(parts[4]) + if err != nil { + return Config{}, nil, nil, ErrInvalidHash + } + digest, err := base64.RawStdEncoding.Strict().DecodeString(parts[5]) + if err != nil { + return Config{}, nil, nil, ErrInvalidHash + } + config := Config{ + MemoryKiB: uint32(memory), Iterations: uint32(iterations), Parallelism: uint8(parallelism), + SaltLength: uint32(len(salt)), KeyLength: uint32(len(digest)), + } + if err := validateConfig(config); err != nil { + return Config{}, nil, nil, ErrInvalidHash + } + return config, salt, digest, nil +} + +func parameter(value, prefix string, bitSize int) (uint64, error) { + if !strings.HasPrefix(value, prefix) { + return 0, ErrInvalidHash + } + return strconv.ParseUint(strings.TrimPrefix(value, prefix), 10, bitSize) +} diff --git a/internal/password/password_test.go b/internal/password/password_test.go new file mode 100644 index 0000000..8e88875 --- /dev/null +++ b/internal/password/password_test.go @@ -0,0 +1,44 @@ +package password + +import ( + "testing" + + "github.com/stretchr/testify/suite" +) + +type PasswordSuite struct { + suite.Suite + hasher *Hasher +} + +func TestPasswordSuite(t *testing.T) { + suite.Run(t, new(PasswordSuite)) +} + +func (s *PasswordSuite) SetupTest() { + hasher, err := New(testConfig()) + s.Require().NoError(err) + s.hasher = hasher +} + +func (s *PasswordSuite) TestHasher() { + hash, err := s.hasher.Hash("T3st-password-123") + s.Require().NoError(err) + + verified, err := s.hasher.Verify("T3st-password-123", hash) + s.Require().NoError(err) + s.True(verified) + + verified, err = s.hasher.Verify("wrong-password", hash) + s.Require().NoError(err) + s.False(verified) +} + +func (s *PasswordSuite) TestVerifyRejectsMalformedHash() { + _, err := s.hasher.Verify("password", "invalid") + s.Error(err) +} + +func testConfig() Config { + return Config{MemoryKiB: 8 * 1024, Iterations: 1, Parallelism: 1, SaltLength: 16, KeyLength: 32} +} diff --git a/internal/postgres/attachment_jobs.go b/internal/postgres/attachment_jobs.go new file mode 100644 index 0000000..0442e4d --- /dev/null +++ b/internal/postgres/attachment_jobs.go @@ -0,0 +1,575 @@ +package postgres + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "time" + + "github.com/google/uuid" + "github.com/jackc/pgx/v5" + + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model" +) + +const attachmentJobTopic = "attachment.jobs" + +const attachmentJobColumns = ` + id, attachment_id, pipeline_version, attempt, status, available_at, + lease_token, lease_owner, lease_until, error_code, created_at, updated_at` + +func (s *Store) QueueAttachmentWithJobAndOutbox( + ctx context.Context, + attachmentID string, + stagingKey string, + job model.AttachmentJob, + traceparent string, + tracestate string, +) (model.Attachment, error) { + if job.ID == "" || job.PipelineVersion < 1 { + return model.Attachment{}, model.ErrInvalidInput + } + if job.Attempt == 0 { + job.Attempt = 1 + } + if job.Attempt != 1 { + return model.Attachment{}, model.ErrInvalidInput + } + now := time.Now().UTC() + if job.AvailableAt.IsZero() { + job.AvailableAt = now + } + + tx, err := s.pool.BeginTx(ctx, pgx.TxOptions{}) + if err != nil { + return model.Attachment{}, fmt.Errorf("begin queue attachment transaction: %w", err) + } + defer func() { _ = tx.Rollback(context.WithoutCancel(ctx)) }() + + query := ` + UPDATE attachments + SET status = 'queued', pipeline_version = $3, staging_key = $2, + ready_at = NULL, error_code = NULL, updated_at = clock_timestamp() + WHERE id = $1 AND object_key = $2 AND status = 'pending' + RETURNING ` + attachmentColumns + var attachment model.Attachment + err = scanAttachment(tx.QueryRow(ctx, query, attachmentID, stagingKey, job.PipelineVersion), &attachment) + if errors.Is(err, pgx.ErrNoRows) { + return s.alreadyQueuedAttachment(ctx, tx, attachmentID, stagingKey, job) + } + if err != nil { + return model.Attachment{}, fmt.Errorf("queue attachment: %w", err) + } + + job.AttachmentID = attachmentID + job.Status = model.AttachmentJobQueued + if err := insertAttachmentJob(ctx, tx, &job); err != nil { + return model.Attachment{}, err + } + if err := insertAttachmentJobOutbox(ctx, tx, job, traceparent, tracestate); err != nil { + return model.Attachment{}, err + } + if err := tx.Commit(ctx); err != nil { + return model.Attachment{}, fmt.Errorf("commit queued attachment: %w", err) + } + return publicAttachment(attachment), nil +} + +func (s *Store) QueueAttachmentPipelineVersion( + ctx context.Context, + attachmentID string, + job model.AttachmentJob, + traceparent string, + tracestate string, +) (model.Attachment, error) { + if job.ID == "" || job.PipelineVersion < 2 { + return model.Attachment{}, model.ErrInvalidInput + } + job.Attempt = 1 + now := time.Now().UTC() + job.AvailableAt = now + + tx, err := s.pool.BeginTx(ctx, pgx.TxOptions{}) + if err != nil { + return model.Attachment{}, fmt.Errorf("begin queue attachment pipeline version: %w", err) + } + defer func() { _ = tx.Rollback(context.WithoutCancel(ctx)) }() + + const selectAttachment = `SELECT ` + attachmentColumns + ` FROM attachments WHERE id = $1 FOR UPDATE` + var attachment model.Attachment + if err := scanAttachment(tx.QueryRow(ctx, selectAttachment, attachmentID), &attachment); err != nil { + if errors.Is(err, pgx.ErrNoRows) { + return model.Attachment{}, ErrNotFound + } + return model.Attachment{}, fmt.Errorf("lock attachment for pipeline update: %w", err) + } + if attachment.Status != model.AttachmentReady || job.PipelineVersion != attachment.PipelineVersion+1 { + return model.Attachment{}, ErrConflict + } + job.AttachmentID = attachment.ID + job.Status = model.AttachmentJobQueued + const queueAttachment = ` + UPDATE attachments + SET status = 'queued', pipeline_version = $2, staging_key = object_key, + ready_at = NULL, error_code = NULL, updated_at = clock_timestamp() + WHERE id = $1 + RETURNING ` + attachmentColumns + if err := scanAttachment(tx.QueryRow(ctx, queueAttachment, attachment.ID, job.PipelineVersion), &attachment); err != nil { + return model.Attachment{}, fmt.Errorf("queue attachment pipeline version: %w", err) + } + if err := insertAttachmentJob(ctx, tx, &job); err != nil { + return model.Attachment{}, err + } + if err := insertAttachmentJobOutbox(ctx, tx, job, traceparent, tracestate); err != nil { + return model.Attachment{}, err + } + if err := tx.Commit(ctx); err != nil { + return model.Attachment{}, fmt.Errorf("commit queued attachment pipeline version: %w", err) + } + return publicAttachment(attachment), nil +} + +func (s *Store) alreadyQueuedAttachment( + ctx context.Context, + tx pgx.Tx, + attachmentID string, + stagingKey string, + job model.AttachmentJob, +) (model.Attachment, error) { + query := ` + SELECT ` + attachmentColumnsWithAlias("a") + ` + FROM attachments a + JOIN attachment_jobs j + ON j.attachment_id = a.id AND j.pipeline_version = a.pipeline_version + WHERE a.id = $1 AND a.staging_key = $2 AND a.status IN ('queued', 'processing', 'ready') + AND j.id = $3 AND j.pipeline_version = $4` + var attachment model.Attachment + if err := scanAttachment(tx.QueryRow(ctx, query, attachmentID, stagingKey, job.ID, job.PipelineVersion), &attachment); err != nil { + if errors.Is(err, pgx.ErrNoRows) { + return model.Attachment{}, ErrConflict + } + return model.Attachment{}, fmt.Errorf("find queued attachment: %w", err) + } + if err := tx.Commit(ctx); err != nil { + return model.Attachment{}, fmt.Errorf("commit repeated queue attachment: %w", err) + } + return publicAttachment(attachment), nil +} + +func insertAttachmentJob(ctx context.Context, tx pgx.Tx, job *model.AttachmentJob) error { + const query = ` + INSERT INTO attachment_jobs ( + id, attachment_id, pipeline_version, attempt, status, available_at, created_at, updated_at + ) + VALUES ($1, $2, $3, $4, 'queued', $5, statement_timestamp(), statement_timestamp()) + RETURNING created_at, updated_at` + if err := tx.QueryRow( + ctx, query, job.ID, job.AttachmentID, job.PipelineVersion, job.Attempt, job.AvailableAt, + ).Scan(&job.CreatedAt, &job.UpdatedAt); err != nil { + return fmt.Errorf("insert attachment job: %w", err) + } + return nil +} + +func insertAttachmentJobOutbox( + ctx context.Context, + tx pgx.Tx, + job model.AttachmentJob, + traceparent string, + tracestate string, +) error { + payload, err := json.Marshal(model.AttachmentJobMessage{ + SchemaVersion: 1, JobID: job.ID, Type: "attachment.process.v1", + AttachmentID: job.AttachmentID, PipelineVersion: job.PipelineVersion, + Attempt: job.Attempt, CreatedAt: job.CreatedAt, + Traceparent: traceparent, Tracestate: tracestate, + }) + if err != nil { + return fmt.Errorf("marshal attachment job event: %w", err) + } + const query = ` + INSERT INTO outbox_events ( + id, topic, partition_key, payload, job_id, attempt, available_at + ) + VALUES ($1, $2, $3, $4, $5, $6, $7)` + if _, err := tx.Exec( + ctx, query, uuid.NewString(), attachmentJobTopic, job.AttachmentID, payload, + job.ID, job.Attempt, job.AvailableAt, + ); err != nil { + return fmt.Errorf("insert attachment job outbox event: %w", err) + } + return nil +} + +func (s *Store) AttachmentJob(ctx context.Context, jobID string) (model.AttachmentJob, error) { + const query = `SELECT ` + attachmentJobColumns + ` FROM attachment_jobs WHERE id = $1` + var job model.AttachmentJob + if err := scanAttachmentJob(s.pool.QueryRow(ctx, query, jobID), &job); err != nil { + if errors.Is(err, pgx.ErrNoRows) { + return model.AttachmentJob{}, ErrNotFound + } + return model.AttachmentJob{}, fmt.Errorf("get attachment job: %w", err) + } + return job, nil +} + +func (s *Store) ClaimAttachmentJob( + ctx context.Context, + jobID string, + attachmentID string, + pipelineVersion int, + attempt int, + workerID string, + leaseDuration time.Duration, +) (model.AttachmentJobLease, error) { + if leaseDuration <= 0 || workerID == "" { + return model.AttachmentJobLease{}, model.ErrInvalidInput + } + tx, err := s.pool.BeginTx(ctx, pgx.TxOptions{}) + if err != nil { + return model.AttachmentJobLease{}, fmt.Errorf("begin claim attachment job: %w", err) + } + defer func() { _ = tx.Rollback(context.WithoutCancel(ctx)) }() + + job, attachment, err := lockAttachmentJob(ctx, tx, jobID) + if err != nil { + return model.AttachmentJobLease{}, err + } + if attachment.ID != attachmentID || job.AttachmentID != attachmentID { + return model.AttachmentJobLease{}, model.ErrInvalidInput + } + if attachment.PipelineVersion != pipelineVersion || job.PipelineVersion != pipelineVersion { + return model.AttachmentJobLease{}, model.ErrStalePipeline + } + if job.Status == model.AttachmentJobReady || attachment.Status == model.AttachmentReady { + return model.AttachmentJobLease{}, model.ErrJobComplete + } + if job.Attempt != attempt { + return model.AttachmentJobLease{}, model.ErrStaleAttempt + } + if job.Status == model.AttachmentJobProcessing { + return model.AttachmentJobLease{}, model.ErrJobBusy + } + if job.Status != model.AttachmentJobQueued { + return model.AttachmentJobLease{}, ErrConflict + } + if job.AvailableAt.After(time.Now()) { + return model.AttachmentJobLease{}, model.ErrJobNotDue + } + + job.LeaseToken = uuid.NewString() + job.LeaseOwner = workerID + leaseUntil := time.Now().UTC().Add(leaseDuration) + job.LeaseUntil = &leaseUntil + job.Status = model.AttachmentJobProcessing + const updateJob = ` + UPDATE attachment_jobs + SET status = 'processing', lease_token = $2, lease_owner = $3, + lease_until = $4, error_code = NULL, updated_at = clock_timestamp() + WHERE id = $1` + if _, err := tx.Exec(ctx, updateJob, job.ID, job.LeaseToken, job.LeaseOwner, leaseUntil); err != nil { + return model.AttachmentJobLease{}, fmt.Errorf("claim attachment job: %w", err) + } + const updateAttachment = ` + UPDATE attachments + SET status = 'processing', error_code = NULL, updated_at = clock_timestamp() + WHERE id = $1 AND pipeline_version = $2 AND status = 'queued'` + result, err := tx.Exec(ctx, updateAttachment, attachment.ID, pipelineVersion) + if err != nil { + return model.AttachmentJobLease{}, fmt.Errorf("mark attachment processing: %w", err) + } + if result.RowsAffected() != 1 { + return model.AttachmentJobLease{}, model.ErrStalePipeline + } + if err := tx.Commit(ctx); err != nil { + return model.AttachmentJobLease{}, fmt.Errorf("commit attachment job claim: %w", err) + } + attachment.Status = model.AttachmentProcessing + attachment.ErrorCode = nil + return model.AttachmentJobLease{Job: job, Attachment: attachment}, nil +} + +func lockAttachmentJob(ctx context.Context, tx pgx.Tx, jobID string) (model.AttachmentJob, model.Attachment, error) { + query := ` + SELECT ` + prefixedAttachmentJobColumns("j") + `, ` + attachmentColumnsWithAlias("a") + ` + FROM attachment_jobs j + JOIN attachments a ON a.id = j.attachment_id + WHERE j.id = $1 + FOR UPDATE OF j, a` + var job model.AttachmentJob + var attachment model.Attachment + var leaseToken, leaseOwner *string + var stagingKey *string + err := tx.QueryRow(ctx, query, jobID).Scan( + &job.ID, &job.AttachmentID, &job.PipelineVersion, &job.Attempt, &job.Status, &job.AvailableAt, + &leaseToken, &leaseOwner, &job.LeaseUntil, &job.ErrorCode, &job.CreatedAt, &job.UpdatedAt, + &attachment.ID, &attachment.ChatID, &attachment.OwnerID, &attachment.Filename, + &attachment.ContentType, &attachment.SizeBytes, &attachment.SHA256, &attachment.ObjectKey, + &attachment.Status, &attachment.CreatedAt, &attachment.UpdatedAt, &attachment.ReadyAt, + &attachment.PipelineVersion, &attachment.ErrorCode, &stagingKey, + ) + if errors.Is(err, pgx.ErrNoRows) { + return model.AttachmentJob{}, model.Attachment{}, ErrNotFound + } + if err != nil { + return model.AttachmentJob{}, model.Attachment{}, fmt.Errorf("lock attachment job: %w", err) + } + if leaseToken != nil { + job.LeaseToken = *leaseToken + } + if leaseOwner != nil { + job.LeaseOwner = *leaseOwner + } + if stagingKey != nil { + attachment.StagingKey = *stagingKey + } + return job, attachment, nil +} + +func prefixedAttachmentJobColumns(alias string) string { + return fmt.Sprintf( + "%[1]s.id, %[1]s.attachment_id, %[1]s.pipeline_version, %[1]s.attempt, %[1]s.status, "+ + "%[1]s.available_at, %[1]s.lease_token, %[1]s.lease_owner, %[1]s.lease_until, "+ + "%[1]s.error_code, %[1]s.created_at, %[1]s.updated_at", + alias, + ) +} + +func scanAttachmentJob(row rowScanner, job *model.AttachmentJob) error { + var leaseToken, leaseOwner *string + err := row.Scan( + &job.ID, &job.AttachmentID, &job.PipelineVersion, &job.Attempt, &job.Status, &job.AvailableAt, + &leaseToken, &leaseOwner, &job.LeaseUntil, &job.ErrorCode, &job.CreatedAt, &job.UpdatedAt, + ) + if leaseToken != nil { + job.LeaseToken = *leaseToken + } + if leaseOwner != nil { + job.LeaseOwner = *leaseOwner + } + return err +} + +func (s *Store) CompleteAttachmentJob( + ctx context.Context, + lease model.AttachmentJobLease, + variants []model.Variant, +) (model.JobMutationResult, error) { + if !validCompletedVariants(variants, lease.Job.PipelineVersion) { + return "", model.ErrInvalidInput + } + tx, err := s.pool.BeginTx(ctx, pgx.TxOptions{}) + if err != nil { + return "", fmt.Errorf("begin complete attachment job: %w", err) + } + defer func() { _ = tx.Rollback(context.WithoutCancel(ctx)) }() + + job, attachment, result, err := lockJobMutation(ctx, tx, lease) + if err != nil || result != model.JobMutationApplied { + return result, err + } + const deleteVariants = ` + DELETE FROM attachment_variants + WHERE attachment_id = $1 AND pipeline_version = $2` + if _, err := tx.Exec(ctx, deleteVariants, attachment.ID, job.PipelineVersion); err != nil { + return "", fmt.Errorf("replace attachment variants: %w", err) + } + + var originalKey string + const insertVariant = ` + INSERT INTO attachment_variants ( + attachment_id, pipeline_version, name, content_type, size_bytes, + sha256, width, height, object_key + ) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)` + for _, variant := range variants { + if variant.Name == "original" { + originalKey = variant.ObjectKey + } + if _, err := tx.Exec( + ctx, insertVariant, attachment.ID, job.PipelineVersion, variant.Name, + variant.ContentType, variant.SizeBytes, variant.SHA256, variant.Width, + variant.Height, variant.ObjectKey, + ); err != nil { + return "", fmt.Errorf("insert attachment variant %s: %w", variant.Name, err) + } + } + + const completeJob = ` + UPDATE attachment_jobs + SET status = 'ready', lease_token = NULL, lease_owner = NULL, lease_until = NULL, + error_code = NULL, updated_at = clock_timestamp() + WHERE id = $1 AND lease_token = $2` + if _, err := tx.Exec(ctx, completeJob, job.ID, lease.Job.LeaseToken); err != nil { + return "", fmt.Errorf("complete attachment job: %w", err) + } + const completeAttachment = ` + UPDATE attachments + SET status = 'ready', object_key = $3, ready_at = clock_timestamp(), + staging_key = NULL, error_code = NULL, updated_at = clock_timestamp() + WHERE id = $1 AND pipeline_version = $2` + if _, err := tx.Exec(ctx, completeAttachment, attachment.ID, job.PipelineVersion, originalKey); err != nil { + return "", fmt.Errorf("complete attachment: %w", err) + } + if err := insertAttachmentTerminalEvent(ctx, tx, job, model.AttachmentJobReady); err != nil { + return "", err + } + if err := tx.Commit(ctx); err != nil { + return "", fmt.Errorf("commit completed attachment job: %w", err) + } + return model.JobMutationApplied, nil +} + +func validCompletedVariants(variants []model.Variant, pipelineVersion int) bool { + if len(variants) != 4 || pipelineVersion < 1 { + return false + } + required := map[string]struct{}{ + "original": {}, "thumbnail": {}, "crop": {}, "optimized": {}, + } + for _, variant := range variants { + if variant.ObjectKey == "" || variant.PipelineVersion != pipelineVersion { + return false + } + if _, exists := required[variant.Name]; !exists { + return false + } + delete(required, variant.Name) + } + return len(required) == 0 +} + +func (s *Store) RetryAttachmentJob( + ctx context.Context, + lease model.AttachmentJobLease, + errorCode string, + nextAttemptAt time.Time, + traceparent string, + tracestate string, +) (model.JobMutationResult, error) { + if !validErrorCode(errorCode) || nextAttemptAt.IsZero() { + return "", model.ErrInvalidInput + } + tx, err := s.pool.BeginTx(ctx, pgx.TxOptions{}) + if err != nil { + return "", fmt.Errorf("begin retry attachment job: %w", err) + } + defer func() { _ = tx.Rollback(context.WithoutCancel(ctx)) }() + + job, attachment, result, err := lockJobMutation(ctx, tx, lease) + if err != nil || result != model.JobMutationApplied { + return result, err + } + job.Attempt++ + job.Status = model.AttachmentJobQueued + job.AvailableAt = nextAttemptAt.UTC() + job.ErrorCode = &errorCode + const retryJob = ` + UPDATE attachment_jobs + SET status = 'queued', attempt = $3, available_at = $4, + lease_token = NULL, lease_owner = NULL, lease_until = NULL, + error_code = $5, updated_at = clock_timestamp() + WHERE id = $1 AND lease_token = $2` + if _, err := tx.Exec( + ctx, retryJob, job.ID, lease.Job.LeaseToken, job.Attempt, job.AvailableAt, errorCode, + ); err != nil { + return "", fmt.Errorf("retry attachment job: %w", err) + } + const queueAttachment = ` + UPDATE attachments + SET status = 'queued', error_code = NULL, updated_at = clock_timestamp() + WHERE id = $1 AND pipeline_version = $2` + if _, err := tx.Exec(ctx, queueAttachment, attachment.ID, job.PipelineVersion); err != nil { + return "", fmt.Errorf("requeue attachment: %w", err) + } + if err := insertAttachmentJobOutbox(ctx, tx, job, traceparent, tracestate); err != nil { + return "", err + } + if err := tx.Commit(ctx); err != nil { + return "", fmt.Errorf("commit attachment job retry: %w", err) + } + return model.JobMutationApplied, nil +} + +func (s *Store) FailAttachmentJob( + ctx context.Context, + lease model.AttachmentJobLease, + errorCode string, +) (model.JobMutationResult, error) { + if !validErrorCode(errorCode) { + return "", model.ErrInvalidInput + } + tx, err := s.pool.BeginTx(ctx, pgx.TxOptions{}) + if err != nil { + return "", fmt.Errorf("begin fail attachment job: %w", err) + } + defer func() { _ = tx.Rollback(context.WithoutCancel(ctx)) }() + + job, attachment, result, err := lockJobMutation(ctx, tx, lease) + if err != nil || result != model.JobMutationApplied { + return result, err + } + const failJob = ` + UPDATE attachment_jobs + SET status = 'failed', lease_token = NULL, lease_owner = NULL, lease_until = NULL, + error_code = $3, updated_at = clock_timestamp() + WHERE id = $1 AND lease_token = $2` + if _, err := tx.Exec(ctx, failJob, job.ID, lease.Job.LeaseToken, errorCode); err != nil { + return "", fmt.Errorf("fail attachment job: %w", err) + } + const failAttachment = ` + UPDATE attachments + SET status = 'failed', error_code = $3, ready_at = NULL, updated_at = clock_timestamp() + WHERE id = $1 AND pipeline_version = $2` + if _, err := tx.Exec(ctx, failAttachment, attachment.ID, job.PipelineVersion, errorCode); err != nil { + return "", fmt.Errorf("fail attachment: %w", err) + } + if err := insertAttachmentTerminalEvent(ctx, tx, job, model.AttachmentJobFailed); err != nil { + return "", err + } + if err := tx.Commit(ctx); err != nil { + return "", fmt.Errorf("commit failed attachment job: %w", err) + } + return model.JobMutationApplied, nil +} + +func insertAttachmentTerminalEvent(ctx context.Context, tx pgx.Tx, job model.AttachmentJob, status string) error { + const query = ` + INSERT INTO attachment_terminal_events (job_id, status, accepted_at) + VALUES ($1, $2, $3) + ON CONFLICT (job_id) DO NOTHING` + if _, err := tx.Exec(ctx, query, job.ID, status, job.CreatedAt); err != nil { + return fmt.Errorf("insert attachment terminal event: %w", err) + } + return nil +} + +func lockJobMutation( + ctx context.Context, + tx pgx.Tx, + lease model.AttachmentJobLease, +) (model.AttachmentJob, model.Attachment, model.JobMutationResult, error) { + job, attachment, err := lockAttachmentJob(ctx, tx, lease.Job.ID) + if err != nil { + return model.AttachmentJob{}, model.Attachment{}, "", err + } + if attachment.PipelineVersion != lease.Job.PipelineVersion || job.PipelineVersion != lease.Job.PipelineVersion { + return job, attachment, model.JobMutationStale, nil + } + if job.Status == model.AttachmentJobReady && attachment.Status == model.AttachmentReady { + return job, attachment, model.JobMutationAlreadyComplete, nil + } + if job.PipelineVersion != lease.Job.PipelineVersion || + job.Attempt != lease.Job.Attempt || + job.Status != model.AttachmentJobProcessing || + job.LeaseToken == "" || job.LeaseToken != lease.Job.LeaseToken || + job.LeaseUntil == nil || !job.LeaseUntil.After(time.Now()) { + return job, attachment, model.JobMutationStale, nil + } + return job, attachment, model.JobMutationApplied, nil +} + +func validErrorCode(errorCode string) bool { + return len(errorCode) >= 1 && len(errorCode) <= 64 +} diff --git a/internal/postgres/attachment_lease.go b/internal/postgres/attachment_lease.go new file mode 100644 index 0000000..9e631a9 --- /dev/null +++ b/internal/postgres/attachment_lease.go @@ -0,0 +1,32 @@ +package postgres + +import ( + "context" + "fmt" + "time" + + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model" +) + +func (s *Store) RenewAttachmentJobLease( + ctx context.Context, + lease model.AttachmentJobLease, + duration time.Duration, +) (bool, error) { + if duration <= 0 || lease.Job.ID == "" || lease.Job.LeaseToken == "" || lease.Job.LeaseOwner == "" { + return false, model.ErrInvalidInput + } + const query = ` + UPDATE attachment_jobs + SET lease_until = clock_timestamp() + ($4::bigint * interval '1 microsecond'), + updated_at = clock_timestamp() + WHERE id = $1 AND lease_token = $2 AND lease_owner = $3 + AND status = 'processing' AND lease_until > clock_timestamp()` + result, err := s.pool.Exec( + ctx, query, lease.Job.ID, lease.Job.LeaseToken, lease.Job.LeaseOwner, duration.Microseconds(), + ) + if err != nil { + return false, fmt.Errorf("renew attachment job lease: %w", err) + } + return result.RowsAffected() == 1, nil +} diff --git a/internal/postgres/attachment_recovery.go b/internal/postgres/attachment_recovery.go new file mode 100644 index 0000000..388dafc --- /dev/null +++ b/internal/postgres/attachment_recovery.go @@ -0,0 +1,209 @@ +package postgres + +import ( + "context" + "errors" + "fmt" + "time" + + "github.com/google/uuid" + "github.com/jackc/pgx/v5" + + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model" +) + +func (s *Store) RecoverExpiredAttachmentJobs( + ctx context.Context, + limit int, + maxAttempts int, + retryAt time.Time, +) (model.RecoveryResult, error) { + if limit < 1 || limit > 1000 || maxAttempts < 1 { + return model.RecoveryResult{}, model.ErrInvalidInput + } + if retryAt.IsZero() { + retryAt = time.Now().UTC() + } + tx, err := s.pool.BeginTx(ctx, pgx.TxOptions{}) + if err != nil { + return model.RecoveryResult{}, fmt.Errorf("begin expired job recovery: %w", err) + } + defer func() { _ = tx.Rollback(context.WithoutCancel(ctx)) }() + + const selectExpired = ` + SELECT id + FROM attachment_jobs + WHERE status = 'processing' AND lease_until <= clock_timestamp() + ORDER BY lease_until, id + LIMIT $1 + FOR UPDATE SKIP LOCKED` + rows, err := tx.Query(ctx, selectExpired, limit) + if err != nil { + return model.RecoveryResult{}, fmt.Errorf("select expired attachment jobs: %w", err) + } + jobIDs, err := pgx.CollectRows(rows, pgx.RowTo[string]) + if err != nil { + return model.RecoveryResult{}, fmt.Errorf("scan expired attachment jobs: %w", err) + } + + var recovered model.RecoveryResult + for _, jobID := range jobIDs { + job, attachment, err := lockAttachmentJob(ctx, tx, jobID) + if err != nil { + return model.RecoveryResult{}, err + } + if job.Status != model.AttachmentJobProcessing || job.LeaseUntil == nil || job.LeaseUntil.After(time.Now()) { + continue + } + if job.PipelineVersion != attachment.PipelineVersion { + if err := markExpiredJobFailed(ctx, tx, job, attachment, "stale_pipeline", false); err != nil { + return model.RecoveryResult{}, err + } + recovered.Failed++ + continue + } + if job.Attempt >= maxAttempts { + if err := markExpiredJobFailed(ctx, tx, job, attachment, "retry_exhausted", true); err != nil { + return model.RecoveryResult{}, err + } + recovered.Failed++ + continue + } + + job.Attempt++ + job.Status = model.AttachmentJobQueued + job.AvailableAt = retryAt.UTC() + const requeueJob = ` + UPDATE attachment_jobs + SET status = 'queued', attempt = $2, available_at = $3, + lease_token = NULL, lease_owner = NULL, lease_until = NULL, + error_code = 'lease_expired', updated_at = clock_timestamp() + WHERE id = $1` + if _, err := tx.Exec(ctx, requeueJob, job.ID, job.Attempt, job.AvailableAt); err != nil { + return model.RecoveryResult{}, fmt.Errorf("requeue expired attachment job: %w", err) + } + const requeueAttachment = ` + UPDATE attachments + SET status = 'queued', error_code = NULL, updated_at = clock_timestamp() + WHERE id = $1 AND pipeline_version = $2` + if _, err := tx.Exec(ctx, requeueAttachment, attachment.ID, job.PipelineVersion); err != nil { + return model.RecoveryResult{}, fmt.Errorf("requeue attachment after expired lease: %w", err) + } + if err := insertAttachmentJobOutbox(ctx, tx, job, "", ""); err != nil { + return model.RecoveryResult{}, err + } + recovered.Requeued++ + } + if err := tx.Commit(ctx); err != nil { + return model.RecoveryResult{}, fmt.Errorf("commit expired attachment job recovery: %w", err) + } + return recovered, nil +} + +func markExpiredJobFailed( + ctx context.Context, + tx pgx.Tx, + job model.AttachmentJob, + attachment model.Attachment, + errorCode string, + failAttachment bool, +) error { + const failJob = ` + UPDATE attachment_jobs + SET status = 'failed', lease_token = NULL, lease_owner = NULL, lease_until = NULL, + error_code = $2, updated_at = clock_timestamp() + WHERE id = $1` + if _, err := tx.Exec(ctx, failJob, job.ID, errorCode); err != nil { + return fmt.Errorf("fail expired attachment job: %w", err) + } + if err := insertAttachmentTerminalEvent(ctx, tx, job, model.AttachmentJobFailed); err != nil { + return err + } + if !failAttachment { + return nil + } + const failAttachmentQuery = ` + UPDATE attachments + SET status = 'failed', error_code = $3, ready_at = NULL, updated_at = clock_timestamp() + WHERE id = $1 AND pipeline_version = $2` + if _, err := tx.Exec(ctx, failAttachmentQuery, attachment.ID, job.PipelineVersion, errorCode); err != nil { + return fmt.Errorf("fail attachment after expired lease: %w", err) + } + return nil +} + +func (s *Store) ReplayFailedAttachment( + ctx context.Context, + attachmentID string, + actor string, + reason string, +) (model.AttachmentJob, error) { + if actor == "" || len(actor) > 128 || reason == "" || len(reason) > 512 { + return model.AttachmentJob{}, model.ErrInvalidInput + } + tx, err := s.pool.BeginTx(ctx, pgx.TxOptions{}) + if err != nil { + return model.AttachmentJob{}, fmt.Errorf("begin failed attachment replay: %w", err) + } + defer func() { _ = tx.Rollback(context.WithoutCancel(ctx)) }() + + const selectJob = ` + SELECT j.id + FROM attachment_jobs j + JOIN attachments a ON a.id = j.attachment_id AND a.pipeline_version = j.pipeline_version + WHERE a.id = $1 AND a.status = 'failed' AND j.status = 'failed' + FOR UPDATE OF j, a` + var jobID string + if err := tx.QueryRow(ctx, selectJob, attachmentID).Scan(&jobID); err != nil { + if errors.Is(err, pgx.ErrNoRows) { + return model.AttachmentJob{}, ErrConflict + } + return model.AttachmentJob{}, fmt.Errorf("select failed attachment job: %w", err) + } + job, attachment, err := lockAttachmentJob(ctx, tx, jobID) + if err != nil { + return model.AttachmentJob{}, err + } + previousAttempt := job.Attempt + job.Attempt++ + job.Status = model.AttachmentJobQueued + job.AvailableAt = time.Now().UTC() + job.ErrorCode = nil + job.LeaseToken = "" + job.LeaseOwner = "" + job.LeaseUntil = nil + const replayJob = ` + UPDATE attachment_jobs + SET status = 'queued', attempt = $2, available_at = $3, + lease_token = NULL, lease_owner = NULL, lease_until = NULL, + error_code = NULL, updated_at = clock_timestamp() + WHERE id = $1` + if _, err := tx.Exec(ctx, replayJob, job.ID, job.Attempt, job.AvailableAt); err != nil { + return model.AttachmentJob{}, fmt.Errorf("replay attachment job: %w", err) + } + const replayAttachment = ` + UPDATE attachments + SET status = 'queued', error_code = NULL, updated_at = clock_timestamp() + WHERE id = $1 AND pipeline_version = $2` + if _, err := tx.Exec(ctx, replayAttachment, attachment.ID, job.PipelineVersion); err != nil { + return model.AttachmentJob{}, fmt.Errorf("replay failed attachment: %w", err) + } + const insertAudit = ` + INSERT INTO attachment_replay_audit ( + id, job_id, attachment_id, previous_attempt, new_attempt, actor, reason + ) + VALUES ($1, $2, $3, $4, $5, $6, $7)` + if _, err := tx.Exec( + ctx, insertAudit, uuid.NewString(), job.ID, attachment.ID, + previousAttempt, job.Attempt, actor, reason, + ); err != nil { + return model.AttachmentJob{}, fmt.Errorf("audit attachment replay: %w", err) + } + if err := insertAttachmentJobOutbox(ctx, tx, job, "", ""); err != nil { + return model.AttachmentJob{}, err + } + if err := tx.Commit(ctx); err != nil { + return model.AttachmentJob{}, fmt.Errorf("commit failed attachment replay: %w", err) + } + return job, nil +} diff --git a/internal/postgres/attachments.go b/internal/postgres/attachments.go new file mode 100644 index 0000000..2765eda --- /dev/null +++ b/internal/postgres/attachments.go @@ -0,0 +1,393 @@ +package postgres + +import ( + "context" + "errors" + "fmt" + "time" + + "github.com/jackc/pgx/v5" + + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model" +) + +const attachmentColumns = ` + id, chat_id, owner_id, filename, content_type, size_bytes, sha256, + object_key, status, created_at, updated_at, ready_at, pipeline_version, error_code, staging_key` + +type rowScanner interface { + Scan(...any) error +} + +func (s *Store) CreatePendingAttachment(ctx context.Context, attachment model.Attachment) (model.Attachment, error) { + query := ` + INSERT INTO attachments ( + id, chat_id, owner_id, filename, content_type, size_bytes, sha256, object_key, status, ready_at + ) + SELECT $1, $2, $3, $4, $5, $6, $7, $8, 'pending', NULL + WHERE EXISTS ( + SELECT 1 FROM chat_members WHERE chat_id = $2 AND user_id = $3 + ) + RETURNING ` + attachmentColumns + + err := scanAttachment(s.pool.QueryRow( + ctx, query, attachment.ID, attachment.ChatID, attachment.OwnerID, attachment.Filename, + attachment.ContentType, attachment.SizeBytes, attachment.SHA256, attachment.ObjectKey, + ), &attachment) + if errors.Is(err, pgx.ErrNoRows) { + return model.Attachment{}, ErrNotFound + } + if err != nil { + return model.Attachment{}, fmt.Errorf("create pending attachment: %w", err) + } + return publicAttachment(attachment), nil +} + +func (s *Store) MarkAttachmentReady(ctx context.Context, attachmentID, objectKey string) (model.Attachment, error) { + query := ` + UPDATE attachments + SET status = 'ready', ready_at = clock_timestamp(), updated_at = clock_timestamp() + WHERE id = $1 AND object_key = $2 AND status = 'pending' + RETURNING ` + attachmentColumns + + var attachment model.Attachment + err := scanAttachment(s.pool.QueryRow(ctx, query, attachmentID, objectKey), &attachment) + if errors.Is(err, pgx.ErrNoRows) { + return model.Attachment{}, ErrNotFound + } + if err != nil { + return model.Attachment{}, fmt.Errorf("mark attachment ready: %w", err) + } + return publicAttachment(attachment), nil +} + +func (s *Store) DeletePendingAttachment(ctx context.Context, attachmentID, objectKey string) error { + const query = `DELETE FROM attachments WHERE id = $1 AND object_key = $2 AND status = 'pending'` + if _, err := s.pool.Exec(ctx, query, attachmentID, objectKey); err != nil { + return fmt.Errorf("delete pending attachment: %w", err) + } + return nil +} + +func (s *Store) AttachmentForUser(ctx context.Context, attachmentID, userID string) (model.Attachment, error) { + query := ` + SELECT ` + attachmentColumnsWithAlias("a") + ` + FROM attachments a + JOIN chat_members member ON member.chat_id = a.chat_id AND member.user_id = $2 + WHERE a.id = $1 AND a.status IN ('queued', 'processing', 'ready', 'failed')` + + var attachment model.Attachment + err := scanAttachment(s.pool.QueryRow(ctx, query, attachmentID, userID), &attachment) + if errors.Is(err, pgx.ErrNoRows) { + return model.Attachment{}, ErrNotFound + } + if err != nil { + return model.Attachment{}, fmt.Errorf("get attachment for user: %w", err) + } + if attachment.Status == model.AttachmentReady { + attachment.Variants, err = s.attachmentVariants(ctx, attachment.ID, attachment.PipelineVersion) + if err != nil { + return model.Attachment{}, err + } + } + return publicAttachment(attachment), nil +} + +func (s *Store) AttachmentVariantForUser( + ctx context.Context, + attachmentID string, + userID string, + variantName string, +) (model.Attachment, model.Variant, error) { + attachment, err := s.AttachmentForUser(ctx, attachmentID, userID) + if err != nil { + return model.Attachment{}, model.Variant{}, err + } + if attachment.Status != model.AttachmentReady { + return model.Attachment{}, model.Variant{}, ErrConflict + } + for _, variant := range attachment.Variants { + if variant.Name == variantName { + return attachment, variant, nil + } + } + if variantName == "original" && attachment.ObjectKey != "" { + return attachment, model.Variant{ + Name: "original", ContentType: attachment.ContentType, SizeBytes: attachment.SizeBytes, + SHA256: attachment.SHA256, ObjectKey: attachment.ObjectKey, + PipelineVersion: attachment.PipelineVersion, + }, nil + } + return model.Attachment{}, model.Variant{}, ErrNotFound +} + +func (s *Store) attachmentVariants(ctx context.Context, attachmentID string, pipelineVersion int) ([]model.Variant, error) { + const query = ` + SELECT name, content_type, size_bytes, sha256, width, height, object_key, pipeline_version + FROM attachment_variants + WHERE attachment_id = $1 AND pipeline_version = $2 + ORDER BY CASE name + WHEN 'original' THEN 0 + WHEN 'thumbnail' THEN 1 + WHEN 'crop' THEN 2 + WHEN 'optimized' THEN 3 + ELSE 4 + END` + rows, err := s.pool.Query(ctx, query, attachmentID, pipelineVersion) + if err != nil { + return nil, fmt.Errorf("list attachment variants: %w", err) + } + defer rows.Close() + + variants, err := pgx.CollectRows(rows, pgx.RowToStructByPos[model.Variant]) + if err != nil { + return nil, fmt.Errorf("scan attachment variants: %w", err) + } + return variants, nil +} + +func (s *Store) AttachmentByStorage(ctx context.Context, attachmentID, objectKey string) (model.Attachment, error) { + const query = `SELECT ` + attachmentColumns + ` FROM attachments WHERE id = $1 AND object_key = $2` + + var attachment model.Attachment + err := scanAttachment(s.pool.QueryRow(ctx, query, attachmentID, objectKey), &attachment) + if errors.Is(err, pgx.ErrNoRows) { + return model.Attachment{}, ErrNotFound + } + if err != nil { + return model.Attachment{}, fmt.Errorf("get attachment by storage: %w", err) + } + return publicAttachment(attachment), nil +} + +func (s *Store) AttachmentExistsByObjectKey(ctx context.Context, objectKey string) (bool, error) { + const query = ` + SELECT EXISTS ( + SELECT 1 FROM attachments WHERE object_key = $1 OR staging_key = $1 + UNION ALL + SELECT 1 FROM attachment_variants WHERE object_key = $1 + )` + var exists bool + if err := s.pool.QueryRow(ctx, query, objectKey).Scan(&exists); err != nil { + return false, fmt.Errorf("check attachment by object key: %w", err) + } + return exists, nil +} + +func (s *Store) AttachmentObjectKeys(ctx context.Context, attachmentID string) ([]string, error) { + const query = ` + SELECT object_key + FROM ( + SELECT object_key + FROM attachments + WHERE id = $1 AND object_key <> '' + UNION + SELECT object_key + FROM attachment_variants + WHERE attachment_id = $1 + ) objects + ORDER BY object_key` + rows, err := s.pool.Query(ctx, query, attachmentID) + if err != nil { + return nil, fmt.Errorf("list attachment object keys: %w", err) + } + defer rows.Close() + + keys, err := pgx.CollectRows(rows, pgx.RowTo[string]) + if err != nil { + return nil, fmt.Errorf("scan attachment object keys: %w", err) + } + return keys, nil +} + +func (s *Store) ListAttachments(ctx context.Context, afterObjectKey string, limit int) ([]model.Attachment, string, error) { + if limit < 1 || limit > 1000 { + return nil, "", fmt.Errorf("attachment list limit must be between 1 and 1000") + } + query := ` + SELECT ` + attachmentColumnsWithAlias("a") + `, + EXISTS (SELECT 1 FROM message_attachments ma WHERE ma.attachment_id = a.id), + objects.object_key + FROM ( + SELECT id AS attachment_id, object_key + FROM attachments + WHERE object_key <> '' + UNION + SELECT attachment_id, object_key + FROM attachment_variants + ) objects + JOIN attachments a ON a.id = objects.attachment_id + WHERE objects.object_key > $1 + ORDER BY objects.object_key + LIMIT $2` + rows, err := s.pool.Query(ctx, query, afterObjectKey, limit+1) + if err != nil { + return nil, "", fmt.Errorf("list attachments: %w", err) + } + defer rows.Close() + + attachments, err := pgx.CollectRows(rows, pgx.RowToFunc[model.Attachment](func(row pgx.CollectableRow) (model.Attachment, error) { + var attachment model.Attachment + var objectKey string + err := scanAttachmentWithReferenceAndObjectKey(row, &attachment, &objectKey) + attachment.ObjectKey = objectKey + return publicAttachment(attachment), err + })) + if err != nil { + return nil, "", fmt.Errorf("scan attachments: %w", err) + } + var next string + if len(attachments) > limit { + attachments = attachments[:limit] + next = attachments[len(attachments)-1].ObjectKey + } + return attachments, next, nil +} + +func scanAttachmentWithReferenceAndObjectKey( + row rowScanner, + attachment *model.Attachment, + objectKey *string, +) error { + var stagingKey *string + err := row.Scan( + &attachment.ID, &attachment.ChatID, &attachment.OwnerID, &attachment.Filename, + &attachment.ContentType, &attachment.SizeBytes, &attachment.SHA256, &attachment.ObjectKey, + &attachment.Status, &attachment.CreatedAt, &attachment.UpdatedAt, &attachment.ReadyAt, + &attachment.PipelineVersion, &attachment.ErrorCode, &stagingKey, + &attachment.Referenced, objectKey, + ) + if stagingKey != nil { + attachment.StagingKey = *stagingKey + } + return err +} + +func scanAttachmentWithReference(row rowScanner, attachment *model.Attachment) error { + var stagingKey *string + err := row.Scan( + &attachment.ID, &attachment.ChatID, &attachment.OwnerID, &attachment.Filename, + &attachment.ContentType, &attachment.SizeBytes, &attachment.SHA256, &attachment.ObjectKey, + &attachment.Status, &attachment.CreatedAt, &attachment.UpdatedAt, &attachment.ReadyAt, + &attachment.PipelineVersion, &attachment.ErrorCode, &stagingKey, + &attachment.Referenced, + ) + if stagingKey != nil { + attachment.StagingKey = *stagingKey + } + return err +} + +func (s *Store) AttachmentReferenced(ctx context.Context, attachmentID string) (bool, error) { + const query = `SELECT EXISTS (SELECT 1 FROM message_attachments WHERE attachment_id = $1)` + var referenced bool + if err := s.pool.QueryRow(ctx, query, attachmentID).Scan(&referenced); err != nil { + return false, fmt.Errorf("check attachment references: %w", err) + } + return referenced, nil +} + +func (s *Store) ClaimAttachmentForDeletion(ctx context.Context, candidate model.Attachment, createdBefore time.Time) (bool, error) { + tx, err := s.pool.BeginTx(ctx, pgx.TxOptions{}) + if err != nil { + return false, fmt.Errorf("begin attachment deletion claim: %w", err) + } + defer func() { _ = tx.Rollback(context.WithoutCancel(ctx)) }() + + var status string + var createdAt time.Time + var readyAt *time.Time + err = tx.QueryRow( + ctx, `SELECT status, created_at, ready_at FROM attachments WHERE id = $1 FOR UPDATE`, candidate.ID, + ).Scan(&status, &createdAt, &readyAt) + if errors.Is(err, pgx.ErrNoRows) { + return false, nil + } + if err != nil { + return false, fmt.Errorf("lock attachment deletion candidate: %w", err) + } + if status != candidate.Status { + return false, nil + } + var age time.Time + switch status { + case model.AttachmentPending: + age = createdAt + case model.AttachmentReady: + if readyAt == nil { + return false, nil + } + age = *readyAt + default: + return false, nil + } + if age.After(createdBefore) { + return false, nil + } + var referenced bool + if err := tx.QueryRow(ctx, `SELECT EXISTS (SELECT 1 FROM message_attachments WHERE attachment_id = $1)`, candidate.ID).Scan(&referenced); err != nil { + return false, fmt.Errorf("recheck attachment references: %w", err) + } + if referenced { + return false, nil + } + if _, err := tx.Exec( + ctx, + `UPDATE attachments SET status = 'deleting', updated_at = clock_timestamp() WHERE id = $1 AND status = $2`, + candidate.ID, candidate.Status, + ); err != nil { + return false, fmt.Errorf("mark attachment deleting: %w", err) + } + if err := tx.Commit(ctx); err != nil { + return false, fmt.Errorf("commit attachment deletion claim: %w", err) + } + return true, nil +} + +func (s *Store) DeleteAttachmentIfDeleting(ctx context.Context, attachmentID string) (bool, error) { + const query = ` + DELETE FROM attachments a + WHERE a.id = $1 + AND a.status = 'deleting' + AND NOT EXISTS (SELECT 1 FROM message_attachments ma WHERE ma.attachment_id = a.id)` + result, err := s.pool.Exec(ctx, query, attachmentID) + if err != nil { + return false, fmt.Errorf("delete claimed attachment: %w", err) + } + return result.RowsAffected() == 1, nil +} + +func scanAttachment(row rowScanner, attachment *model.Attachment) error { + var stagingKey *string + err := row.Scan( + &attachment.ID, &attachment.ChatID, &attachment.OwnerID, &attachment.Filename, + &attachment.ContentType, &attachment.SizeBytes, &attachment.SHA256, &attachment.ObjectKey, + &attachment.Status, &attachment.CreatedAt, &attachment.UpdatedAt, &attachment.ReadyAt, + &attachment.PipelineVersion, &attachment.ErrorCode, &stagingKey, + ) + if stagingKey != nil { + attachment.StagingKey = *stagingKey + } + return err +} + +func attachmentColumnsWithAlias(alias string) string { + return fmt.Sprintf( + "%[1]s.id, %[1]s.chat_id, %[1]s.owner_id, %[1]s.filename, %[1]s.content_type, "+ + "%[1]s.size_bytes, %[1]s.sha256, %[1]s.object_key, %[1]s.status, "+ + "%[1]s.created_at, %[1]s.updated_at, %[1]s.ready_at, %[1]s.pipeline_version, "+ + "%[1]s.error_code, %[1]s.staging_key", + alias, + ) +} + +func publicAttachment(attachment model.Attachment) model.Attachment { + variants := make([]model.Variant, 0, len(attachment.Variants)) + for _, variant := range attachment.Variants { + if variant.Name != "original" { + variants = append(variants, variant) + } + } + attachment.Variants = variants + return attachment +} diff --git a/internal/postgres/errors.go b/internal/postgres/errors.go new file mode 100644 index 0000000..39509ad --- /dev/null +++ b/internal/postgres/errors.go @@ -0,0 +1,17 @@ +package postgres + +import "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model" + +var ( + ErrConflict = model.ErrConflict + ErrNotFound = model.ErrNotFound + ErrInvalidMembers = model.ErrInvalidMembers + ErrInvalidCursor = model.ErrInvalidCursor + ErrIdempotencyConflict = model.ErrIdempotencyConflict + ErrStaleLease = model.ErrStaleLease + ErrStaleAttempt = model.ErrStaleAttempt + ErrStalePipeline = model.ErrStalePipeline + ErrJobBusy = model.ErrJobBusy + ErrJobComplete = model.ErrJobComplete + ErrJobNotDue = model.ErrJobNotDue +) diff --git a/internal/postgres/idempotency.go b/internal/postgres/idempotency.go new file mode 100644 index 0000000..4bbb95b --- /dev/null +++ b/internal/postgres/idempotency.go @@ -0,0 +1,374 @@ +package postgres + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/json" + "errors" + "fmt" + "time" + + "github.com/google/uuid" + "github.com/jackc/pgx/v5" + + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model" +) + +const expiredIdempotencyCleanupBatch = 100 + +func (s *Store) CreateMessage( + ctx context.Context, + chatID string, + senderID string, + text string, + attachmentIDs []string, + idempotencyKey string, + idempotencyTTL time.Duration, +) (model.Message, bool, error) { + if attachmentIDs == nil { + attachmentIDs = []string{} + } + var ( + tx pgx.Tx + err error + visible bool + requestHash [sha256.Size]byte + message model.Message + replay bool + ) + + tx, err = s.pool.BeginTx(ctx, pgx.TxOptions{}) + if err != nil { + return model.Message{}, false, fmt.Errorf("begin create message transaction: %w", err) + } + defer func() { _ = tx.Rollback(context.WithoutCancel(ctx)) }() + + visible, err = chatVisibleInTransaction(ctx, tx, chatID, senderID) + if err != nil { + return model.Message{}, false, err + } + if !visible { + return model.Message{}, false, ErrNotFound + } + validAttachments, failedAttachment, err := attachmentsBelongToChat(ctx, tx, chatID, attachmentIDs) + if err != nil { + return model.Message{}, false, err + } + if !validAttachments { + return model.Message{}, false, ErrNotFound + } + if failedAttachment { + return model.Message{}, false, ErrConflict + } + if err = deleteExpiredIdempotencyRecords(ctx, tx, expiredIdempotencyCleanupBatch); err != nil { + return model.Message{}, false, err + } + + requestHash, err = messageRequestHash(text, attachmentIDs) + if err != nil { + return model.Message{}, false, err + } + message, replay, err = createIdempotentMessage( + ctx, tx, chatID, senderID, text, attachmentIDs, idempotencyKey, requestHash[:], idempotencyTTL, + ) + if err != nil { + return model.Message{}, false, err + } + if err := tx.Commit(ctx); err != nil { + return model.Message{}, false, fmt.Errorf("commit create message transaction: %w", err) + } + return message, replay, nil +} + +func messageRequestHash(text string, attachmentIDs []string) ([sha256.Size]byte, error) { + if attachmentIDs == nil { + attachmentIDs = []string{} + } + payload, err := json.Marshal(struct { + Text string `json:"text"` + AttachmentIDs []string `json:"attachment_ids"` + }{text, attachmentIDs}) + if err != nil { + return [sha256.Size]byte{}, fmt.Errorf("marshal message request: %w", err) + } + return sha256.Sum256(payload), nil +} + +func attachmentsBelongToChat(ctx context.Context, tx pgx.Tx, chatID string, attachmentIDs []string) (bool, bool, error) { + if len(attachmentIDs) == 0 { + return true, false, nil + } + const query = ` + SELECT id, status + FROM attachments + WHERE chat_id = $1 AND id::text = ANY($2) + AND status IN ('queued', 'processing', 'ready', 'failed') + FOR UPDATE` + rows, err := tx.Query(ctx, query, chatID, attachmentIDs) + if err != nil { + return false, false, fmt.Errorf("validate message attachments: %w", err) + } + defer rows.Close() + var count int + var failed bool + for rows.Next() { + var id, status string + if err := rows.Scan(&id, &status); err != nil { + return false, false, fmt.Errorf("scan message attachment: %w", err) + } + count++ + failed = failed || status == model.AttachmentFailed + } + if err := rows.Err(); err != nil { + return false, false, fmt.Errorf("validate message attachments: %w", err) + } + return count == len(attachmentIDs), failed, nil +} + +func chatVisibleInTransaction(ctx context.Context, tx pgx.Tx, chatID, userID string) (bool, error) { + const query = ` + SELECT EXISTS ( + SELECT 1 + FROM chat_members + WHERE chat_id = $1 AND user_id = $2 + )` + + var visible bool + if err := tx.QueryRow(ctx, query, chatID, userID).Scan(&visible); err != nil { + return false, fmt.Errorf("check chat visibility: %w", err) + } + return visible, nil +} + +func createIdempotentMessage( + ctx context.Context, + tx pgx.Tx, + chatID string, + senderID string, + text string, + attachmentIDs []string, + idempotencyKey string, + requestHash []byte, + idempotencyTTL time.Duration, +) (model.Message, bool, error) { + for { + var ( + record idempotencyRecord + found bool + err error + messageID string + reserved bool + message model.Message + ) + + record, found, err = lockIdempotencyRecord(ctx, tx, senderID, chatID, idempotencyKey) + if err != nil { + return model.Message{}, false, err + } + if found && record.active { + if !bytes.Equal(record.requestHash, requestHash) { + return model.Message{}, false, ErrIdempotencyConflict + } + message, err = messageByID(ctx, tx, record.messageID) + return message, true, err + } + if found { + if err := deleteIdempotencyRecord(ctx, tx, senderID, chatID, idempotencyKey); err != nil { + return model.Message{}, false, err + } + } + + messageID = uuid.NewString() + reserved, err = reserveIdempotencyKey( + ctx, tx, senderID, chatID, idempotencyKey, requestHash, messageID, idempotencyTTL, + ) + if err != nil { + return model.Message{}, false, err + } + if !reserved { + continue + } + + message, err = insertMessage(ctx, tx, messageID, chatID, senderID, text, attachmentIDs) + return message, false, err + } +} + +func deleteExpiredIdempotencyRecords(ctx context.Context, tx pgx.Tx, limit int) error { + const query = ` + WITH expired AS ( + SELECT ctid + FROM message_idempotency + WHERE expires_at <= statement_timestamp() + ORDER BY expires_at + LIMIT $1 + FOR UPDATE SKIP LOCKED + ) + DELETE FROM message_idempotency records + USING expired + WHERE records.ctid = expired.ctid` + + if _, err := tx.Exec(ctx, query, limit); err != nil { + return fmt.Errorf("delete expired idempotency records: %w", err) + } + return nil +} + +type idempotencyRecord struct { + requestHash []byte + messageID string + active bool +} + +func lockIdempotencyRecord( + ctx context.Context, + tx pgx.Tx, + userID string, + chatID string, + key string, +) (idempotencyRecord, bool, error) { + const query = ` + SELECT request_hash, message_id, expires_at > statement_timestamp() + FROM message_idempotency + WHERE user_id = $1 AND chat_id = $2 AND key = $3 + FOR UPDATE` + + var ( + record idempotencyRecord + err error + ) + err = tx.QueryRow(ctx, query, userID, chatID, key).Scan( + &record.requestHash, + &record.messageID, + &record.active, + ) + if errors.Is(err, pgx.ErrNoRows) { + return idempotencyRecord{}, false, nil + } + if err != nil { + return idempotencyRecord{}, false, fmt.Errorf("lock idempotency key: %w", err) + } + return record, true, nil +} + +func deleteIdempotencyRecord(ctx context.Context, tx pgx.Tx, userID, chatID, key string) error { + const query = ` + DELETE FROM message_idempotency + WHERE user_id = $1 AND chat_id = $2 AND key = $3` + if _, err := tx.Exec(ctx, query, userID, chatID, key); err != nil { + return fmt.Errorf("delete expired idempotency key: %w", err) + } + return nil +} + +func reserveIdempotencyKey( + ctx context.Context, + tx pgx.Tx, + userID string, + chatID string, + key string, + requestHash []byte, + messageID string, + ttl time.Duration, +) (bool, error) { + const query = ` + INSERT INTO message_idempotency (user_id, chat_id, key, request_hash, message_id, expires_at) + VALUES ($1, $2, $3, $4, $5, statement_timestamp() + $6 * INTERVAL '1 second') + ON CONFLICT (user_id, chat_id, key) DO NOTHING + RETURNING true` + + var ( + inserted bool + err error + ) + err = tx.QueryRow(ctx, query, userID, chatID, key, requestHash, messageID, int64(ttl/time.Second)).Scan(&inserted) + if errors.Is(err, pgx.ErrNoRows) { + return false, nil + } + if err != nil { + return false, fmt.Errorf("reserve idempotency key: %w", err) + } + return inserted, nil +} + +func insertMessage( + ctx context.Context, + tx pgx.Tx, + messageID string, + chatID string, + senderID string, + text string, + attachmentIDs []string, +) (model.Message, error) { + const query = ` + INSERT INTO messages (id, chat_id, sender_id, text) + VALUES ($1, $2, $3, $4) + RETURNING id, chat_id, sender_id, text, created_at` + + var ( + message model.Message + err error + ) + err = tx.QueryRow(ctx, query, messageID, chatID, senderID, text).Scan( + &message.ID, + &message.ChatID, + &message.SenderID, + &message.Text, + &message.CreatedAt, + ) + if err != nil { + return model.Message{}, fmt.Errorf("insert message: %w", err) + } + if err := insertMessageAttachments(ctx, tx, messageID, chatID, attachmentIDs); err != nil { + return model.Message{}, err + } + message.AttachmentIDs = make([]string, len(attachmentIDs)) + copy(message.AttachmentIDs, attachmentIDs) + return message, nil +} + +func insertMessageAttachments(ctx context.Context, tx pgx.Tx, messageID, chatID string, attachmentIDs []string) error { + const query = ` + INSERT INTO message_attachments (message_id, chat_id, attachment_id, position) + VALUES ($1, $2, $3, $4)` + for position, attachmentID := range attachmentIDs { + if _, err := tx.Exec(ctx, query, messageID, chatID, attachmentID, position); err != nil { + return fmt.Errorf("insert message attachment: %w", err) + } + } + return nil +} + +func messageByID(ctx context.Context, tx pgx.Tx, messageID string) (model.Message, error) { + const query = ` + SELECT m.id, m.chat_id, m.sender_id, m.text, m.created_at, + ARRAY( + SELECT ma.attachment_id::text + FROM message_attachments ma + WHERE ma.message_id = m.id + ORDER BY ma.position + ) + FROM messages m + WHERE m.id = $1` + + var ( + message model.Message + err error + ) + err = tx.QueryRow(ctx, query, messageID).Scan( + &message.ID, + &message.ChatID, + &message.SenderID, + &message.Text, + &message.CreatedAt, + &message.AttachmentIDs, + ) + if errors.Is(err, pgx.ErrNoRows) { + return model.Message{}, fmt.Errorf("idempotency result message not found") + } + if err != nil { + return model.Message{}, fmt.Errorf("get idempotency result message: %w", err) + } + return message, nil +} diff --git a/internal/postgres/outbox.go b/internal/postgres/outbox.go new file mode 100644 index 0000000..0817f7c --- /dev/null +++ b/internal/postgres/outbox.go @@ -0,0 +1,136 @@ +package postgres + +import ( + "context" + "fmt" + "time" + + "github.com/google/uuid" + "github.com/jackc/pgx/v5" + + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model" +) + +func (s *Store) ClaimOutboxEvents( + ctx context.Context, + limit int, + leaseDuration time.Duration, +) ([]model.OutboxEvent, error) { + if limit < 1 || limit > 1000 || leaseDuration <= 0 { + return nil, model.ErrInvalidInput + } + claimToken := uuid.NewString() + query := ` + WITH candidates AS ( + SELECT id + FROM outbox_events + WHERE published_at IS NULL + AND available_at <= clock_timestamp() + AND (claim_until IS NULL OR claim_until <= clock_timestamp()) + ORDER BY available_at, id + LIMIT $1 + FOR UPDATE SKIP LOCKED + ) + UPDATE outbox_events event + SET claim_token = $2, + claim_until = clock_timestamp() + $3 * INTERVAL '1 millisecond', + publish_tries = event.publish_tries + 1 + FROM candidates + WHERE event.id = candidates.id + RETURNING event.id, event.topic, event.partition_key, event.payload, event.job_id, + event.attempt, event.available_at, event.publish_tries, event.claim_token, + event.claim_until, event.last_error_code, event.created_at` + rows, err := s.pool.Query(ctx, query, limit, claimToken, leaseDuration.Milliseconds()) + if err != nil { + return nil, fmt.Errorf("claim outbox events: %w", err) + } + defer rows.Close() + + events, err := pgx.CollectRows(rows, pgx.RowToFunc[model.OutboxEvent](func(row pgx.CollectableRow) (model.OutboxEvent, error) { + var event model.OutboxEvent + err := row.Scan( + &event.ID, &event.Topic, &event.PartitionKey, &event.Payload, &event.JobID, + &event.Attempt, &event.AvailableAt, &event.PublishTries, &event.LeaseToken, + &event.LeaseUntil, &event.LastErrorCode, &event.CreatedAt, + ) + return event, err + })) + if err != nil { + return nil, fmt.Errorf("scan claimed outbox events: %w", err) + } + return events, nil +} + +func (s *Store) MarkOutboxPublished(ctx context.Context, eventID, leaseToken string) (bool, error) { + const query = ` + UPDATE outbox_events + SET published_at = clock_timestamp(), claim_token = NULL, claim_until = NULL, + last_error_code = NULL + WHERE id = $1 AND claim_token = $2 AND published_at IS NULL + AND claim_until > clock_timestamp()` + result, err := s.pool.Exec(ctx, query, eventID, leaseToken) + if err != nil { + return false, fmt.Errorf("mark outbox event published: %w", err) + } + return result.RowsAffected() == 1, nil +} + +func (s *Store) ReleaseOutboxEvent( + ctx context.Context, + eventID string, + leaseToken string, + errorCode string, + nextAttemptAt time.Time, +) (bool, error) { + if !validErrorCode(errorCode) || nextAttemptAt.IsZero() { + return false, model.ErrInvalidInput + } + const query = ` + UPDATE outbox_events + SET available_at = $3, claim_token = NULL, claim_until = NULL, last_error_code = $4 + WHERE id = $1 AND claim_token = $2 AND published_at IS NULL + AND claim_until > clock_timestamp()` + result, err := s.pool.Exec(ctx, query, eventID, leaseToken, nextAttemptAt.UTC(), errorCode) + if err != nil { + return false, fmt.Errorf("release outbox event: %w", err) + } + return result.RowsAffected() == 1, nil +} + +func (s *Store) RecoverExpiredOutboxEvents(ctx context.Context, now time.Time) (int, error) { + if now.IsZero() { + now = time.Now().UTC() + } + const query = ` + UPDATE outbox_events + SET claim_token = NULL, claim_until = NULL + WHERE published_at IS NULL AND claim_until <= $1` + result, err := s.pool.Exec(ctx, query, now.UTC()) + if err != nil { + return 0, fmt.Errorf("recover expired outbox events: %w", err) + } + return int(result.RowsAffected()), nil +} + +func (s *Store) DeletePublishedOutboxEvents(ctx context.Context, publishedBefore time.Time, limit int) (int, error) { + if publishedBefore.IsZero() || limit < 1 || limit > 1000 { + return 0, model.ErrInvalidInput + } + const query = ` + WITH candidates AS ( + SELECT id + FROM outbox_events + WHERE published_at < $1 + ORDER BY published_at, id + LIMIT $2 + FOR UPDATE SKIP LOCKED + ) + DELETE FROM outbox_events event + USING candidates + WHERE event.id = candidates.id` + result, err := s.pool.Exec(ctx, query, publishedBefore.UTC(), limit) + if err != nil { + return 0, fmt.Errorf("delete published outbox events: %w", err) + } + return int(result.RowsAffected()), nil +} diff --git a/internal/postgres/store.go b/internal/postgres/store.go new file mode 100644 index 0000000..44980b7 --- /dev/null +++ b/internal/postgres/store.go @@ -0,0 +1,442 @@ +package postgres + +import ( + "context" + "errors" + "fmt" + + "github.com/google/uuid" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgconn" + "github.com/jackc/pgx/v5/pgxpool" + + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model" +) + +type Store struct { + pool *pgxpool.Pool +} + +func New(pool *pgxpool.Pool) *Store { + return &Store{pool: pool} +} + +func (s *Store) Ping(ctx context.Context) error { + return s.pool.Ping(ctx) +} + +func (s *Store) CreateUser( + ctx context.Context, + username string, + displayName string, + passwordHash string, +) (model.User, error) { + const query = ` + INSERT INTO users (id, username, display_name, password_hash) + VALUES ($1, $2, $3, $4) + RETURNING id, username, display_name, created_at` + + var user model.User + err := s.pool.QueryRow(ctx, query, uuid.NewString(), username, displayName, passwordHash).Scan( + &user.ID, + &user.Username, + &user.DisplayName, + &user.CreatedAt, + ) + if isUniqueViolation(err) { + return model.User{}, ErrConflict + } + if err != nil { + return model.User{}, fmt.Errorf("create user: %w", err) + } + + return user, nil +} + +func (s *Store) UserByID(ctx context.Context, id string) (model.User, error) { + const query = ` + SELECT id, username, display_name, created_at + FROM users + WHERE id = $1` + + var user model.User + err := s.pool.QueryRow(ctx, query, id).Scan( + &user.ID, + &user.Username, + &user.DisplayName, + &user.CreatedAt, + ) + if errors.Is(err, pgx.ErrNoRows) { + return model.User{}, ErrNotFound + } + if err != nil { + return model.User{}, fmt.Errorf("get user by id: %w", err) + } + + return user, nil +} + +func (s *Store) UserByUsername(ctx context.Context, username string) (model.UserCredentials, error) { + const query = ` + SELECT id, username, display_name, created_at, password_hash + FROM users + WHERE username = $1` + + var credentials model.UserCredentials + err := s.pool.QueryRow(ctx, query, username).Scan( + &credentials.ID, + &credentials.Username, + &credentials.DisplayName, + &credentials.CreatedAt, + &credentials.PasswordHash, + ) + if errors.Is(err, pgx.ErrNoRows) { + return model.UserCredentials{}, ErrNotFound + } + if err != nil { + return model.UserCredentials{}, fmt.Errorf("get user by username: %w", err) + } + + return credentials, nil +} + +func (s *Store) CreateChat( + ctx context.Context, + title string, + memberIDs []string, + callerID string, +) (model.Chat, error) { + if !validMembers(memberIDs, callerID) { + return model.Chat{}, ErrInvalidMembers + } + + tx, err := s.pool.BeginTx(ctx, pgx.TxOptions{}) + if err != nil { + return model.Chat{}, fmt.Errorf("begin create chat transaction: %w", err) + } + defer func() { _ = tx.Rollback(context.WithoutCancel(ctx)) }() + + valid, err := allUsersExist(ctx, tx, memberIDs) + if err != nil { + return model.Chat{}, err + } + if !valid { + return model.Chat{}, ErrInvalidMembers + } + + chat, err := insertChat(ctx, tx, title) + if err != nil { + return model.Chat{}, err + } + if err := insertMembers(ctx, tx, chat.ID, memberIDs); err != nil { + return model.Chat{}, err + } + if err := tx.Commit(ctx); err != nil { + return model.Chat{}, fmt.Errorf("commit create chat transaction: %w", err) + } + + chat.MemberIDs = append([]string(nil), memberIDs...) + return chat, nil +} + +func validMembers(memberIDs []string, callerID string) bool { + callerUUID, err := uuid.Parse(callerID) + if err != nil { + return false + } + + seen := make(map[uuid.UUID]struct{}, len(memberIDs)) + callerIncluded := false + for _, memberID := range memberIDs { + memberUUID, err := uuid.Parse(memberID) + if err != nil { + return false + } + if _, exists := seen[memberUUID]; exists { + return false + } + + seen[memberUUID] = struct{}{} + callerIncluded = callerIncluded || memberUUID == callerUUID + } + + return callerIncluded +} + +func allUsersExist(ctx context.Context, tx pgx.Tx, memberIDs []string) (bool, error) { + const query = ` + SELECT count(*) + FROM users + WHERE id::text = ANY($1)` + + var count int + if err := tx.QueryRow(ctx, query, memberIDs).Scan(&count); err != nil { + return false, fmt.Errorf("check chat members: %w", err) + } + + return count == len(memberIDs), nil +} + +func insertChat(ctx context.Context, tx pgx.Tx, title string) (model.Chat, error) { + const query = ` + INSERT INTO chats (id, title) + VALUES ($1, $2) + RETURNING id, title, created_at` + + var chat model.Chat + if err := tx.QueryRow(ctx, query, uuid.NewString(), title).Scan(&chat.ID, &chat.Title, &chat.CreatedAt); err != nil { + return model.Chat{}, fmt.Errorf("insert chat: %w", err) + } + + return chat, nil +} + +func insertMembers(ctx context.Context, tx pgx.Tx, chatID string, memberIDs []string) error { + const query = ` + INSERT INTO chat_members (chat_id, user_id, position) + VALUES ($1, $2, $3)` + + batch := &pgx.Batch{} + for position, memberID := range memberIDs { + batch.Queue(query, chatID, memberID, position) + } + + results := tx.SendBatch(ctx, batch) + defer func() { _ = results.Close() }() + for range memberIDs { + if _, err := results.Exec(); err != nil { + return fmt.Errorf("insert chat member: %w", err) + } + } + if err := results.Close(); err != nil { + return fmt.Errorf("close chat members batch: %w", err) + } + + return nil +} + +func (s *Store) ChatForUser(ctx context.Context, chatID, userID string) (model.Chat, error) { + const query = ` + SELECT c.id, c.title, c.created_at, + array_agg(cm.user_id::text ORDER BY cm.position) + FROM chats c + JOIN chat_members cm ON cm.chat_id = c.id + WHERE c.id = $1 + AND EXISTS ( + SELECT 1 + FROM chat_members visible + WHERE visible.chat_id = c.id AND visible.user_id = $2 + ) + GROUP BY c.id, c.title, c.created_at` + + var chat model.Chat + err := s.pool.QueryRow(ctx, query, chatID, userID).Scan( + &chat.ID, + &chat.Title, + &chat.CreatedAt, + &chat.MemberIDs, + ) + if errors.Is(err, pgx.ErrNoRows) { + return model.Chat{}, ErrNotFound + } + if err != nil { + return model.Chat{}, fmt.Errorf("get chat for user: %w", err) + } + + return chat, nil +} + +func (s *Store) ListChats( + ctx context.Context, + userID string, + after *model.Position, + limit int, +) ([]model.Chat, bool, error) { + if after != nil { + valid, err := s.visibleChatPositionExists(ctx, userID, *after) + if err != nil { + return nil, false, err + } + if !valid { + return nil, false, ErrInvalidCursor + } + } + + const query = ` + SELECT c.id, c.title, c.created_at, + array_agg(all_members.user_id::text ORDER BY all_members.position) + FROM chats c + JOIN chat_members visible ON visible.chat_id = c.id AND visible.user_id = $1 + JOIN chat_members all_members ON all_members.chat_id = c.id + WHERE ($2::timestamptz IS NULL OR (c.created_at, c.id) > ($2, $3::uuid)) + GROUP BY c.id, c.title, c.created_at + ORDER BY c.created_at, c.id + LIMIT $4` + + var afterTime any + var afterID any + if after != nil { + afterTime = after.CreatedAt + afterID = after.ID + } + + rows, err := s.pool.Query(ctx, query, userID, afterTime, afterID, limit+1) + if err != nil { + return nil, false, fmt.Errorf("list chats: %w", err) + } + defer rows.Close() + + chats, err := pgx.CollectRows(rows, pgx.RowToFunc[model.Chat](func(row pgx.CollectableRow) (model.Chat, error) { + var chat model.Chat + err := row.Scan(&chat.ID, &chat.Title, &chat.CreatedAt, &chat.MemberIDs) + return chat, err + })) + if err != nil { + return nil, false, fmt.Errorf("scan chats: %w", err) + } + + return page(chats, limit) +} + +func (s *Store) visibleChatPositionExists( + ctx context.Context, + userID string, + position model.Position, +) (bool, error) { + const query = ` + SELECT EXISTS ( + SELECT 1 + FROM chats c + JOIN chat_members cm ON cm.chat_id = c.id + WHERE cm.user_id = $1 AND c.created_at = $2 AND c.id = $3 + )` + + var exists bool + if err := s.pool.QueryRow(ctx, query, userID, position.CreatedAt, position.ID).Scan(&exists); err != nil { + return false, fmt.Errorf("validate chat cursor: %w", err) + } + + return exists, nil +} + +func (s *Store) ListMessages( + ctx context.Context, + chatID string, + userID string, + after *model.Position, + limit int, +) ([]model.Message, bool, error) { + visible, err := s.chatVisible(ctx, chatID, userID) + if err != nil { + return nil, false, err + } + if !visible { + return nil, false, ErrNotFound + } + if after != nil { + valid, err := s.messagePositionExists(ctx, chatID, *after) + if err != nil { + return nil, false, err + } + if !valid { + return nil, false, ErrInvalidCursor + } + } + + const query = ` + SELECT m.id, m.chat_id, m.sender_id, m.text, m.created_at, + ARRAY( + SELECT ma.attachment_id::text + FROM message_attachments ma + WHERE ma.message_id = m.id + ORDER BY ma.position + ) + FROM messages m + WHERE m.chat_id = $1 + AND ($2::timestamptz IS NULL OR (m.created_at, m.id) > ($2, $3::uuid)) + ORDER BY m.created_at, m.id + LIMIT $4` + + var afterTime any + var afterID any + if after != nil { + afterTime = after.CreatedAt + afterID = after.ID + } + + rows, err := s.pool.Query(ctx, query, chatID, afterTime, afterID, limit+1) + if err != nil { + return nil, false, fmt.Errorf("list messages: %w", err) + } + defer rows.Close() + + messages, err := pgx.CollectRows(rows, pgx.RowToFunc[model.Message](func(row pgx.CollectableRow) (model.Message, error) { + var message model.Message + err := row.Scan( + &message.ID, + &message.ChatID, + &message.SenderID, + &message.Text, + &message.CreatedAt, + &message.AttachmentIDs, + ) + return message, err + })) + if err != nil { + return nil, false, fmt.Errorf("scan messages: %w", err) + } + + return page(messages, limit) +} + +func (s *Store) chatVisible(ctx context.Context, chatID, userID string) (bool, error) { + const query = ` + SELECT EXISTS ( + SELECT 1 + FROM chat_members + WHERE chat_id = $1 AND user_id = $2 + )` + + var exists bool + if err := s.pool.QueryRow(ctx, query, chatID, userID).Scan(&exists); err != nil { + return false, fmt.Errorf("check chat visibility: %w", err) + } + + return exists, nil +} + +func (s *Store) messagePositionExists( + ctx context.Context, + chatID string, + position model.Position, +) (bool, error) { + const query = ` + SELECT EXISTS ( + SELECT 1 + FROM messages + WHERE chat_id = $1 AND created_at = $2 AND id = $3 + )` + + var exists bool + if err := s.pool.QueryRow(ctx, query, chatID, position.CreatedAt, position.ID).Scan(&exists); err != nil { + return false, fmt.Errorf("validate message cursor: %w", err) + } + + return exists, nil +} + +func page[T any](items []T, limit int) ([]T, bool, error) { + if limit < 1 { + return nil, false, fmt.Errorf("limit must be positive") + } + if len(items) <= limit { + return items, false, nil + } + + return items[:limit], true, nil +} + +func isUniqueViolation(err error) bool { + var postgresError *pgconn.PgError + return errors.As(err, &postgresError) && postgresError.Code == "23505" +} diff --git a/internal/postgres/store_integration_test.go b/internal/postgres/store_integration_test.go new file mode 100644 index 0000000..e71ab6b --- /dev/null +++ b/internal/postgres/store_integration_test.go @@ -0,0 +1,925 @@ +package postgres_test + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "os" + "strings" + "sync" + "testing" + "time" + + "github.com/google/uuid" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" + "github.com/stretchr/testify/suite" + + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/database" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/postgres" + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/reconcile" + "gitea.privcloud.tech/dizi/mtusi2026lab6/migrations" +) + +type PostgresSuite struct { + suite.Suite + pool *pgxpool.Pool + store *postgres.Store +} + +func TestPostgresSuite(t *testing.T) { + suite.Run(t, new(PostgresSuite)) +} + +func (s *PostgresSuite) SetupSuite() { + s.pool = openTestDatabase(s.T()) + s.store = postgres.New(s.pool) +} + +func (s *PostgresSuite) TearDownSuite() { + if s.pool != nil { + s.pool.Close() + } +} + +func (s *PostgresSuite) TestConcurrentUsernameCreation() { + t := s.T() + store := s.store + username := "race_" + strings.ReplaceAll(uuid.NewString(), "-", "")[:16] + + const workers = 8 + results := make(chan error, workers) + var group sync.WaitGroup + for range workers { + group.Add(1) + go func() { + defer group.Done() + _, err := store.CreateUser(context.Background(), username, "Race user", "password-hash") + results <- err + }() + } + group.Wait() + close(results) + + var created, conflicts int + for err := range results { + switch { + case err == nil: + created++ + case errors.Is(err, postgres.ErrConflict): + conflicts++ + default: + t.Fatalf("CreateUser() error = %v", err) + } + } + if created != 1 || conflicts != workers-1 { + t.Fatalf("created = %d, conflicts = %d", created, conflicts) + } +} + +func (s *PostgresSuite) TestCreateChatRollsBackForUnknownMember() { + t := s.T() + pool := s.pool + store := s.store + owner, err := store.CreateUser(context.Background(), uniqueUsername(), "Owner", "password-hash") + if err != nil { + t.Fatalf("CreateUser() error = %v", err) + } + + before := chatCount(t, pool) + _, err = store.CreateChat(context.Background(), "Invalid chat", []string{owner.ID, uuid.NewString()}, owner.ID) + if !errors.Is(err, postgres.ErrInvalidMembers) { + t.Fatalf("CreateChat() error = %v, want ErrInvalidMembers", err) + } + if after := chatCount(t, pool); after != before { + t.Fatalf("chat count changed from %d to %d", before, after) + } +} + +func (s *PostgresSuite) TestMessageIdempotencyReplayAndConflict() { + t := s.T() + pool := s.pool + store := s.store + owner, chatID := createOwnerAndChat(t, store) + key := uuid.NewString() + + first, replay, err := store.CreateMessage(context.Background(), chatID, owner.ID, "same body", nil, key, 24*time.Hour) + if err != nil || replay { + t.Fatalf("first CreateMessage() = (%v, %t, %v)", first, replay, err) + } + second, replay, err := store.CreateMessage(context.Background(), chatID, owner.ID, "same body", []string{}, key, 24*time.Hour) + if err != nil || !replay { + t.Fatalf("replay CreateMessage() = (%v, %t, %v)", second, replay, err) + } + s.Equal(first, second) + if _, _, err := store.CreateMessage(context.Background(), chatID, owner.ID, "different body", nil, key, 24*time.Hour); !errors.Is(err, postgres.ErrIdempotencyConflict) { + t.Fatalf("conflicting CreateMessage() error = %v, want ErrIdempotencyConflict", err) + } + if count := messageCount(t, pool, chatID); count != 1 { + t.Fatalf("message count = %d, want 1", count) + } +} + +func (s *PostgresSuite) TestAttachmentAccessAndMessageLinking() { + owner, chatID := createOwnerAndChat(s.T(), s.store) + outsider, err := s.store.CreateUser(context.Background(), uniqueUsername(), "Outsider", "password-hash") + s.Require().NoError(err) + attachment := model.Attachment{ + ID: uuid.NewString(), ChatID: chatID, OwnerID: owner.ID, Filename: "photo.png", + ContentType: "image/png", SizeBytes: 128, SHA256: strings.Repeat("a", 64), + ObjectKey: "attachments/" + uuid.NewString() + "/original", + } + pending, err := s.store.CreatePendingAttachment(context.Background(), attachment) + s.Require().NoError(err) + created, err := s.store.MarkAttachmentReady(context.Background(), pending.ID, pending.ObjectKey) + s.Require().NoError(err) + s.Equal("ready", created.Status) + s.Empty(created.Variants) + s.Nil(created.ErrorCode) + + visible, err := s.store.AttachmentForUser(context.Background(), created.ID, owner.ID) + s.Require().NoError(err) + s.Equal(created, visible) + _, err = s.store.AttachmentForUser(context.Background(), created.ID, outsider.ID) + s.ErrorIs(err, postgres.ErrNotFound) + + message, replay, err := s.store.CreateMessage( + context.Background(), chatID, owner.ID, "", []string{created.ID}, uuid.NewString(), 24*time.Hour, + ) + s.Require().NoError(err) + s.False(replay) + s.Equal([]string{created.ID}, message.AttachmentIDs) + + messages, _, err := s.store.ListMessages(context.Background(), chatID, owner.ID, nil, 10) + s.Require().NoError(err) + s.Len(messages, 1) + s.Equal([]string{created.ID}, messages[0].AttachmentIDs) + referenced, err := s.store.AttachmentReferenced(context.Background(), created.ID) + s.Require().NoError(err) + s.True(referenced) + deleted, err := s.store.ClaimAttachmentForDeletion(context.Background(), created, time.Now().Add(time.Hour)) + s.Require().NoError(err) + s.False(deleted) +} + +func (s *PostgresSuite) TestAttachmentJobLifecycleAndOutboxFencing() { + owner, chatID := createOwnerAndChat(s.T(), s.store) + attachment, job := createQueuedAttachment(s.T(), s.store, owner.ID, chatID) + s.Equal(model.AttachmentQueued, attachment.Status) + + events, err := s.store.ClaimOutboxEvents(context.Background(), 10, time.Minute) + s.Require().NoError(err) + s.Require().NotEmpty(events) + var event model.OutboxEvent + for _, candidate := range events { + if candidate.JobID == job.ID { + event = candidate + break + } + } + s.Require().NotEmpty(event.ID) + var message model.AttachmentJobMessage + s.Require().NoError(json.Unmarshal(event.Payload, &message)) + s.Equal(job.ID, message.JobID) + s.Equal(attachment.ID, message.AttachmentID) + s.Equal(1, message.Attempt) + published, err := s.store.MarkOutboxPublished(context.Background(), event.ID, event.LeaseToken) + s.Require().NoError(err) + s.True(published) + + _, replay, err := s.store.CreateMessage( + context.Background(), chatID, owner.ID, "queued attachment", + []string{attachment.ID}, uuid.NewString(), 24*time.Hour, + ) + s.Require().NoError(err) + s.False(replay) + + _, err = s.store.ClaimAttachmentJob( + context.Background(), job.ID, uuid.NewString(), job.PipelineVersion, job.Attempt, "worker-invalid", time.Minute, + ) + s.ErrorIs(err, model.ErrInvalidInput) + + lease, err := s.store.ClaimAttachmentJob( + context.Background(), job.ID, attachment.ID, job.PipelineVersion, job.Attempt, "worker-1", time.Minute, + ) + s.Require().NoError(err) + s.Equal(model.AttachmentProcessing, lease.Attachment.Status) + s.NotEmpty(lease.Attachment.StagingKey) + + variants := completedVariants(attachment.ID, 1) + result, err := s.store.CompleteAttachmentJob(context.Background(), lease, variants) + s.Require().NoError(err) + s.Equal(model.JobMutationApplied, result) + s.assertTerminalEvent(job.ID, model.AttachmentJobReady, lease.Job.CreatedAt) + result, err = s.store.CompleteAttachmentJob(context.Background(), lease, variants) + s.Require().NoError(err) + s.Equal(model.JobMutationAlreadyComplete, result) + + metadata, variant, err := s.store.AttachmentVariantForUser( + context.Background(), attachment.ID, owner.ID, "thumbnail", + ) + s.Require().NoError(err) + s.Equal(model.AttachmentReady, metadata.Status) + s.Len(metadata.Variants, 3) + s.Equal(variants[1].ObjectKey, variant.ObjectKey) + listedKeys := make(map[string]struct{}) + var after string + for { + listed, next, err := s.store.ListAttachments(context.Background(), after, 1000) + s.Require().NoError(err) + for _, candidate := range listed { + if candidate.ID == attachment.ID { + listedKeys[candidate.ObjectKey] = struct{}{} + } + } + if next == "" { + break + } + after = next + } + for _, candidate := range variants { + s.Contains(listedKeys, candidate.ObjectKey) + } + + v2 := model.AttachmentJob{ID: uuid.NewString(), PipelineVersion: 2} + queuedV2, err := s.store.QueueAttachmentPipelineVersion( + context.Background(), attachment.ID, v2, "", "", + ) + s.Require().NoError(err) + s.Equal(model.AttachmentQueued, queuedV2.Status) + s.Equal(2, queuedV2.PipelineVersion) + result, err = s.store.CompleteAttachmentJob(context.Background(), lease, variants) + s.Require().NoError(err) + s.Equal(model.JobMutationStale, result) +} + +func (s *PostgresSuite) TestAttachmentJobCreationUsesDatabaseClock() { + owner, chatID := createOwnerAndChat(s.T(), s.store) + attachmentID := uuid.NewString() + stagingKey := "staging/" + attachmentID + "/original" + pending, err := s.store.CreatePendingAttachment(context.Background(), model.Attachment{ + ID: attachmentID, ChatID: chatID, OwnerID: owner.ID, Filename: "future.png", + ContentType: "image/png", SizeBytes: 128, SHA256: strings.Repeat("f", 64), ObjectKey: stagingKey, + }) + s.Require().NoError(err) + future := time.Now().Add(24 * time.Hour) + job := model.AttachmentJob{ + ID: uuid.NewString(), AttachmentID: attachmentID, PipelineVersion: 1, + Attempt: 1, CreatedAt: future, + } + _, err = s.store.QueueAttachmentWithJobAndOutbox(context.Background(), pending.ID, stagingKey, job, "", "") + s.Require().NoError(err) + stored, err := s.store.AttachmentJob(context.Background(), job.ID) + s.Require().NoError(err) + s.True(stored.CreatedAt.Before(future)) + + lease, err := s.store.ClaimAttachmentJob( + context.Background(), job.ID, attachmentID, 1, 1, "worker-clock", time.Minute, + ) + s.Require().NoError(err) + result, err := s.store.CompleteAttachmentJob(context.Background(), lease, completedVariants(attachmentID, 1)) + s.Require().NoError(err) + s.Equal(model.JobMutationApplied, result) + s.assertTerminalEvent(job.ID, model.AttachmentJobReady, stored.CreatedAt) + s.assertOutboxCreationTime(job.ID, 1, stored.CreatedAt) +} + +func (s *PostgresSuite) TestTerminalLedgerMigrationBackfillsExistingJob() { + owner, chatID := createOwnerAndChat(s.T(), s.store) + replayedAttachment, replayedJob := createQueuedAttachment(s.T(), s.store, owner.ID, chatID) + skewedAttachment, skewedJob := createQueuedAttachment(s.T(), s.store, owner.ID, chatID) + replayedAcceptedAt := time.Now().Add(-2 * time.Minute).UTC().Truncate(time.Microsecond) + firstFailureAt := replayedAcceptedAt.Add(20 * time.Second) + replayedReadyAt := firstFailureAt.Add(time.Minute) + skewedTerminalAt := time.Now().Add(-time.Minute).UTC().Truncate(time.Microsecond) + skewedCreatedAt := skewedTerminalAt.Add(time.Minute) + tx, err := s.pool.Begin(context.Background()) + s.Require().NoError(err) + defer func() { s.Require().NoError(tx.Rollback(context.Background())) }() + _, err = tx.Exec( + context.Background(), + `UPDATE attachment_jobs SET status = 'ready', attempt = 2, error_code = NULL, created_at = $2, updated_at = $3 + WHERE id = $1`, + replayedJob.ID, + replayedAcceptedAt, + replayedReadyAt, + ) + s.Require().NoError(err) + _, err = tx.Exec( + context.Background(), + `UPDATE attachments SET status = 'ready', error_code = NULL, ready_at = $2 + WHERE id = $1`, + replayedAttachment.ID, + replayedReadyAt, + ) + s.Require().NoError(err) + _, err = tx.Exec( + context.Background(), + `INSERT INTO attachment_replay_audit ( + id, job_id, attachment_id, previous_attempt, new_attempt, actor, reason, created_at + ) VALUES ($1, $2, $3, 1, 2, 'legacy-operator', 'legacy replay', $4)`, + uuid.NewString(), + replayedJob.ID, + replayedAttachment.ID, + firstFailureAt, + ) + s.Require().NoError(err) + _, err = tx.Exec( + context.Background(), + `UPDATE attachment_jobs SET status = 'failed', error_code = 'legacy_failure', created_at = $2, updated_at = $3 + WHERE id = $1`, + skewedJob.ID, + skewedCreatedAt, + skewedTerminalAt, + ) + s.Require().NoError(err) + _, err = tx.Exec( + context.Background(), + `UPDATE attachments SET status = 'failed', error_code = 'legacy_failure', ready_at = NULL + WHERE id = $1`, + skewedAttachment.ID, + ) + s.Require().NoError(err) + _, err = tx.Exec(context.Background(), `ALTER INDEX attachment_terminal_events_status_terminal_at_idx RENAME TO attachment_terminal_events_status_terminal_at_idx_old`) + s.Require().NoError(err) + _, err = tx.Exec(context.Background(), `ALTER TABLE attachment_terminal_events RENAME TO attachment_terminal_events_old`) + s.Require().NoError(err) + migration, err := migrations.FS.ReadFile("0008_attachment_terminal_events.sql") + s.Require().NoError(err) + _, err = tx.Exec(context.Background(), string(migration), pgx.QueryExecModeSimpleProtocol) + s.Require().NoError(err) + + s.assertTerminalEventInTx(tx, replayedJob.ID, model.AttachmentJobFailed, replayedAcceptedAt, firstFailureAt) + s.assertTerminalEventInTx(tx, skewedJob.ID, model.AttachmentJobFailed, skewedTerminalAt, skewedTerminalAt) +} + +func (s *PostgresSuite) assertTerminalEventInTx( + tx pgx.Tx, + jobID string, + status string, + acceptedAt time.Time, + terminalAt time.Time, +) { + s.T().Helper() + var ( + actualStatus string + actualAcceptedAt time.Time + actualTerminalAt time.Time + ) + err := tx.QueryRow( + context.Background(), + `SELECT status, accepted_at, terminal_at FROM attachment_terminal_events WHERE job_id = $1`, + jobID, + ).Scan(&actualStatus, &actualAcceptedAt, &actualTerminalAt) + s.Require().NoError(err) + s.Equal(status, actualStatus) + s.True(acceptedAt.Equal(actualAcceptedAt)) + s.True(terminalAt.Equal(actualTerminalAt)) +} + +func (s *PostgresSuite) TestPublishedOutboxRetentionIsBatched() { + owner, chatID := createOwnerAndChat(s.T(), s.store) + _, firstJob := createQueuedAttachment(s.T(), s.store, owner.ID, chatID) + _, secondJob := createQueuedAttachment(s.T(), s.store, owner.ID, chatID) + events, err := s.store.ClaimOutboxEvents(context.Background(), 1000, time.Minute) + s.Require().NoError(err) + + eventIDs := make([]string, 0, 2) + for _, event := range events { + if event.JobID != firstJob.ID && event.JobID != secondJob.ID { + continue + } + published, err := s.store.MarkOutboxPublished(context.Background(), event.ID, event.LeaseToken) + s.Require().NoError(err) + s.True(published) + eventIDs = append(eventIDs, event.ID) + } + s.Require().Len(eventIDs, 2) + _, err = s.pool.Exec( + context.Background(), + `UPDATE outbox_events SET published_at = clock_timestamp() - INTERVAL '2 days' WHERE id IN ($1, $2)`, + eventIDs[0], eventIDs[1], + ) + s.Require().NoError(err) + + deleted, err := s.store.DeletePublishedOutboxEvents(context.Background(), time.Now().Add(-24*time.Hour), 1) + s.Require().NoError(err) + s.Equal(1, deleted) + deleted, err = s.store.DeletePublishedOutboxEvents(context.Background(), time.Now().Add(-24*time.Hour), 1) + s.Require().NoError(err) + s.Equal(1, deleted) + + var remaining int + err = s.pool.QueryRow( + context.Background(), `SELECT count(*) FROM outbox_events WHERE id IN ($1, $2)`, eventIDs[0], eventIDs[1], + ).Scan(&remaining) + s.Require().NoError(err) + s.Zero(remaining) +} + +func (s *PostgresSuite) TestFailedAttachmentReplayIsAudited() { + owner, chatID := createOwnerAndChat(s.T(), s.store) + attachment, job := createQueuedAttachment(s.T(), s.store, owner.ID, chatID) + original, err := s.store.AttachmentJob(context.Background(), job.ID) + s.Require().NoError(err) + lease, err := s.store.ClaimAttachmentJob( + context.Background(), job.ID, attachment.ID, job.PipelineVersion, job.Attempt, "worker-1", time.Minute, + ) + s.Require().NoError(err) + result, err := s.store.FailAttachmentJob(context.Background(), lease, "invalid_image") + s.Require().NoError(err) + s.Equal(model.JobMutationApplied, result) + firstTerminalAt := s.assertTerminalEvent(job.ID, model.AttachmentJobFailed, original.CreatedAt) + + _, _, err = s.store.CreateMessage( + context.Background(), chatID, owner.ID, "failed attachment", + []string{attachment.ID}, uuid.NewString(), 24*time.Hour, + ) + s.ErrorIs(err, postgres.ErrConflict) + metadata, err := s.store.AttachmentForUser(context.Background(), attachment.ID, owner.ID) + s.Require().NoError(err) + s.Equal(model.AttachmentFailed, metadata.Status) + s.Require().NotNil(metadata.ErrorCode) + s.Equal("invalid_image", *metadata.ErrorCode) + + replayed, err := s.store.ReplayFailedAttachment( + context.Background(), attachment.ID, "operator@example", "input repaired", + ) + s.Require().NoError(err) + s.Equal(2, replayed.Attempt) + s.Equal(model.AttachmentJobQueued, replayed.Status) + s.Equal(original.CreatedAt, replayed.CreatedAt) + s.assertOutboxCreationTime(job.ID, 2, original.CreatedAt) + replayedLease, err := s.store.ClaimAttachmentJob( + context.Background(), job.ID, attachment.ID, replayed.PipelineVersion, replayed.Attempt, "worker-2", time.Minute, + ) + s.Require().NoError(err) + result, err = s.store.CompleteAttachmentJob( + context.Background(), replayedLease, completedVariants(attachment.ID, replayed.PipelineVersion), + ) + s.Require().NoError(err) + s.Equal(model.JobMutationApplied, result) + s.Equal(firstTerminalAt, s.assertTerminalEvent(job.ID, model.AttachmentJobFailed, original.CreatedAt)) + + var auditCount int + err = s.pool.QueryRow( + context.Background(), `SELECT count(*) FROM attachment_replay_audit WHERE job_id = $1`, job.ID, + ).Scan(&auditCount) + s.Require().NoError(err) + s.Equal(1, auditCount) +} + +func (s *PostgresSuite) TestExpiredLeaseRecoveryFencesOldWorker() { + owner, chatID := createOwnerAndChat(s.T(), s.store) + attachment, job := createQueuedAttachment(s.T(), s.store, owner.ID, chatID) + lease, err := s.store.ClaimAttachmentJob( + context.Background(), job.ID, attachment.ID, job.PipelineVersion, job.Attempt, "worker-1", time.Minute, + ) + s.Require().NoError(err) + original, err := s.store.AttachmentJob(context.Background(), job.ID) + s.Require().NoError(err) + _, err = s.pool.Exec( + context.Background(), + `UPDATE attachment_jobs SET lease_until = clock_timestamp() - INTERVAL '1 second' WHERE id = $1`, + job.ID, + ) + s.Require().NoError(err) + + recovered, err := s.store.RecoverExpiredAttachmentJobs(context.Background(), 10, 5, time.Now()) + s.Require().NoError(err) + s.Equal(1, recovered.Requeued) + s.Zero(recovered.Failed) + current, err := s.store.AttachmentJob(context.Background(), job.ID) + s.Require().NoError(err) + s.Equal(2, current.Attempt) + s.Equal(model.AttachmentJobQueued, current.Status) + s.Equal(original.CreatedAt, current.CreatedAt) + s.assertOutboxCreationTime(job.ID, 2, original.CreatedAt) + + result, err := s.store.CompleteAttachmentJob(context.Background(), lease, completedVariants(attachment.ID, 1)) + s.Require().NoError(err) + s.Equal(model.JobMutationStale, result) +} + +func (s *PostgresSuite) TestExpiredLeaseRecoveryFailsExhaustedJobOnce() { + owner, chatID := createOwnerAndChat(s.T(), s.store) + attachment, job := createQueuedAttachment(s.T(), s.store, owner.ID, chatID) + original, err := s.store.AttachmentJob(context.Background(), job.ID) + s.Require().NoError(err) + _, err = s.store.ClaimAttachmentJob( + context.Background(), job.ID, attachment.ID, job.PipelineVersion, job.Attempt, "worker-1", time.Minute, + ) + s.Require().NoError(err) + _, err = s.pool.Exec( + context.Background(), + `UPDATE attachment_jobs SET lease_until = clock_timestamp() - INTERVAL '1 second' WHERE id = $1`, + job.ID, + ) + s.Require().NoError(err) + + recovered, err := s.store.RecoverExpiredAttachmentJobs(context.Background(), 10, 1, time.Now()) + s.Require().NoError(err) + s.Zero(recovered.Requeued) + s.Equal(1, recovered.Failed) + firstTerminalAt := s.assertTerminalEvent(job.ID, model.AttachmentJobFailed, original.CreatedAt) + + recovered, err = s.store.RecoverExpiredAttachmentJobs(context.Background(), 10, 1, time.Now()) + s.Require().NoError(err) + s.Zero(recovered.Requeued) + s.Zero(recovered.Failed) + current, err := s.store.AttachmentJob(context.Background(), job.ID) + s.Require().NoError(err) + s.Equal(model.AttachmentJobFailed, current.Status) + s.Equal(firstTerminalAt, s.assertTerminalEvent(job.ID, model.AttachmentJobFailed, original.CreatedAt)) +} + +func (s *PostgresSuite) assertTerminalEvent(jobID string, status string, acceptedAt time.Time) time.Time { + s.T().Helper() + var ( + actualStatus string + actualAcceptedAt time.Time + terminalAt time.Time + count int + ) + err := s.pool.QueryRow( + context.Background(), + `SELECT min(status), min(accepted_at), min(terminal_at), count(*) + FROM attachment_terminal_events WHERE job_id = $1`, + jobID, + ).Scan(&actualStatus, &actualAcceptedAt, &terminalAt, &count) + s.Require().NoError(err) + s.Equal(1, count) + s.Equal(status, actualStatus) + s.True(acceptedAt.Equal(actualAcceptedAt), "expected %s, got %s", acceptedAt, actualAcceptedAt) + return terminalAt +} + +func (s *PostgresSuite) assertOutboxCreationTime(jobID string, attempt int, expected time.Time) { + s.T().Helper() + var payload []byte + err := s.pool.QueryRow( + context.Background(), + `SELECT payload FROM outbox_events WHERE job_id = $1 AND attempt = $2`, + jobID, + attempt, + ).Scan(&payload) + s.Require().NoError(err) + var message model.AttachmentJobMessage + s.Require().NoError(json.Unmarshal(payload, &message)) + s.True(expected.Equal(message.CreatedAt), "expected %s, got %s", expected, message.CreatedAt) +} + +func (s *PostgresSuite) TestCompleteAttachmentJobRejectsIncompleteOrMismatchedVariants() { + owner, chatID := createOwnerAndChat(s.T(), s.store) + attachment, job := createQueuedAttachment(s.T(), s.store, owner.ID, chatID) + lease, err := s.store.ClaimAttachmentJob( + context.Background(), job.ID, attachment.ID, job.PipelineVersion, job.Attempt, "worker-1", time.Minute, + ) + s.Require().NoError(err) + + complete := completedVariants(attachment.ID, job.PipelineVersion) + _, err = s.store.CompleteAttachmentJob(context.Background(), lease, complete[:3]) + s.ErrorIs(err, model.ErrInvalidInput) + + duplicate := append([]model.Variant(nil), complete...) + duplicate[3].Name = duplicate[2].Name + _, err = s.store.CompleteAttachmentJob(context.Background(), lease, duplicate) + s.ErrorIs(err, model.ErrInvalidInput) + + mismatched := append([]model.Variant(nil), complete...) + mismatched[3].PipelineVersion++ + _, err = s.store.CompleteAttachmentJob(context.Background(), lease, mismatched) + s.ErrorIs(err, model.ErrInvalidInput) +} + +func (s *PostgresSuite) TestAttachmentDeletionClaimChecksLifecycleUnderLock() { + owner, chatID := createOwnerAndChat(s.T(), s.store) + pending, err := s.store.CreatePendingAttachment(context.Background(), model.Attachment{ + ID: uuid.NewString(), ChatID: chatID, OwnerID: owner.ID, Filename: "pending.png", + ContentType: "image/png", SizeBytes: 128, SHA256: strings.Repeat("c", 64), + ObjectKey: "attachments/" + uuid.NewString() + "/original", + }) + s.Require().NoError(err) + + claimed, err := s.store.ClaimAttachmentForDeletion(context.Background(), pending, time.Now().Add(time.Hour)) + s.Require().NoError(err) + s.True(claimed) + deleted, err := s.store.DeleteAttachmentIfDeleting(context.Background(), pending.ID) + s.Require().NoError(err) + s.True(deleted) + + pending, err = s.store.CreatePendingAttachment(context.Background(), model.Attachment{ + ID: uuid.NewString(), ChatID: chatID, OwnerID: owner.ID, Filename: "completed.png", + ContentType: "image/png", SizeBytes: 128, SHA256: strings.Repeat("d", 64), + ObjectKey: "attachments/" + uuid.NewString() + "/original", + }) + s.Require().NoError(err) + ready, err := s.store.MarkAttachmentReady(context.Background(), pending.ID, pending.ObjectKey) + s.Require().NoError(err) + + claimed, err = s.store.ClaimAttachmentForDeletion(context.Background(), pending, time.Now().Add(time.Hour)) + s.Require().NoError(err) + s.False(claimed) + visible, err := s.store.AttachmentForUser(context.Background(), ready.ID, owner.ID) + s.Require().NoError(err) + s.Equal(model.AttachmentReady, visible.Status) +} + +func (s *PostgresSuite) TestReconciliationDryRunPreservesDeletingAttachment() { + owner, chatID := createOwnerAndChat(s.T(), s.store) + pending, err := s.store.CreatePendingAttachment(context.Background(), model.Attachment{ + ID: uuid.NewString(), ChatID: chatID, OwnerID: owner.ID, Filename: "deleting.png", + ContentType: "image/png", SizeBytes: 128, SHA256: strings.Repeat("e", 64), + ObjectKey: "attachments/" + uuid.NewString() + "/original", + }) + s.Require().NoError(err) + claimed, err := s.store.ClaimAttachmentForDeletion(context.Background(), pending, time.Now().Add(time.Hour)) + s.Require().NoError(err) + s.True(claimed) + + reconciler, err := reconcile.New(s.store, emptyObjectStore{}, reconcile.Config{ + Prefix: "attachments/", PageSize: 1000, GraceTime: time.Minute, + UnreferencedRetention: time.Hour, + }) + s.Require().NoError(err) + report, err := reconciler.Run(context.Background()) + s.Require().NoError(err) + s.Zero(report.DeletedUnreferencedRows) + + var status string + err = s.pool.QueryRow(context.Background(), "SELECT status FROM attachments WHERE id = $1", pending.ID).Scan(&status) + s.Require().NoError(err) + s.Equal(model.AttachmentDeleting, status) +} + +type emptyObjectStore struct{} + +func (emptyObjectStore) List(context.Context, string, string, int) ([]model.StoredObject, string, error) { + return nil, "", nil +} + +func (emptyObjectStore) Delete(context.Context, string) error { + return errors.New("unexpected object deletion") +} + +func (s *PostgresSuite) TestMessageRejectsAttachmentFromAnotherChat() { + owner, firstChatID := createOwnerAndChat(s.T(), s.store) + secondChat, err := s.store.CreateChat(context.Background(), "Second", []string{owner.ID}, owner.ID) + s.Require().NoError(err) + pending, err := s.store.CreatePendingAttachment(context.Background(), model.Attachment{ + ID: uuid.NewString(), ChatID: firstChatID, OwnerID: owner.ID, Filename: "photo.png", + ContentType: "image/png", SizeBytes: 128, SHA256: strings.Repeat("b", 64), + ObjectKey: "attachments/" + uuid.NewString() + "/original", + }) + s.Require().NoError(err) + attachment, err := s.store.MarkAttachmentReady(context.Background(), pending.ID, pending.ObjectKey) + s.Require().NoError(err) + _, _, err = s.store.CreateMessage( + context.Background(), secondChat.ID, owner.ID, "wrong chat", []string{attachment.ID}, uuid.NewString(), 24*time.Hour, + ) + s.ErrorIs(err, postgres.ErrNotFound) +} + +func (s *PostgresSuite) TestConcurrentMessageIdempotency() { + t := s.T() + pool := s.pool + stores := []*postgres.Store{postgres.New(pool), postgres.New(pool)} + owner, chatID := createOwnerAndChat(t, stores[0]) + key := uuid.NewString() + + const workers = 8 + type result struct { + message model.Message + replay bool + err error + } + results := make(chan result, workers) + var group sync.WaitGroup + for worker := range workers { + group.Add(1) + go func() { + defer group.Done() + message, replay, err := stores[worker%len(stores)].CreateMessage( + context.Background(), chatID, owner.ID, "concurrent body", nil, key, 24*time.Hour, + ) + results <- result{message: message, replay: replay, err: err} + }() + } + group.Wait() + close(results) + + var messageID string + var created int + for result := range results { + if result.err != nil { + t.Fatalf("CreateMessage() error = %v", result.err) + } + if messageID == "" { + messageID = result.message.ID + } + if result.message.ID != messageID { + t.Fatalf("message ID = %s, want %s", result.message.ID, messageID) + } + if !result.replay { + created++ + } + } + if created != 1 { + t.Fatalf("new message results = %d, want 1", created) + } + if count := messageCount(t, pool, chatID); count != 1 { + t.Fatalf("message count = %d, want 1", count) + } +} + +func (s *PostgresSuite) TestMessageIdempotencyScopeAndExpiry() { + t := s.T() + pool := s.pool + store := s.store + owner, chatID := createOwnerAndChat(t, store) + second, err := store.CreateUser(context.Background(), uniqueUsername(), "Second", "password-hash") + if err != nil { + t.Fatalf("CreateUser() error = %v", err) + } + chat, err := store.CreateChat(context.Background(), "Shared", []string{owner.ID, second.ID}, owner.ID) + if err != nil { + t.Fatalf("CreateChat() error = %v", err) + } + key := uuid.NewString() + + if _, _, err := store.CreateMessage(context.Background(), chat.ID, owner.ID, "owner", nil, key, 24*time.Hour); err != nil { + t.Fatalf("owner CreateMessage() error = %v", err) + } + if _, _, err := store.CreateMessage(context.Background(), chat.ID, second.ID, "second", nil, key, 24*time.Hour); err != nil { + t.Fatalf("second user CreateMessage() error = %v", err) + } + if _, _, err := store.CreateMessage(context.Background(), chatID, owner.ID, "other chat", nil, key, 24*time.Hour); err != nil { + t.Fatalf("other chat CreateMessage() error = %v", err) + } + if _, err := pool.Exec(context.Background(), ` + UPDATE message_idempotency + SET created_at = clock_timestamp() - INTERVAL '2 days', + expires_at = clock_timestamp() - INTERVAL '1 day' + WHERE user_id = $1 AND chat_id = $2 AND key = $3`, owner.ID, chat.ID, key); err != nil { + t.Fatalf("expire idempotency key: %v", err) + } + message, replay, err := store.CreateMessage(context.Background(), chat.ID, owner.ID, "after expiry", nil, key, 24*time.Hour) + if err != nil || replay || message.Text != "after expiry" { + t.Fatalf("expired CreateMessage() = (%v, %t, %v)", message, replay, err) + } +} + +func (s *PostgresSuite) TestExpiredIdempotencyRecordsAreCleanedOnNewMessages() { + t := s.T() + owner, chatID := createOwnerAndChat(t, s.store) + expiredKey := uuid.NewString() + if _, _, err := s.store.CreateMessage(context.Background(), chatID, owner.ID, "expired", nil, expiredKey, 24*time.Hour); err != nil { + t.Fatalf("create expired message: %v", err) + } + if _, err := s.pool.Exec(context.Background(), ` + UPDATE message_idempotency + SET created_at = clock_timestamp() - INTERVAL '2 days', + expires_at = clock_timestamp() - INTERVAL '1 day' + WHERE user_id = $1 AND chat_id = $2 AND key = $3`, owner.ID, chatID, expiredKey); err != nil { + t.Fatalf("expire idempotency key: %v", err) + } + if _, _, err := s.store.CreateMessage( + context.Background(), chatID, owner.ID, "new message", nil, uuid.NewString(), 24*time.Hour, + ); err != nil { + t.Fatalf("create new message: %v", err) + } + + var remaining int + if err := s.pool.QueryRow(context.Background(), ` + SELECT count(*) + FROM message_idempotency + WHERE user_id = $1 AND chat_id = $2 AND key = $3`, owner.ID, chatID, expiredKey).Scan(&remaining); err != nil { + t.Fatalf("count expired idempotency records: %v", err) + } + if remaining != 0 { + t.Fatalf("expired idempotency records = %d, want 0", remaining) + } +} + +func createOwnerAndChat(t *testing.T, store *postgres.Store) (model.User, string) { + t.Helper() + owner, err := store.CreateUser(context.Background(), uniqueUsername(), "Owner", "password-hash") + if err != nil { + t.Fatalf("CreateUser() error = %v", err) + } + chat, err := store.CreateChat(context.Background(), "Chat", []string{owner.ID}, owner.ID) + if err != nil { + t.Fatalf("CreateChat() error = %v", err) + } + return owner, chat.ID +} + +func createQueuedAttachment( + t *testing.T, + store *postgres.Store, + ownerID string, + chatID string, +) (model.Attachment, model.AttachmentJob) { + t.Helper() + attachmentID := uuid.NewString() + stagingKey := "staging/" + attachmentID + "/original" + pending, err := store.CreatePendingAttachment(context.Background(), model.Attachment{ + ID: attachmentID, ChatID: chatID, OwnerID: ownerID, Filename: "photo.png", + ContentType: "image/png", SizeBytes: 128, SHA256: strings.Repeat("a", 64), + ObjectKey: stagingKey, + }) + if err != nil { + t.Fatalf("CreatePendingAttachment() error = %v", err) + } + job := model.AttachmentJob{ + ID: uuid.NewString(), AttachmentID: attachmentID, PipelineVersion: 1, Attempt: 1, + } + queued, err := store.QueueAttachmentWithJobAndOutbox( + context.Background(), pending.ID, stagingKey, job, "", "", + ) + if err != nil { + t.Fatalf("QueueAttachmentWithJobAndOutbox() error = %v", err) + } + return queued, job +} + +func completedVariants(attachmentID string, pipelineVersion int) []model.Variant { + types := []struct { + name string + contentType string + extension string + }{ + {name: "original", contentType: "image/png", extension: "png"}, + {name: "thumbnail", contentType: "image/jpeg", extension: "jpg"}, + {name: "crop", contentType: "image/jpeg", extension: "jpg"}, + {name: "optimized", contentType: "image/webp", extension: "webp"}, + } + variants := make([]model.Variant, 0, len(types)) + for index, variant := range types { + variants = append(variants, model.Variant{ + Name: variant.name, ContentType: variant.contentType, SizeBytes: int64(128 - index*16), + SHA256: strings.Repeat(string(rune('a'+index)), 64), Width: 320 - index*32, Height: 200 - index*20, + ObjectKey: fmt.Sprintf("attachments/%s/v%d/%s.%s", attachmentID, pipelineVersion, variant.name, variant.extension), + PipelineVersion: pipelineVersion, + }) + } + return variants +} + +func messageCount(t *testing.T, pool *pgxpool.Pool, chatID string) int { + t.Helper() + var count int + if err := pool.QueryRow(context.Background(), "SELECT count(*) FROM messages WHERE chat_id = $1", chatID).Scan(&count); err != nil { + t.Fatalf("count messages: %v", err) + } + return count +} + +func openTestDatabase(t *testing.T) *pgxpool.Pool { + t.Helper() + url := testSecret(t, "TEST_DATABASE_URL", "TEST_DATABASE_URL_FILE") + pool, err := database.Open(context.Background(), database.Config{ + URL: url, MinConnections: 0, MaxConnections: 10, + MaxConnectionAge: time.Minute, MaxConnectionIdle: time.Minute, + HealthCheckPeriod: time.Minute, ConnectTimeout: 5 * time.Second, + }) + if err != nil { + t.Fatalf("database.Open() error = %v", err) + } + if err := database.Migrate(context.Background(), pool, 30*time.Second); err != nil { + pool.Close() + t.Fatalf("database.Migrate() error = %v", err) + } + return pool +} + +func testSecret(t *testing.T, valueName, fileName string) string { + t.Helper() + if value := strings.TrimSpace(os.Getenv(valueName)); value != "" { + return value + } + path := strings.TrimSpace(os.Getenv(fileName)) + if path == "" { + t.Skipf("set %s or %s to run integration tests", valueName, fileName) + } + data, err := os.ReadFile(path) + if err != nil { + t.Fatalf("read %s: %v", fileName, err) + } + return strings.TrimSpace(string(data)) +} + +func uniqueUsername() string { + return "u_" + strings.ReplaceAll(uuid.NewString(), "-", "")[:20] +} + +func chatCount(t *testing.T, pool *pgxpool.Pool) int { + t.Helper() + var count int + if err := pool.QueryRow(context.Background(), "SELECT count(*) FROM chats").Scan(&count); err != nil { + t.Fatalf("count chats: %v", err) + } + return count +} diff --git a/internal/reconcile/reconcile.go b/internal/reconcile/reconcile.go new file mode 100644 index 0000000..15607f8 --- /dev/null +++ b/internal/reconcile/reconcile.go @@ -0,0 +1,410 @@ +package reconcile + +import ( + "context" + "errors" + "fmt" + "time" + + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model" +) + +type MetadataStore interface { + ListAttachments(context.Context, string, int) ([]model.Attachment, string, error) + AttachmentObjectKeys(context.Context, string) ([]string, error) + AttachmentExistsByObjectKey(context.Context, string) (bool, error) + ClaimAttachmentForDeletion(context.Context, model.Attachment, time.Time) (bool, error) + DeleteAttachmentIfDeleting(context.Context, string) (bool, error) +} + +type ObjectStore interface { + List(context.Context, string, string, int) ([]model.StoredObject, string, error) + Delete(context.Context, string) error +} + +type Config struct { + Prefix string + OrphanPrefixes []string + PageSize int + GraceTime time.Duration + UnreferencedRetention time.Duration + Apply bool + Now func() time.Time +} + +type Report struct { + MetadataScanned int `json:"metadata_scanned"` + ObjectsScanned int `json:"objects_scanned"` + Consistent int `json:"consistent"` + FreshPendingAttachments int `json:"fresh_pending_attachments"` + OldPendingAttachments int `json:"old_pending_attachments"` + DeletingAttachments int `json:"deleting_attachments"` + FreshUnreferencedAttachments int `json:"fresh_unreferenced_attachments"` + OldUnreferencedAttachments int `json:"old_unreferenced_attachments"` + DeletedAttachmentObjects int `json:"deleted_attachment_objects"` + FreshOrphanObjects int `json:"fresh_orphan_objects"` + OldOrphanObjects int `json:"old_orphan_objects"` + DeletedOrphanObjects int `json:"deleted_orphan_objects"` + FreshMissingObjects int `json:"fresh_missing_objects"` + ReferencedMissingObjects int `json:"referenced_missing_objects"` + OldUnreferencedMetadata int `json:"old_unreferenced_metadata"` + DeletedUnreferencedRows int `json:"deleted_unreferenced_rows"` +} + +type Reconciler struct { + metadata MetadataStore + objects ObjectStore + config Config +} + +func New(metadata MetadataStore, objects ObjectStore, config Config) (*Reconciler, error) { + if config.PageSize < 1 || config.PageSize > 1000 { + return nil, errors.New("page size must be between 1 and 1000") + } + if config.GraceTime < 0 { + return nil, errors.New("grace time must not be negative") + } + if config.UnreferencedRetention <= 0 { + return nil, errors.New("unreferenced retention must be positive") + } + for _, prefix := range config.OrphanPrefixes { + if prefix == "" || prefix == config.Prefix { + return nil, errors.New("orphan prefix must be non-empty and distinct") + } + } + if config.Now == nil { + config.Now = time.Now + } + return &Reconciler{metadata: metadata, objects: objects, config: config}, nil +} + +func (r *Reconciler) Run(ctx context.Context) (Report, error) { + metadata, err := newMetadataIterator(ctx, r.metadata, r.config.PageSize) + if err != nil { + return Report{}, err + } + objects, err := newObjectIterator(ctx, r.objects, r.config.Prefix, r.config.PageSize) + if err != nil { + return Report{}, err + } + + now := r.config.Now() + anomalyCutoff := now.Add(-r.config.GraceTime) + retentionCutoff := now.Add(-r.config.UnreferencedRetention) + var report Report + for metadata.current != nil || objects.current != nil { + switch { + case metadata.current == nil || objects.current != nil && objects.current.Key < metadata.current.ObjectKey: + if err := r.handleOrphanObject(ctx, *objects.current, anomalyCutoff, &report); err != nil { + return report, err + } + if err := objects.advance(ctx); err != nil { + return report, err + } + case objects.current == nil || metadata.current.ObjectKey < objects.current.Key: + if err := r.handleMissingObject(ctx, *metadata.current, anomalyCutoff, &report); err != nil { + return report, err + } + if err := metadata.advance(ctx); err != nil { + return report, err + } + default: + if err := r.handlePair(ctx, *metadata.current, retentionCutoff, anomalyCutoff, &report); err != nil { + return report, err + } + if err := metadata.advance(ctx); err != nil { + return report, err + } + if err := objects.advance(ctx); err != nil { + return report, err + } + } + } + for _, prefix := range r.config.OrphanPrefixes { + if err := r.scanOrphanPrefix(ctx, prefix, anomalyCutoff, &report); err != nil { + return report, err + } + } + return report, nil +} + +func (r *Reconciler) scanOrphanPrefix( + ctx context.Context, + prefix string, + cutoff time.Time, + report *Report, +) error { + objects, err := newObjectIterator(ctx, r.objects, prefix, r.config.PageSize) + if err != nil { + return err + } + for objects.current != nil { + if err := r.handleOrphanObject(ctx, *objects.current, cutoff, report); err != nil { + return err + } + if err := objects.advance(ctx); err != nil { + return err + } + } + return nil +} + +func (r *Reconciler) handlePair( + ctx context.Context, + metadata model.Attachment, + retentionCutoff time.Time, + anomalyCutoff time.Time, + report *Report, +) error { + report.MetadataScanned++ + report.ObjectsScanned++ + switch metadata.Status { + case model.AttachmentQueued, model.AttachmentProcessing, model.AttachmentFailed: + report.Consistent++ + return nil + case model.AttachmentPending: + if metadata.CreatedAt.After(anomalyCutoff) { + report.FreshPendingAttachments++ + report.Consistent++ + return nil + } + report.OldPendingAttachments++ + return r.deletePair(ctx, metadata, anomalyCutoff, report) + case model.AttachmentDeleting: + report.DeletingAttachments++ + return r.deleteClaimedPair(ctx, metadata, report) + case model.AttachmentReady: + if metadata.Referenced { + report.Consistent++ + return nil + } + age := metadata.CreatedAt + if metadata.ReadyAt != nil { + age = *metadata.ReadyAt + } + if age.After(retentionCutoff) { + report.FreshUnreferencedAttachments++ + report.Consistent++ + return nil + } + report.OldUnreferencedAttachments++ + return r.deletePair(ctx, metadata, retentionCutoff, report) + default: + return fmt.Errorf("attachment %q has unknown status %q", metadata.ID, metadata.Status) + } +} + +func (r *Reconciler) deletePair(ctx context.Context, attachment model.Attachment, cutoff time.Time, report *Report) error { + if !r.config.Apply { + return nil + } + claimed, err := r.metadata.ClaimAttachmentForDeletion(ctx, attachment, cutoff) + if err != nil { + return fmt.Errorf("claim attachment %q: %w", attachment.ID, err) + } + if !claimed { + return nil + } + return r.deleteClaimedPair(ctx, attachment, report) +} + +func (r *Reconciler) deleteClaimedPair(ctx context.Context, attachment model.Attachment, report *Report) error { + if !r.config.Apply { + return nil + } + objectKeys, err := r.metadata.AttachmentObjectKeys(ctx, attachment.ID) + if err != nil { + return fmt.Errorf("list attachment %q object keys: %w", attachment.ID, err) + } + for _, objectKey := range objectKeys { + if err := r.objects.Delete(ctx, objectKey); err != nil { + return fmt.Errorf("delete attachment object %q: %w", objectKey, err) + } + report.DeletedAttachmentObjects++ + } + deleted, err := r.metadata.DeleteAttachmentIfDeleting(ctx, attachment.ID) + if err != nil { + return fmt.Errorf("delete attachment metadata %q: %w", attachment.ID, err) + } + if deleted { + report.DeletedUnreferencedRows++ + } + return nil +} + +func (r *Reconciler) handleOrphanObject( + ctx context.Context, + object model.StoredObject, + cutoff time.Time, + report *Report, +) error { + report.ObjectsScanned++ + exists, err := r.metadata.AttachmentExistsByObjectKey(ctx, object.Key) + if err != nil { + return fmt.Errorf("recheck orphan object %q: %w", object.Key, err) + } + if exists { + report.Consistent++ + return nil + } + if object.LastModified.After(cutoff) { + report.FreshOrphanObjects++ + return nil + } + report.OldOrphanObjects++ + if !r.config.Apply { + return nil + } + if err := r.objects.Delete(ctx, object.Key); err != nil { + return fmt.Errorf("delete orphan object %q: %w", object.Key, err) + } + report.DeletedOrphanObjects++ + return nil +} + +func (r *Reconciler) handleMissingObject( + ctx context.Context, + metadata model.Attachment, + cutoff time.Time, + report *Report, +) error { + report.MetadataScanned++ + if metadata.Status == model.AttachmentQueued || + metadata.Status == model.AttachmentProcessing || + metadata.Status == model.AttachmentFailed { + return nil + } + if metadata.Status == model.AttachmentPending { + if metadata.CreatedAt.After(cutoff) { + report.FreshPendingAttachments++ + return nil + } + report.OldPendingAttachments++ + return r.deletePair(ctx, metadata, cutoff, report) + } + if metadata.Status == model.AttachmentDeleting { + report.DeletingAttachments++ + return r.deleteClaimedMetadata(ctx, metadata, report) + } + if metadata.CreatedAt.After(cutoff) { + report.FreshMissingObjects++ + return nil + } + if metadata.Referenced { + report.ReferencedMissingObjects++ + return nil + } + report.OldUnreferencedMetadata++ + if !r.config.Apply { + return nil + } + claimed, err := r.metadata.ClaimAttachmentForDeletion(ctx, metadata, cutoff) + if err != nil { + return fmt.Errorf("claim missing attachment metadata %q: %w", metadata.ID, err) + } + if !claimed { + return nil + } + return r.deleteClaimedMetadata(ctx, metadata, report) +} + +func (r *Reconciler) deleteClaimedMetadata(ctx context.Context, metadata model.Attachment, report *Report) error { + if !r.config.Apply { + return nil + } + deleted, err := r.metadata.DeleteAttachmentIfDeleting(ctx, metadata.ID) + if err != nil { + return fmt.Errorf("delete missing attachment metadata %q: %w", metadata.ID, err) + } + if deleted { + report.DeletedUnreferencedRows++ + } + return nil +} + +type metadataIterator struct { + store MetadataStore + pageSize int + after string + page []model.Attachment + index int + current *model.Attachment + done bool +} + +func newMetadataIterator(ctx context.Context, store MetadataStore, pageSize int) (*metadataIterator, error) { + iterator := &metadataIterator{store: store, pageSize: pageSize} + if err := iterator.advance(ctx); err != nil { + return nil, err + } + return iterator, nil +} + +func (i *metadataIterator) advance(ctx context.Context) error { + for { + if i.index < len(i.page) { + i.current = &i.page[i.index] + i.index++ + return nil + } + if i.done { + i.current = nil + return nil + } + page, next, err := i.store.ListAttachments(ctx, i.after, i.pageSize) + if err != nil { + return fmt.Errorf("list attachment metadata: %w", err) + } + i.page = page + i.index = 0 + i.after = next + i.done = next == "" + if len(page) == 0 && !i.done { + return errors.New("metadata page is empty but has continuation") + } + } +} + +type objectIterator struct { + store ObjectStore + prefix string + pageSize int + after string + page []model.StoredObject + index int + current *model.StoredObject + done bool +} + +func newObjectIterator(ctx context.Context, store ObjectStore, prefix string, pageSize int) (*objectIterator, error) { + iterator := &objectIterator{store: store, prefix: prefix, pageSize: pageSize} + if err := iterator.advance(ctx); err != nil { + return nil, err + } + return iterator, nil +} + +func (i *objectIterator) advance(ctx context.Context) error { + for { + if i.index < len(i.page) { + i.current = &i.page[i.index] + i.index++ + return nil + } + if i.done { + i.current = nil + return nil + } + page, next, err := i.store.List(ctx, i.prefix, i.after, i.pageSize) + if err != nil { + return fmt.Errorf("list objects: %w", err) + } + i.page = page + i.index = 0 + i.after = next + i.done = next == "" + if len(page) == 0 && !i.done { + return errors.New("object page is empty but has continuation") + } + } +} diff --git a/internal/reconcile/reconcile_test.go b/internal/reconcile/reconcile_test.go new file mode 100644 index 0000000..36a2955 --- /dev/null +++ b/internal/reconcile/reconcile_test.go @@ -0,0 +1,340 @@ +package reconcile + +import ( + "context" + "sort" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model" +) + +func TestReconcilerLifecycleAndRepeat(t *testing.T) { + now := time.Date(2026, 10, 9, 12, 0, 0, 0, time.UTC) + old := now.Add(-48 * time.Hour) + fresh := now.Add(-10 * time.Minute) + metadata := &metadataStub{items: map[string]model.Attachment{ + "attachments/a/original": attachment("a", "attachments/a/original", model.AttachmentReady, old, false), + "attachments/b/original": attachment("b", "attachments/b/original", model.AttachmentReady, old, true), + "attachments/c/original": attachment("c", "attachments/c/original", model.AttachmentPending, old, false), + "attachments/d/original": attachment("d", "attachments/d/original", model.AttachmentReady, fresh, false), + "attachments/missing/original": attachment("missing", "attachments/missing/original", model.AttachmentReady, old, false), + "staging/pending/original": attachment("pending", "staging/pending/original", model.AttachmentPending, old, false), + }} + objects := &objectStub{items: map[string]model.StoredObject{ + "attachments/a/original": {Key: "attachments/a/original", LastModified: old}, + "attachments/b/original": {Key: "attachments/b/original", LastModified: old}, + "attachments/c/original": {Key: "attachments/c/original", LastModified: old}, + "attachments/d/original": {Key: "attachments/d/original", LastModified: fresh}, + "attachments/orphan/original": {Key: "attachments/orphan/original", LastModified: old}, + "staging/orphan/original": {Key: "staging/orphan/original", LastModified: old}, + "staging/pending/original": {Key: "staging/pending/original", LastModified: old}, + }} + reconciler, err := New(metadata, objects, Config{ + Prefix: "attachments/", PageSize: 2, GraceTime: time.Hour, + OrphanPrefixes: []string{"staging/"}, + UnreferencedRetention: 24 * time.Hour, Apply: true, Now: func() time.Time { return now }, + }) + require.NoError(t, err) + + report, err := reconciler.Run(context.Background()) + require.NoError(t, err) + require.Equal(t, 2, report.OldPendingAttachments) + require.Equal(t, 1, report.OldUnreferencedAttachments) + require.Equal(t, 1, report.FreshUnreferencedAttachments) + require.Equal(t, 2, report.DeletedOrphanObjects) + require.Equal(t, 3, report.DeletedAttachmentObjects) + require.Equal(t, 4, report.DeletedUnreferencedRows) + require.Contains(t, metadata.items, "attachments/b/original") + require.Contains(t, metadata.items, "attachments/d/original") + require.NotContains(t, metadata.items, "attachments/a/original") + require.NotContains(t, metadata.items, "attachments/c/original") + require.NotContains(t, metadata.items, "attachments/missing/original") + require.NotContains(t, metadata.items, "staging/pending/original") + require.NotContains(t, objects.items, "attachments/orphan/original") + require.NotContains(t, objects.items, "staging/orphan/original") + require.NotContains(t, objects.items, "staging/pending/original") + + repeated, err := reconciler.Run(context.Background()) + require.NoError(t, err) + require.Zero(t, repeated.DeletedOrphanObjects) + require.Zero(t, repeated.DeletedAttachmentObjects) + require.Zero(t, repeated.DeletedUnreferencedRows) +} + +func TestReconcilerRechecksOrphanBeforeDelete(t *testing.T) { + now := time.Now().UTC() + metadata := &metadataStub{items: map[string]model.Attachment{}, appeared: map[string]bool{"attachments/a/original": true}} + objects := &objectStub{items: map[string]model.StoredObject{ + "attachments/a/original": {Key: "attachments/a/original", LastModified: now.Add(-time.Hour)}, + }} + reconciler, err := New(metadata, objects, Config{ + Prefix: "attachments/", PageSize: 1, GraceTime: time.Minute, + UnreferencedRetention: time.Hour, Apply: true, Now: func() time.Time { return now }, + }) + require.NoError(t, err) + + report, err := reconciler.Run(context.Background()) + require.NoError(t, err) + require.Zero(t, report.DeletedOrphanObjects) + require.Contains(t, objects.items, "attachments/a/original") +} + +func TestReconcilerDeletingAttachmentRespectsApplyMode(t *testing.T) { + now := time.Now().UTC() + key := "attachments/a/original" + tests := []struct { + name string + apply bool + objectPresent bool + }{ + {name: "dry run with object", objectPresent: true}, + {name: "dry run without object"}, + {name: "apply with object", apply: true, objectPresent: true}, + {name: "apply without object", apply: true}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + metadata := &metadataStub{items: map[string]model.Attachment{ + key: attachment("a", key, model.AttachmentDeleting, now.Add(-time.Hour), false), + }} + objects := &objectStub{items: make(map[string]model.StoredObject)} + if test.objectPresent { + objects.items[key] = model.StoredObject{Key: key, LastModified: now.Add(-time.Hour)} + } + reconciler, err := New(metadata, objects, Config{ + Prefix: "attachments/", PageSize: 1, GraceTime: time.Minute, + UnreferencedRetention: time.Hour, Apply: test.apply, Now: func() time.Time { return now }, + }) + require.NoError(t, err) + + report, err := reconciler.Run(context.Background()) + require.NoError(t, err) + require.Equal(t, 1, report.DeletingAttachments) + var expectedDeletedRows, expectedDeletedObjects int + if test.apply { + expectedDeletedRows = 1 + } + if test.apply && test.objectPresent { + expectedDeletedObjects = 1 + } + require.Equal(t, expectedDeletedRows, report.DeletedUnreferencedRows) + require.Equal(t, expectedDeletedObjects, report.DeletedAttachmentObjects) + _, metadataExists := metadata.items[key] + require.Equal(t, !test.apply, metadataExists) + _, objectExists := objects.items[key] + require.Equal(t, test.objectPresent && !test.apply, objectExists) + }) + } +} + +func TestReconcilerDoesNotReportTrackedStagingObjectAsOrphan(t *testing.T) { + now := time.Now().UTC() + key := "staging/a/original" + metadata := &metadataStub{items: map[string]model.Attachment{ + key: attachment("a", key, model.AttachmentQueued, now.Add(-time.Hour), false), + }} + objects := &objectStub{items: map[string]model.StoredObject{ + key: {Key: key, LastModified: now.Add(-time.Hour)}, + }} + reconciler, err := New(metadata, objects, Config{ + Prefix: "attachments/", OrphanPrefixes: []string{"staging/"}, PageSize: 1, + GraceTime: time.Minute, UnreferencedRetention: time.Hour, Now: func() time.Time { return now }, + }) + require.NoError(t, err) + + report, err := reconciler.Run(context.Background()) + require.NoError(t, err) + require.Zero(t, report.OldOrphanObjects) + require.Zero(t, report.FreshOrphanObjects) + require.Contains(t, objects.items, key) +} + +func TestReconcilerPreservesAttachmentDuringReprocessing(t *testing.T) { + now := time.Date(2026, 10, 9, 12, 0, 0, 0, time.UTC) + for _, status := range []string{ + model.AttachmentQueued, + model.AttachmentProcessing, + model.AttachmentFailed, + } { + t.Run(status, func(t *testing.T) { + original := attachment("a", "attachments/a/v1/original.png", status, now.Add(-48*time.Hour), false) + original.PipelineVersion = 2 + missing := attachment("a", "attachments/a/v1/optimized.webp", status, now.Add(-48*time.Hour), false) + missing.PipelineVersion = 2 + metadata := &metadataStub{items: map[string]model.Attachment{ + original.ObjectKey: original, + missing.ObjectKey: missing, + }} + objects := &objectStub{items: map[string]model.StoredObject{ + original.ObjectKey: {Key: original.ObjectKey, LastModified: now.Add(-48 * time.Hour)}, + }} + reconciler, err := New(metadata, objects, Config{ + Prefix: "attachments/", PageSize: 10, GraceTime: time.Hour, + UnreferencedRetention: 24 * time.Hour, Apply: true, Now: func() time.Time { return now }, + }) + require.NoError(t, err) + + report, err := reconciler.Run(context.Background()) + require.NoError(t, err) + require.Zero(t, report.DeletedAttachmentObjects) + require.Zero(t, report.DeletedUnreferencedRows) + require.Contains(t, metadata.items, original.ObjectKey) + require.Contains(t, metadata.items, missing.ObjectKey) + require.Contains(t, objects.items, original.ObjectKey) + }) + } +} + +func TestReconcilerDeletesAllAttachmentVariantsOnce(t *testing.T) { + now := time.Date(2026, 10, 9, 12, 0, 0, 0, time.UTC) + old := now.Add(-48 * time.Hour) + keys := []string{ + "attachments/a/v1/crop.jpg", + "attachments/a/v1/optimized.webp", + "attachments/a/v1/original.png", + "attachments/a/v1/thumbnail.jpg", + } + metadata := &metadataStub{items: make(map[string]model.Attachment, len(keys))} + objects := &objectStub{items: make(map[string]model.StoredObject, len(keys))} + for _, key := range keys { + metadata.items[key] = attachment("a", key, model.AttachmentReady, old, false) + objects.items[key] = model.StoredObject{Key: key, LastModified: old} + } + reconciler, err := New(metadata, objects, Config{ + Prefix: "attachments/", PageSize: 10, GraceTime: time.Hour, + UnreferencedRetention: 24 * time.Hour, Apply: true, Now: func() time.Time { return now }, + }) + require.NoError(t, err) + + report, err := reconciler.Run(context.Background()) + require.NoError(t, err) + require.Equal(t, len(keys), report.DeletedAttachmentObjects) + require.Equal(t, 1, report.DeletedUnreferencedRows) + require.Empty(t, metadata.items) + require.Empty(t, objects.items) + + repeated, err := reconciler.Run(context.Background()) + require.NoError(t, err) + require.Zero(t, repeated.DeletedAttachmentObjects) + require.Zero(t, repeated.DeletedUnreferencedRows) +} + +func TestNewRejectsInvalidRetention(t *testing.T) { + _, err := New(&metadataStub{}, &objectStub{}, Config{PageSize: 1}) + require.Error(t, err) +} + +func attachment(id, key, status string, createdAt time.Time, referenced bool) model.Attachment { + readyAt := createdAt + return model.Attachment{ + ID: id, ObjectKey: key, Status: status, CreatedAt: createdAt, + ReadyAt: &readyAt, Referenced: referenced, + } +} + +type metadataStub struct { + items map[string]model.Attachment + appeared map[string]bool +} + +func (s *metadataStub) AttachmentObjectKeys(_ context.Context, attachmentID string) ([]string, error) { + var keys []string + for key, item := range s.items { + if item.ID == attachmentID { + keys = append(keys, key) + } + } + sort.Strings(keys) + return keys, nil +} + +func (s *metadataStub) ListAttachments(_ context.Context, after string, limit int) ([]model.Attachment, string, error) { + keys := make([]string, 0, len(s.items)) + for key := range s.items { + if key > after { + keys = append(keys, key) + } + } + sort.Strings(keys) + var next string + if len(keys) > limit { + keys = keys[:limit] + next = keys[len(keys)-1] + } + items := make([]model.Attachment, 0, len(keys)) + for _, key := range keys { + items = append(items, s.items[key]) + } + return items, next, nil +} + +func (s *metadataStub) AttachmentExistsByObjectKey(_ context.Context, key string) (bool, error) { + if s.appeared[key] { + return true, nil + } + _, exists := s.items[key] + return exists, nil +} + +func (s *metadataStub) ClaimAttachmentForDeletion(_ context.Context, candidate model.Attachment, cutoff time.Time) (bool, error) { + claim := false + for key, item := range s.items { + if item.ID != candidate.ID || item.Status != candidate.Status || item.Referenced { + continue + } + age := item.CreatedAt + if item.Status == model.AttachmentReady && item.ReadyAt != nil { + age = *item.ReadyAt + } + if age.After(cutoff) { + return false, nil + } + item.Status = model.AttachmentDeleting + s.items[key] = item + claim = true + } + return claim, nil +} + +func (s *metadataStub) DeleteAttachmentIfDeleting(_ context.Context, id string) (bool, error) { + deleted := false + for key, item := range s.items { + if item.ID == id && item.Status == model.AttachmentDeleting && !item.Referenced { + delete(s.items, key) + deleted = true + } + } + return deleted, nil +} + +type objectStub struct { + items map[string]model.StoredObject +} + +func (s *objectStub) List(_ context.Context, prefix, after string, limit int) ([]model.StoredObject, string, error) { + keys := make([]string, 0, len(s.items)) + for key := range s.items { + if key > after && len(key) >= len(prefix) && key[:len(prefix)] == prefix { + keys = append(keys, key) + } + } + sort.Strings(keys) + var next string + if len(keys) > limit { + keys = keys[:limit] + next = keys[len(keys)-1] + } + items := make([]model.StoredObject, 0, len(keys)) + for _, key := range keys { + items = append(items, s.items[key]) + } + return items, next, nil +} + +func (s *objectStub) Delete(_ context.Context, key string) error { + delete(s.items, key) + return nil +} diff --git a/internal/session/session.go b/internal/session/session.go new file mode 100644 index 0000000..a7df4b9 --- /dev/null +++ b/internal/session/session.go @@ -0,0 +1,163 @@ +package session + +import ( + "context" + "crypto/rand" + "crypto/sha256" + "encoding/base64" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "strings" + "time" + + "github.com/valkey-io/valkey-go" + + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model" +) + +const ( + tokenLength = 32 + keyPrefix = "session:" +) + +var ( + ErrNotFound = model.ErrSessionNotFound + ErrInvalidToken = model.ErrInvalidToken + ErrInvalidRecord = errors.New("invalid session record") + ErrInvalidTTL = errors.New("invalid session TTL") + ErrTokenGeneration = errors.New("session token generation failed") + ErrStore = model.ErrUnavailable +) + +type record struct { + UserID string `json:"user_id"` + ExpiresAt time.Time `json:"expires_at"` +} + +type Store struct { + client valkey.Client +} + +func New(rawURL string) (*Store, error) { + option, err := valkey.ParseURL(rawURL) + if err != nil { + return nil, fmt.Errorf("%w: parse Valkey URL: %w", ErrStore, err) + } + + client, err := valkey.NewClient(option) + if err != nil { + return nil, fmt.Errorf("%w: create Valkey client: %w", ErrStore, err) + } + + return &Store{client: client}, nil +} + +func (s *Store) Create(ctx context.Context, userID string, ttl time.Duration) (string, time.Time, error) { + if strings.TrimSpace(userID) == "" { + return "", time.Time{}, ErrInvalidRecord + } + if ttl < time.Second { + return "", time.Time{}, ErrInvalidTTL + } + + token, err := newToken() + if err != nil { + return "", time.Time{}, err + } + + record := record{ + UserID: userID, + ExpiresAt: time.Now().UTC().Add(ttl).Truncate(time.Microsecond), + } + payload, err := json.Marshal(record) + if err != nil { + return "", time.Time{}, fmt.Errorf("%w: encode session: %w", ErrStore, err) + } + + command := s.client.B().Set().Key(s.key(token)).Value(valkey.BinaryString(payload)).Ex(ttl).Build() + if err := s.client.Do(ctx, command).Error(); err != nil { + return "", time.Time{}, fmt.Errorf("%w: create session: %w", ErrStore, err) + } + + return token, record.ExpiresAt, nil +} + +func newToken() (string, error) { + random := make([]byte, tokenLength) + if _, err := rand.Read(random); err != nil { + return "", fmt.Errorf("%w: %v", ErrTokenGeneration, err) + } + return base64.RawURLEncoding.EncodeToString(random), nil +} + +func (s *Store) UserID(ctx context.Context, token string) (string, error) { + key, err := s.sessionKey(token) + if err != nil { + return "", err + } + + payload, err := s.client.Do(ctx, s.client.B().Get().Key(key).Build()).AsBytes() + if valkey.IsValkeyNil(err) { + return "", ErrNotFound + } + if err != nil { + return "", fmt.Errorf("%w: get session: %w", ErrStore, err) + } + + var record record + if err := json.Unmarshal(payload, &record); err != nil { + return "", fmt.Errorf("%w: decode session: %w", ErrStore, err) + } + if strings.TrimSpace(record.UserID) == "" || record.ExpiresAt.IsZero() { + return "", fmt.Errorf("%w: %w", ErrStore, ErrInvalidRecord) + } + if !record.ExpiresAt.After(time.Now()) { + return "", ErrNotFound + } + + return record.UserID, nil +} + +func (s *Store) Delete(ctx context.Context, token string) error { + key, err := s.sessionKey(token) + if err != nil { + return err + } + + deleted, err := s.client.Do(ctx, s.client.B().Del().Key(key).Build()).AsInt64() + if err != nil { + return fmt.Errorf("%w: delete session: %w", ErrStore, err) + } + if deleted == 0 { + return ErrNotFound + } + + return nil +} + +func (s *Store) Ping(ctx context.Context) error { + if err := s.client.Do(ctx, s.client.B().Ping().Build()).Error(); err != nil { + return fmt.Errorf("%w: ping Valkey: %w", ErrStore, err) + } + return nil +} + +func (s *Store) Close() error { + s.client.Close() + return nil +} + +func (s *Store) sessionKey(token string) (string, error) { + decoded, err := base64.RawURLEncoding.Strict().DecodeString(token) + if err != nil || len(decoded) != tokenLength { + return "", ErrInvalidToken + } + return s.key(token), nil +} + +func (s *Store) key(token string) string { + digest := sha256.Sum256([]byte(token)) + return keyPrefix + hex.EncodeToString(digest[:]) +} diff --git a/internal/session/session_integration_test.go b/internal/session/session_integration_test.go new file mode 100644 index 0000000..a82e5eb --- /dev/null +++ b/internal/session/session_integration_test.go @@ -0,0 +1,72 @@ +package session_test + +import ( + "context" + "errors" + "os" + "strings" + "testing" + "time" + + "github.com/google/uuid" + "github.com/stretchr/testify/suite" + + "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/session" +) + +type SessionSuite struct { + suite.Suite + store *session.Store +} + +func TestSessionSuite(t *testing.T) { + suite.Run(t, new(SessionSuite)) +} + +func (s *SessionSuite) SetupSuite() { + rawURL := strings.TrimSpace(os.Getenv("TEST_CACHE_URL")) + if rawURL == "" { + path := strings.TrimSpace(os.Getenv("TEST_CACHE_URL_FILE")) + if path == "" { + s.T().Skip("set TEST_CACHE_URL or TEST_CACHE_URL_FILE to run integration tests") + } + data, err := os.ReadFile(path) + s.Require().NoError(err) + rawURL = strings.TrimSpace(string(data)) + } + store, err := session.New(rawURL) + s.Require().NoError(err) + s.store = store +} + +func (s *SessionSuite) TearDownSuite() { + if s.store != nil { + s.Require().NoError(s.store.Close()) + } +} + +func (s *SessionSuite) TestSessionLifecycle() { + userID := uuid.NewString() + token, expiresAt, err := s.store.Create(context.Background(), userID, time.Minute) + s.Require().NoError(err) + s.NotEmpty(token) + s.True(expiresAt.After(time.Now())) + + actualUserID, err := s.store.UserID(context.Background(), token) + s.Require().NoError(err) + s.Equal(userID, actualUserID) + s.Require().NoError(s.store.Delete(context.Background(), token)) + + _, err = s.store.UserID(context.Background(), token) + s.ErrorIs(err, session.ErrNotFound) +} + +func (s *SessionSuite) TestSessionExpires() { + token, _, err := s.store.Create(context.Background(), uuid.NewString(), time.Second) + s.Require().NoError(err) + + s.Eventually(func() bool { + _, err = s.store.UserID(context.Background(), token) + return errors.Is(err, session.ErrNotFound) + }, 3*time.Second, 50*time.Millisecond) +} diff --git a/internal/telemetry/database.go b/internal/telemetry/database.go new file mode 100644 index 0000000..6390cd3 --- /dev/null +++ b/internal/telemetry/database.go @@ -0,0 +1,39 @@ +package telemetry + +import ( + "context" + + "github.com/jackc/pgx/v5/pgxpool" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/metric" +) + +var ( + connectionStateUsed = attribute.String("state", "used") + connectionStateIdle = attribute.String("state", "idle") + connectionStateConnecting = attribute.String("state", "connecting") +) + +func ObserveDatabasePool(meter metric.Meter, pool *pgxpool.Pool) error { + _, err := meter.Int64ObservableGauge("messenger_db_pool_connections", + metric.WithDescription("PostgreSQL pool connections by state."), + metric.WithInt64Callback(func(_ context.Context, observer metric.Int64Observer) error { + stats := pool.Stat() + observer.Observe(int64(stats.AcquiredConns()), metric.WithAttributes(connectionStateUsed)) + observer.Observe(int64(stats.IdleConns()), metric.WithAttributes(connectionStateIdle)) + observer.Observe(int64(stats.ConstructingConns()), metric.WithAttributes(connectionStateConnecting)) + return nil + }), + ) + if err != nil { + return err + } + _, err = meter.Int64ObservableCounter("messenger_db_pool_waited", + metric.WithDescription("Connection acquisitions that waited because the PostgreSQL pool was empty."), + metric.WithInt64Callback(func(_ context.Context, observer metric.Int64Observer) error { + observer.Observe(pool.Stat().EmptyAcquireCount()) + return nil + }), + ) + return err +} diff --git a/internal/telemetry/logging.go b/internal/telemetry/logging.go new file mode 100644 index 0000000..880d349 --- /dev/null +++ b/internal/telemetry/logging.go @@ -0,0 +1,74 @@ +package telemetry + +import ( + "context" + "log/slog" + "strings" + + "go.opentelemetry.io/otel/trace" +) + +type correlationHandler struct { + stdout slog.Handler + otel slog.Handler +} + +func newCorrelationHandler(stdout, otel slog.Handler) slog.Handler { + return correlationHandler{stdout: stdout, otel: otel} +} + +func (h correlationHandler) Enabled(ctx context.Context, level slog.Level) bool { + return h.stdout.Enabled(ctx, level) || h.otel.Enabled(ctx, level) +} + +func (h correlationHandler) Handle(ctx context.Context, record slog.Record) error { + span := trace.SpanContextFromContext(ctx) + if span.IsValid() { + record.AddAttrs(slog.String("trace_id", span.TraceID().String()), slog.String("span_id", span.SpanID().String())) + } + record = sanitizedRecord(record) + if err := h.stdout.Handle(ctx, record); err != nil { + return err + } + return h.otel.Handle(ctx, record) +} + +func (h correlationHandler) WithAttrs(attributes []slog.Attr) slog.Handler { + attributes = sanitizedAttributes(attributes) + return correlationHandler{stdout: h.stdout.WithAttrs(attributes), otel: h.otel.WithAttrs(attributes)} +} + +func (h correlationHandler) WithGroup(name string) slog.Handler { + return correlationHandler{stdout: h.stdout.WithGroup(name), otel: h.otel.WithGroup(name)} +} + +func sanitizedRecord(record slog.Record) slog.Record { + copy := slog.NewRecord(record.Time, record.Level, record.Message, record.PC) + record.Attrs(func(attribute slog.Attr) bool { + if !sensitiveKey(attribute.Key) { + copy.AddAttrs(attribute) + } + return true + }) + return copy +} + +func sanitizedAttributes(attributes []slog.Attr) []slog.Attr { + result := make([]slog.Attr, 0, len(attributes)) + for _, attribute := range attributes { + if !sensitiveKey(attribute.Key) { + result = append(result, attribute) + } + } + return result +} + +func sensitiveKey(key string) bool { + key = strings.ToLower(key) + for _, marker := range []string{"authorization", "password", "secret", "token", "cookie", "body", "payload"} { + if strings.Contains(key, marker) { + return true + } + } + return false +} diff --git a/internal/telemetry/propagation.go b/internal/telemetry/propagation.go new file mode 100644 index 0000000..92c3dac --- /dev/null +++ b/internal/telemetry/propagation.go @@ -0,0 +1,70 @@ +package telemetry + +import ( + "context" + "encoding/json" + + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/propagation" +) + +type MessageCarrier struct { + Traceparent string + Tracestate string +} + +func Extract(ctx context.Context, carrier propagation.TextMapCarrier) context.Context { + return otel.GetTextMapPropagator().Extract(ctx, carrier) +} + +func Inject(ctx context.Context, carrier propagation.TextMapCarrier) { + otel.GetTextMapPropagator().Inject(ctx, carrier) +} + +func ContextCarrier(ctx context.Context) MessageCarrier { + carrier := propagation.MapCarrier{} + Inject(ctx, carrier) + return MessageCarrier{ + Traceparent: carrier.Get("traceparent"), + Tracestate: carrier.Get("tracestate"), + } +} + +func ExtractMessage(ctx context.Context, carrier MessageCarrier) context.Context { + headers := propagation.MapCarrier{ + "traceparent": carrier.Traceparent, + "tracestate": carrier.Tracestate, + } + return Extract(ctx, headers) +} + +func ExtractJSONMessage(ctx context.Context, payload []byte) context.Context { + var carrier struct { + Traceparent string `json:"traceparent"` + Tracestate string `json:"tracestate"` + } + if json.Unmarshal(payload, &carrier) != nil { + return ctx + } + return ExtractMessage(ctx, MessageCarrier(carrier)) +} + +func InjectJSONMessage(ctx context.Context, payload []byte) ([]byte, error) { + var value map[string]json.RawMessage + if err := json.Unmarshal(payload, &value); err != nil { + return nil, err + } + carrier := ContextCarrier(ctx) + setJSONField(value, "traceparent", carrier.Traceparent) + setJSONField(value, "tracestate", carrier.Tracestate) + return json.Marshal(value) +} + +func setJSONField(value map[string]json.RawMessage, key, field string) { + if field == "" { + delete(value, key) + return + } + encoded, _ := json.Marshal(field) + value[key] = encoded +} diff --git a/internal/telemetry/propagation_test.go b/internal/telemetry/propagation_test.go new file mode 100644 index 0000000..31895a5 --- /dev/null +++ b/internal/telemetry/propagation_test.go @@ -0,0 +1,33 @@ +package telemetry + +import ( + "context" + "encoding/json" + "testing" + + "github.com/stretchr/testify/require" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/propagation" + "go.opentelemetry.io/otel/trace" +) + +func TestJSONMessagePropagationPreservesTraceContext(t *testing.T) { + otel.SetTextMapPropagator(propagation.TraceContext{}) + traceID, err := trace.TraceIDFromHex("4bf92f3577b34da6a3ce929d0e0e4736") + require.NoError(t, err) + spanID, err := trace.SpanIDFromHex("00f067aa0ba902b7") + require.NoError(t, err) + ctx := trace.ContextWithSpanContext(context.Background(), trace.NewSpanContext(trace.SpanContextConfig{ + TraceID: traceID, SpanID: spanID, TraceFlags: trace.FlagsSampled, + })) + + payload, err := InjectJSONMessage(ctx, []byte(`{"job_id":"job-1"}`)) + require.NoError(t, err) + var value map[string]any + require.NoError(t, json.Unmarshal(payload, &value)) + require.Equal(t, "00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01", value["traceparent"]) + require.NotContains(t, value, "baggage") + + extracted := ExtractJSONMessage(context.Background(), payload) + require.Equal(t, traceID, trace.SpanContextFromContext(extracted).TraceID()) +} diff --git a/internal/telemetry/runtime.go b/internal/telemetry/runtime.go new file mode 100644 index 0000000..90267e6 --- /dev/null +++ b/internal/telemetry/runtime.go @@ -0,0 +1,252 @@ +package telemetry + +import ( + "context" + "errors" + "fmt" + "log/slog" + "net/http" + "os" + "strings" + "time" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/collectors" + "github.com/prometheus/client_golang/prometheus/promhttp" + "go.opentelemetry.io/contrib/bridges/otelslog" + otelruntime "go.opentelemetry.io/contrib/instrumentation/runtime" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc" + "go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc" + "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc" + otelprometheus "go.opentelemetry.io/otel/exporters/prometheus" + "go.opentelemetry.io/otel/propagation" + "go.opentelemetry.io/otel/sdk/log" + "go.opentelemetry.io/otel/sdk/metric" + "go.opentelemetry.io/otel/sdk/resource" + "go.opentelemetry.io/otel/sdk/trace" + semconv "go.opentelemetry.io/otel/semconv/v1.34.0" +) + +const ( + defaultEnvironment = "lab" + defaultNamespace = "mtusi-highload" +) + +type Config struct { + ServiceName string + Version string + InstanceID string + Endpoint string + Environment string + Namespace string + Insecure bool +} + +type Runtime struct { + traces *trace.TracerProvider + metrics *metric.MeterProvider + logs *log.LoggerProvider + logger *slog.Logger + handler http.Handler +} + +func ConfigFromEnvironment(serviceName, instanceID string) Config { + resourceAttributes := parseResourceAttributes(os.Getenv("OTEL_RESOURCE_ATTRIBUTES")) + return Config{ + ServiceName: serviceName, + Version: firstValue(os.Getenv("SERVICE_VERSION"), resourceAttributes["service.version"], "dev"), + InstanceID: instanceID, + Endpoint: strings.TrimSpace(os.Getenv("OTEL_EXPORTER_OTLP_ENDPOINT")), + Environment: firstValue(os.Getenv("DEPLOYMENT_ENVIRONMENT"), resourceAttributes["deployment.environment.name"], defaultEnvironment), + Namespace: firstValue(os.Getenv("SERVICE_NAMESPACE"), resourceAttributes["service.namespace"], defaultNamespace), + Insecure: true, + } +} + +func parseResourceAttributes(raw string) map[string]string { + result := make(map[string]string) + for _, pair := range strings.Split(raw, ",") { + key, value, found := strings.Cut(pair, "=") + if found && strings.TrimSpace(key) != "" { + result[strings.TrimSpace(key)] = strings.TrimSpace(value) + } + } + return result +} + +func firstValue(values ...string) string { + for _, value := range values { + if value = strings.TrimSpace(value); value != "" { + return value + } + } + return "" +} + +func New(ctx context.Context, config Config) (*Runtime, error) { + if err := config.validate(); err != nil { + return nil, err + } + res, err := newResource(ctx, config) + if err != nil { + return nil, err + } + registry := prometheus.NewRegistry() + prometheusReader, err := otelprometheus.New(otelprometheus.WithRegisterer(registry)) + if err != nil { + return nil, fmt.Errorf("create Prometheus reader: %w", err) + } + + traceOptions := []trace.TracerProviderOption{trace.WithResource(res)} + metricOptions := []metric.Option{metric.WithResource(res), metric.WithReader(prometheusReader)} + logOptions := []log.LoggerProviderOption{log.WithResource(res)} + if config.Endpoint != "" { + traceExporter, err := newTraceExporter(ctx, config) + if err != nil { + return nil, err + } + metricExporter, err := newMetricExporter(ctx, config) + if err != nil { + return nil, err + } + logExporter, err := newLogExporter(ctx, config) + if err != nil { + return nil, err + } + traceOptions = append(traceOptions, trace.WithBatcher(traceExporter, + trace.WithMaxQueueSize(2048), trace.WithMaxExportBatchSize(512), + trace.WithBatchTimeout(time.Second), trace.WithExportTimeout(5*time.Second), + )) + metricOptions = append(metricOptions, metric.WithReader(metric.NewPeriodicReader( + metricExporter, metric.WithInterval(5*time.Second), metric.WithTimeout(5*time.Second), + ))) + logOptions = append(logOptions, log.WithProcessor(log.NewBatchProcessor(logExporter, + log.WithMaxQueueSize(2048), log.WithExportMaxBatchSize(512), + log.WithExportInterval(time.Second), log.WithExportTimeout(5*time.Second), + ))) + } + + traces := trace.NewTracerProvider(traceOptions...) + metrics := metric.NewMeterProvider(metricOptions...) + logs := log.NewLoggerProvider(logOptions...) + propagator := propagation.NewCompositeTextMapPropagator(propagation.TraceContext{}) + otel.SetTracerProvider(traces) + otel.SetMeterProvider(metrics) + otel.SetTextMapPropagator(propagator) + if err := otelruntime.Start(otelruntime.WithMeterProvider(metrics)); err != nil { + return nil, fmt.Errorf("start Go runtime metrics: %w", err) + } + + stdout := slog.NewJSONHandler(os.Stdout, nil) + otelHandler := otelslog.NewHandler(config.ServiceName, otelslog.WithLoggerProvider(logs)) + logger := slog.New(newCorrelationHandler(stdout, otelHandler)) + registry.MustRegister(collectors.NewProcessCollector(collectors.ProcessCollectorOpts{})) + return &Runtime{ + traces: traces, metrics: metrics, logs: logs, logger: logger, + handler: promhttp.HandlerFor(registry, promhttp.HandlerOpts{}), + }, nil +} + +func (c Config) validate() error { + switch { + case strings.TrimSpace(c.ServiceName) == "": + return errors.New("telemetry service name is required") + case strings.TrimSpace(c.InstanceID) == "": + return errors.New("telemetry instance ID is required") + case strings.TrimSpace(c.Environment) == "": + return errors.New("telemetry environment is required") + case strings.TrimSpace(c.Namespace) == "": + return errors.New("telemetry service namespace is required") + default: + return nil + } +} + +func newResource(ctx context.Context, config Config) (*resource.Resource, error) { + res, err := resource.New(ctx, + resource.WithFromEnv(), resource.WithTelemetrySDK(), resource.WithHost(), + resource.WithAttributes( + semconv.ServiceName(config.ServiceName), semconv.ServiceVersion(config.Version), + semconv.ServiceInstanceID(config.InstanceID), semconv.ServiceNamespace(config.Namespace), + semconv.DeploymentEnvironmentName(config.Environment), + ), + ) + if err != nil { + return nil, fmt.Errorf("create telemetry resource: %w", err) + } + return res, nil +} + +func newTraceExporter(ctx context.Context, config Config) (trace.SpanExporter, error) { + var options []otlptracegrpc.Option + if strings.Contains(config.Endpoint, "://") { + options = append(options, otlptracegrpc.WithEndpointURL(config.Endpoint)) + } else { + options = append(options, otlptracegrpc.WithEndpoint(config.Endpoint)) + } + if config.Insecure { + options = append(options, otlptracegrpc.WithInsecure()) + } + exporter, err := otlptracegrpc.New(ctx, options...) + if err != nil { + return nil, fmt.Errorf("create OTLP trace exporter: %w", err) + } + return exporter, nil +} + +func newMetricExporter(ctx context.Context, config Config) (*otlpmetricgrpc.Exporter, error) { + var options []otlpmetricgrpc.Option + if strings.Contains(config.Endpoint, "://") { + options = append(options, otlpmetricgrpc.WithEndpointURL(config.Endpoint)) + } else { + options = append(options, otlpmetricgrpc.WithEndpoint(config.Endpoint)) + } + if config.Insecure { + options = append(options, otlpmetricgrpc.WithInsecure()) + } + exporter, err := otlpmetricgrpc.New(ctx, options...) + if err != nil { + return nil, fmt.Errorf("create OTLP metric exporter: %w", err) + } + return exporter, nil +} + +func newLogExporter(ctx context.Context, config Config) (*otlploggrpc.Exporter, error) { + var options []otlploggrpc.Option + if strings.Contains(config.Endpoint, "://") { + options = append(options, otlploggrpc.WithEndpointURL(config.Endpoint)) + } else { + options = append(options, otlploggrpc.WithEndpoint(config.Endpoint)) + } + if config.Insecure { + options = append(options, otlploggrpc.WithInsecure()) + } + exporter, err := otlploggrpc.New(ctx, options...) + if err != nil { + return nil, fmt.Errorf("create OTLP log exporter: %w", err) + } + return exporter, nil +} + +func (r *Runtime) Logger() *slog.Logger { + return r.logger +} + +func (r *Runtime) MeterProvider() *metric.MeterProvider { + return r.metrics +} + +func (r *Runtime) MetricsHandler() http.Handler { + return r.handler +} + +func (r *Runtime) Shutdown(ctx context.Context) error { + var result error + for _, shutdown := range []func(context.Context) error{r.logs.Shutdown, r.metrics.Shutdown, r.traces.Shutdown} { + if err := shutdown(ctx); err != nil { + result = errors.Join(result, err) + } + } + return result +} diff --git a/internal/telemetry/runtime_test.go b/internal/telemetry/runtime_test.go new file mode 100644 index 0000000..e8b41b1 --- /dev/null +++ b/internal/telemetry/runtime_test.go @@ -0,0 +1,59 @@ +package telemetry + +import ( + "context" + "log/slog" + "testing" + "time" + + "github.com/stretchr/testify/require" + "go.opentelemetry.io/otel/trace" +) + +func TestConfigValidation(t *testing.T) { + valid := Config{ServiceName: "messenger-api", InstanceID: "api-1", Environment: "lab", Namespace: "mtusi-highload"} + require.NoError(t, valid.validate()) + + invalid := valid + invalid.ServiceName = "" + require.EqualError(t, invalid.validate(), "telemetry service name is required") +} + +func TestCorrelationHandlerAddsTraceAndRedactsSecrets(t *testing.T) { + traceID, err := trace.TraceIDFromHex("4bf92f3577b34da6a3ce929d0e0e4736") + require.NoError(t, err) + spanID, err := trace.SpanIDFromHex("00f067aa0ba902b7") + require.NoError(t, err) + ctx := trace.ContextWithSpanContext(context.Background(), trace.NewSpanContext(trace.SpanContextConfig{ + TraceID: traceID, SpanID: spanID, TraceFlags: trace.FlagsSampled, + })) + first := &recordHandler{} + second := &recordHandler{} + handler := newCorrelationHandler(first, second) + record := slog.NewRecord(time.Now(), slog.LevelInfo, "request", 0) + record.AddAttrs(slog.String("event", "http_request_completed"), slog.String("authorization", "secret")) + + require.NoError(t, handler.Handle(ctx, record)) + for _, captured := range []*recordHandler{first, second} { + require.Equal(t, traceID.String(), captured.attributes["trace_id"]) + require.Equal(t, spanID.String(), captured.attributes["span_id"]) + require.Equal(t, "http_request_completed", captured.attributes["event"]) + require.NotContains(t, captured.attributes, "authorization") + } +} + +type recordHandler struct { + attributes map[string]string +} + +func (*recordHandler) Enabled(context.Context, slog.Level) bool { return true } +func (h *recordHandler) Handle(_ context.Context, record slog.Record) error { + h.attributes = make(map[string]string) + record.Attrs(func(attribute slog.Attr) bool { + h.attributes[attribute.Key] = attribute.Value.String() + return true + }) + return nil +} +func (h *recordHandler) WithAttrs([]slog.Attr) slog.Handler { return h } +func (h *recordHandler) WithGroup(string) slog.Handler { return h } diff --git a/lab6.env.example b/lab6.env.example new file mode 100644 index 0000000..ca34a40 --- /dev/null +++ b/lab6.env.example @@ -0,0 +1,93 @@ +# Изолированное имя Compose-проекта и его volumes/networks. +COMPOSE_PROJECT_NAME=mtusi2026lab6 + +# Адрес и внутренний порт HTTP-сервера. +APP_HOST=0.0.0.0 +APP_PORT=8080 +MESSENGER_CONFIG_PATH=config/messenger.yaml + +# Адреса публикации сервисов на локальной машине. +API_BIND_ADDRESS=127.0.0.1:8080 +SWAGGER_BIND_ADDRESS=127.0.0.1:8081 +PROMETHEUS_BIND_ADDRESS=127.0.0.1:9090 +GRAFANA_BIND_ADDRESS=127.0.0.1:3000 +LOKI_BIND_ADDRESS=127.0.0.1:3100 +TEMPO_BIND_ADDRESS=127.0.0.1:3200 + +# Ресурсные атрибуты и внутренний OTLP endpoint приложений. +DEPLOYMENT_ENVIRONMENT=lab +SERVICE_VERSION=lab6 +OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4317 +OTEL_EXPORTER_OTLP_PROTOCOL=grpc + +# Число API-реплик за Nginx. +API_REPLICAS=2 + +# Учётные данные локального PostgreSQL и строка подключения API. +POSTGRES_USER=messenger +POSTGRES_DB=messenger +POSTGRES_PASSWORD=local-postgres-password +DATABASE_URL=postgres://messenger:local-postgres-password@postgres:5432/messenger?sslmode=disable + +# Пароль локального Valkey и строка подключения API. +VALKEY_PASSWORD=local-valkey-password +CACHE_URL=redis://default:local-valkey-password@valkey:6379/0 + +# Redis Streams используется только как устойчивый брокер заданий. +REDIS_BROKER_PASSWORD=local-redis-broker-password +BROKER_URL=redis://default:local-redis-broker-password@redis-broker:6379/0 + +# Административные данные MinIO используются только при bootstrap. +MINIO_ROOT_USER=local-minio-admin +MINIO_ROOT_PASSWORD=local-minio-admin-password + +# API использует отдельного пользователя с доступом только к bucket вложений. +S3_ENDPOINT=s3:9000 +S3_REGION=us-east-1 +S3_BUCKET=messenger-attachments +S3_ACCESS_KEY=messenger-app +S3_SECRET_KEY=local-messenger-s3-password + +# Порт Swagger внутри конфигурации CORS API. +SWAGGER_PORT=8081 + +# Ограничения ресурсов контейнеров. +API_CPUS=1 +API_MEMORY=512m +MIGRATE_CPUS=0.5 +MIGRATE_MEMORY=128m +NGINX_CPUS=0.5 +NGINX_MEMORY=128m +SWAGGER_CPUS=0.25 +SWAGGER_MEMORY=128m +TEST_CPUS=2 +TEST_MEMORY=512m +EXPORTER_CPUS=0.25 +EXPORTER_MEMORY=128m +PROMETHEUS_CPUS=1 +PROMETHEUS_MEMORY=512m +GRAFANA_CPUS=1 +GRAFANA_MEMORY=512m +OTEL_COLLECTOR_CPUS=0.5 +OTEL_COLLECTOR_MEMORY=256m +LOKI_CPUS=1 +LOKI_MEMORY=512m +TEMPO_CPUS=1 +TEMPO_MEMORY=512m +POSTGRES_CPUS=2 +POSTGRES_MEMORY=1g +VALKEY_CPUS=1 +VALKEY_MEMORY=256m +REDIS_BROKER_CPUS=1 +REDIS_BROKER_MEMORY=256m +DISPATCHER_CPUS=0.5 +DISPATCHER_MEMORY=128m +WORKER_CPUS=2 +WORKER_MEMORY=1g +S3_CPUS=1 +S3_MEMORY=512m +S3_BOOTSTRAP_CPUS=0.25 +S3_BOOTSTRAP_MEMORY=256m + +# Пароль локального администратора Grafana. +GRAFANA_ADMIN_PASSWORD=local-grafana-password diff --git a/migrations/0001_initial.sql b/migrations/0001_initial.sql new file mode 100644 index 0000000..2f9b90b --- /dev/null +++ b/migrations/0001_initial.sql @@ -0,0 +1,50 @@ +CREATE TABLE users ( + id UUID PRIMARY KEY, + username TEXT NOT NULL, + display_name TEXT NOT NULL, + password_hash TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp(), + CONSTRAINT users_username_unique UNIQUE (username), + CONSTRAINT users_username_format CHECK (username ~ '^[a-z0-9_]{3,32}$'), + CONSTRAINT users_display_name_length CHECK (char_length(display_name) BETWEEN 1 AND 100), + CONSTRAINT users_display_name_not_blank CHECK (display_name ~ '[^[:space:]]'), + CONSTRAINT users_password_hash_length CHECK (char_length(password_hash) BETWEEN 1 AND 1024), + CONSTRAINT users_password_hash_not_blank CHECK (password_hash ~ '[^[:space:]]') +); + +CREATE TABLE chats ( + id UUID PRIMARY KEY, + title TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp(), + CONSTRAINT chats_title_length CHECK (char_length(title) BETWEEN 1 AND 100), + CONSTRAINT chats_title_not_blank CHECK (title ~ '[^[:space:]]') +); + +CREATE INDEX chats_created_at_id_idx ON chats (created_at, id); + +CREATE TABLE chat_members ( + chat_id UUID NOT NULL, + user_id UUID NOT NULL, + position SMALLINT NOT NULL, + CONSTRAINT chat_members_pk PRIMARY KEY (chat_id, user_id), + CONSTRAINT chat_members_chat_position_unique UNIQUE (chat_id, position), + CONSTRAINT chat_members_position_range CHECK (position BETWEEN 0 AND 99), + CONSTRAINT chat_members_chat_fk FOREIGN KEY (chat_id) REFERENCES chats (id) ON DELETE CASCADE, + CONSTRAINT chat_members_user_fk FOREIGN KEY (user_id) REFERENCES users (id) ON DELETE CASCADE +); + +CREATE INDEX chat_members_user_id_chat_id_idx ON chat_members (user_id, chat_id); + +CREATE TABLE messages ( + id UUID PRIMARY KEY, + chat_id UUID NOT NULL, + sender_id UUID NOT NULL, + text TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp(), + CONSTRAINT messages_text_length CHECK (char_length(text) BETWEEN 1 AND 4000), + CONSTRAINT messages_text_not_blank CHECK (text ~ '[^[:space:]]'), + CONSTRAINT messages_sender_membership_fk FOREIGN KEY (chat_id, sender_id) + REFERENCES chat_members (chat_id, user_id) ON DELETE CASCADE +); + +CREATE INDEX messages_chat_id_created_at_id_idx ON messages (chat_id, created_at, id); diff --git a/migrations/0002_message_idempotency.sql b/migrations/0002_message_idempotency.sql new file mode 100644 index 0000000..91443a4 --- /dev/null +++ b/migrations/0002_message_idempotency.sql @@ -0,0 +1,19 @@ +CREATE TABLE message_idempotency ( + user_id UUID NOT NULL, + chat_id UUID NOT NULL, + key TEXT NOT NULL, + request_hash BYTEA NOT NULL, + message_id UUID NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp(), + expires_at TIMESTAMPTZ NOT NULL, + CONSTRAINT message_idempotency_pk PRIMARY KEY (user_id, chat_id, key), + CONSTRAINT message_idempotency_user_fk FOREIGN KEY (user_id) REFERENCES users (id) ON DELETE CASCADE, + CONSTRAINT message_idempotency_chat_fk FOREIGN KEY (chat_id) REFERENCES chats (id) ON DELETE CASCADE, + CONSTRAINT message_idempotency_message_fk FOREIGN KEY (message_id) REFERENCES messages (id) + ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED, + CONSTRAINT message_idempotency_key_format CHECK (key ~ '^[A-Za-z0-9._:-]{1,128}$'), + CONSTRAINT message_idempotency_request_hash_length CHECK (octet_length(request_hash) = 32), + CONSTRAINT message_idempotency_expiry CHECK (expires_at > created_at) +); + +CREATE INDEX message_idempotency_expires_at_idx ON message_idempotency (expires_at); diff --git a/migrations/0003_attachments.sql b/migrations/0003_attachments.sql new file mode 100644 index 0000000..91d1cc9 --- /dev/null +++ b/migrations/0003_attachments.sql @@ -0,0 +1,47 @@ +ALTER TABLE messages + DROP CONSTRAINT messages_text_length, + DROP CONSTRAINT messages_text_not_blank; + +ALTER TABLE messages + ADD CONSTRAINT messages_text_length CHECK (char_length(text) BETWEEN 0 AND 4000), + ADD CONSTRAINT messages_id_chat_unique UNIQUE (id, chat_id); + +CREATE TABLE attachments ( + id UUID PRIMARY KEY, + chat_id UUID NOT NULL, + owner_id UUID NOT NULL, + filename TEXT NOT NULL, + content_type TEXT NOT NULL, + size_bytes BIGINT NOT NULL, + sha256 TEXT NOT NULL, + object_key TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp(), + CONSTRAINT attachments_id_chat_unique UNIQUE (id, chat_id), + CONSTRAINT attachments_object_key_unique UNIQUE (object_key), + CONSTRAINT attachments_owner_membership_fk FOREIGN KEY (chat_id, owner_id) + REFERENCES chat_members (chat_id, user_id) ON DELETE CASCADE, + CONSTRAINT attachments_filename_length CHECK (char_length(filename) BETWEEN 1 AND 255), + CONSTRAINT attachments_content_type CHECK (content_type IN ('image/png', 'image/jpeg')), + CONSTRAINT attachments_size CHECK (size_bytes BETWEEN 1 AND 10485760), + CONSTRAINT attachments_sha256 CHECK (sha256 ~ '^[a-f0-9]{64}$'), + CONSTRAINT attachments_object_key_length CHECK (char_length(object_key) BETWEEN 1 AND 1024) +); + +CREATE INDEX attachments_chat_id_created_at_id_idx ON attachments (chat_id, created_at, id); + +CREATE TABLE message_attachments ( + message_id UUID NOT NULL, + chat_id UUID NOT NULL, + attachment_id UUID NOT NULL, + position SMALLINT NOT NULL, + CONSTRAINT message_attachments_pk PRIMARY KEY (message_id, attachment_id), + CONSTRAINT message_attachments_position_unique UNIQUE (message_id, position), + CONSTRAINT message_attachments_position_range CHECK (position BETWEEN 0 AND 9), + CONSTRAINT message_attachments_message_chat_fk FOREIGN KEY (message_id, chat_id) + REFERENCES messages (id, chat_id) ON DELETE CASCADE, + CONSTRAINT message_attachments_attachment_fk FOREIGN KEY (attachment_id, chat_id) + REFERENCES attachments (id, chat_id) ON DELETE RESTRICT +); + +CREATE INDEX message_attachments_attachment_id_idx ON message_attachments (attachment_id); diff --git a/migrations/0004_attachment_lifecycle.sql b/migrations/0004_attachment_lifecycle.sql new file mode 100644 index 0000000..1596c99 --- /dev/null +++ b/migrations/0004_attachment_lifecycle.sql @@ -0,0 +1,16 @@ +ALTER TABLE attachments + ADD COLUMN status TEXT NOT NULL DEFAULT 'ready', + ADD COLUMN ready_at TIMESTAMPTZ DEFAULT clock_timestamp(); + +UPDATE attachments +SET ready_at = created_at; + +ALTER TABLE attachments + ADD CONSTRAINT attachments_status CHECK (status IN ('pending', 'ready', 'deleting')), + ADD CONSTRAINT attachments_ready_at CHECK ( + (status = 'pending' AND ready_at IS NULL) OR + (status IN ('ready', 'deleting') AND ready_at IS NOT NULL) + ); + +CREATE INDEX attachments_status_created_at_object_key_idx + ON attachments (status, created_at, object_key); diff --git a/migrations/0005_allow_pending_deletion.sql b/migrations/0005_allow_pending_deletion.sql new file mode 100644 index 0000000..0b20acb --- /dev/null +++ b/migrations/0005_allow_pending_deletion.sql @@ -0,0 +1,9 @@ +ALTER TABLE attachments + DROP CONSTRAINT attachments_ready_at; + +ALTER TABLE attachments + ADD CONSTRAINT attachments_ready_at CHECK ( + (status = 'pending' AND ready_at IS NULL) OR + (status = 'ready' AND ready_at IS NOT NULL) OR + status = 'deleting' + ); diff --git a/migrations/0006_async_attachment_pipeline.sql b/migrations/0006_async_attachment_pipeline.sql new file mode 100644 index 0000000..2b061b6 --- /dev/null +++ b/migrations/0006_async_attachment_pipeline.sql @@ -0,0 +1,135 @@ +ALTER TABLE attachments + DROP CONSTRAINT attachments_status, + DROP CONSTRAINT attachments_ready_at; + +ALTER TABLE attachments + ADD COLUMN pipeline_version INTEGER NOT NULL DEFAULT 1, + ADD COLUMN error_code TEXT, + ADD COLUMN staging_key TEXT; + +UPDATE attachments +SET staging_key = object_key +WHERE status = 'pending'; + +ALTER TABLE attachments + ADD CONSTRAINT attachments_status CHECK ( + status IN ('pending', 'queued', 'processing', 'ready', 'failed', 'deleting') + ), + ADD CONSTRAINT attachments_pipeline_version CHECK (pipeline_version >= 1), + ADD CONSTRAINT attachments_error_code CHECK ( + (status = 'failed' AND error_code IS NOT NULL AND char_length(error_code) BETWEEN 1 AND 64) OR + (status <> 'failed' AND error_code IS NULL) + ), + ADD CONSTRAINT attachments_ready_at CHECK ( + (status = 'ready' AND ready_at IS NOT NULL) OR + (status <> 'ready') + ), + ADD CONSTRAINT attachments_staging_key_length CHECK ( + staging_key IS NULL OR char_length(staging_key) BETWEEN 1 AND 1024 + ); + +CREATE INDEX attachments_status_updated_at_idx + ON attachments (status, updated_at, id); + +CREATE TABLE attachment_jobs ( + id UUID PRIMARY KEY, + attachment_id UUID NOT NULL REFERENCES attachments (id) ON DELETE CASCADE, + pipeline_version INTEGER NOT NULL, + attempt INTEGER NOT NULL DEFAULT 1, + status TEXT NOT NULL DEFAULT 'queued', + available_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp(), + lease_token UUID, + lease_owner TEXT, + lease_until TIMESTAMPTZ, + error_code TEXT, + created_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp(), + CONSTRAINT attachment_jobs_effect_unique UNIQUE (attachment_id, pipeline_version), + CONSTRAINT attachment_jobs_pipeline_version CHECK (pipeline_version >= 1), + CONSTRAINT attachment_jobs_attempt CHECK (attempt >= 1), + CONSTRAINT attachment_jobs_status CHECK (status IN ('queued', 'processing', 'ready', 'failed')), + CONSTRAINT attachment_jobs_lease CHECK ( + (status = 'processing' AND lease_token IS NOT NULL AND lease_owner IS NOT NULL AND lease_until IS NOT NULL) OR + (status <> 'processing' AND lease_token IS NULL AND lease_owner IS NULL AND lease_until IS NULL) + ), + CONSTRAINT attachment_jobs_error_code CHECK ( + error_code IS NULL OR char_length(error_code) BETWEEN 1 AND 64 + ) +); + +CREATE INDEX attachment_jobs_due_idx + ON attachment_jobs (available_at, id) + WHERE status = 'queued'; + +CREATE INDEX attachment_jobs_expired_lease_idx + ON attachment_jobs (lease_until, id) + WHERE status = 'processing'; + +CREATE TABLE attachment_variants ( + attachment_id UUID NOT NULL REFERENCES attachments (id) ON DELETE CASCADE, + pipeline_version INTEGER NOT NULL, + name TEXT NOT NULL, + content_type TEXT NOT NULL, + size_bytes BIGINT NOT NULL, + sha256 TEXT NOT NULL, + width INTEGER NOT NULL, + height INTEGER NOT NULL, + object_key TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp(), + CONSTRAINT attachment_variants_pk PRIMARY KEY (attachment_id, pipeline_version, name), + CONSTRAINT attachment_variants_object_key_unique UNIQUE (object_key), + CONSTRAINT attachment_variants_pipeline_version CHECK (pipeline_version >= 1), + CONSTRAINT attachment_variants_name CHECK (name IN ('original', 'thumbnail', 'crop', 'optimized')), + CONSTRAINT attachment_variants_content_type CHECK (content_type IN ('image/png', 'image/jpeg', 'image/webp')), + CONSTRAINT attachment_variants_size CHECK (size_bytes >= 1), + CONSTRAINT attachment_variants_sha256 CHECK (sha256 ~ '^[a-f0-9]{64}$'), + CONSTRAINT attachment_variants_dimensions CHECK (width >= 1 AND height >= 1), + CONSTRAINT attachment_variants_object_key_length CHECK (char_length(object_key) BETWEEN 1 AND 1024) +); + +CREATE TABLE outbox_events ( + id UUID PRIMARY KEY, + topic TEXT NOT NULL, + partition_key TEXT NOT NULL, + payload JSONB NOT NULL, + job_id UUID NOT NULL REFERENCES attachment_jobs (id) ON DELETE CASCADE, + attempt INTEGER NOT NULL, + available_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp(), + publish_tries INTEGER NOT NULL DEFAULT 0, + claim_token UUID, + claim_until TIMESTAMPTZ, + last_error_code TEXT, + published_at TIMESTAMPTZ, + created_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp(), + CONSTRAINT outbox_events_job_attempt_unique UNIQUE (job_id, attempt), + CONSTRAINT outbox_events_attempt CHECK (attempt >= 1), + CONSTRAINT outbox_events_publish_tries CHECK (publish_tries >= 0), + CONSTRAINT outbox_events_claim CHECK ( + (claim_token IS NULL AND claim_until IS NULL) OR + (claim_token IS NOT NULL AND claim_until IS NOT NULL) + ), + CONSTRAINT outbox_events_last_error_code CHECK ( + last_error_code IS NULL OR char_length(last_error_code) BETWEEN 1 AND 64 + ) +); + +CREATE INDEX outbox_events_due_idx + ON outbox_events (available_at, id) + WHERE published_at IS NULL; + +CREATE TABLE attachment_replay_audit ( + id UUID PRIMARY KEY, + job_id UUID NOT NULL REFERENCES attachment_jobs (id) ON DELETE CASCADE, + attachment_id UUID NOT NULL REFERENCES attachments (id) ON DELETE CASCADE, + previous_attempt INTEGER NOT NULL, + new_attempt INTEGER NOT NULL, + actor TEXT NOT NULL, + reason TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp(), + CONSTRAINT attachment_replay_attempts CHECK (previous_attempt >= 1 AND new_attempt = previous_attempt + 1), + CONSTRAINT attachment_replay_actor_length CHECK (char_length(actor) BETWEEN 1 AND 128), + CONSTRAINT attachment_replay_reason_length CHECK (char_length(reason) BETWEEN 1 AND 512) +); + +CREATE INDEX attachment_replay_audit_job_created_at_idx + ON attachment_replay_audit (job_id, created_at); diff --git a/migrations/0007_outbox_retention.sql b/migrations/0007_outbox_retention.sql new file mode 100644 index 0000000..cc936f6 --- /dev/null +++ b/migrations/0007_outbox_retention.sql @@ -0,0 +1,3 @@ +CREATE INDEX outbox_events_published_at_idx + ON outbox_events (published_at, id) + WHERE published_at IS NOT NULL; diff --git a/migrations/0008_attachment_terminal_events.sql b/migrations/0008_attachment_terminal_events.sql new file mode 100644 index 0000000..53f17ce --- /dev/null +++ b/migrations/0008_attachment_terminal_events.sql @@ -0,0 +1,33 @@ +CREATE TABLE attachment_terminal_events ( + job_id UUID PRIMARY KEY, + status TEXT NOT NULL, + accepted_at TIMESTAMPTZ NOT NULL, + terminal_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp(), + CONSTRAINT attachment_terminal_events_status CHECK (status IN ('ready', 'failed')), + CONSTRAINT attachment_terminal_events_time CHECK (terminal_at >= accepted_at) +); + +CREATE INDEX attachment_terminal_events_status_terminal_at_idx + ON attachment_terminal_events (status, terminal_at); + +WITH first_replay AS ( + SELECT job_id, MIN(created_at) AS created_at + FROM attachment_replay_audit + GROUP BY job_id +) +INSERT INTO attachment_terminal_events (job_id, status, accepted_at, terminal_at) +SELECT jobs.id, 'failed', LEAST(jobs.created_at, first_replay.created_at), first_replay.created_at +FROM attachment_jobs jobs +JOIN first_replay ON first_replay.job_id = jobs.id +ON CONFLICT (job_id) DO NOTHING; + +INSERT INTO attachment_terminal_events (job_id, status, accepted_at, terminal_at) +SELECT jobs.id, jobs.status, LEAST(jobs.created_at, jobs.updated_at), jobs.updated_at +FROM attachment_jobs jobs +WHERE jobs.status IN ('ready', 'failed') + AND NOT EXISTS ( + SELECT 1 + FROM attachment_replay_audit replay + WHERE replay.job_id = jobs.id + ) +ON CONFLICT (job_id) DO NOTHING; diff --git a/migrations/embed.go b/migrations/embed.go new file mode 100644 index 0000000..08bbf60 --- /dev/null +++ b/migrations/embed.go @@ -0,0 +1,8 @@ +package migrations + +import "embed" + +// FS contains database migrations applied by the application at startup. +// +//go:embed *.sql +var FS embed.FS diff --git a/monitoring/grafana/dashboards/messenger-lab6.json b/monitoring/grafana/dashboards/messenger-lab6.json new file mode 100644 index 0000000..d3b39d9 --- /dev/null +++ b/monitoring/grafana/dashboards/messenger-lab6.json @@ -0,0 +1,1512 @@ +{ + "annotations": { + "list": [] + }, + "editable": false, + "graphTooltip": 1, + "panels": [ + { + "type": "row", + "title": "API and edge", + "id": 100, + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "panels": [ + { + "type": "timeseries", + "title": "API requests by route", + "id": 1, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 1 + }, + "targets": [ + { + "expr": "sum by (method, route) (rate(messenger_http_requests_total[1m]))", + "legendFormat": "{{method}} {{route}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "reqps" + }, + "overrides": [] + } + }, + { + "type": "timeseries", + "title": "API responses by status class", + "id": 2, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 1 + }, + "targets": [ + { + "expr": "sum by (status_class) (rate(messenger_http_requests_total[1m]))", + "legendFormat": "{{status_class}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "reqps" + }, + "overrides": [] + } + }, + { + "type": "timeseries", + "title": "API latency p50 / p95 / p99 by route", + "id": 3, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 9 + }, + "targets": [ + { + "expr": "histogram_quantile(0.50, sum by (le, method, route) (rate(messenger_http_request_duration_seconds_bucket[5m])))", + "legendFormat": "p50 {{method}} {{route}}", + "refId": "A" + }, + { + "expr": "histogram_quantile(0.95, sum by (le, method, route) (rate(messenger_http_request_duration_seconds_bucket[5m])))", + "legendFormat": "p95 {{method}} {{route}}", + "refId": "B" + }, + { + "expr": "histogram_quantile(0.99, sum by (le, method, route) (rate(messenger_http_request_duration_seconds_bucket[5m])))", + "legendFormat": "p99 {{method}} {{route}}", + "refId": "C" + } + ], + "fieldConfig": { + "defaults": { + "unit": "s" + }, + "overrides": [] + } + }, + { + "type": "timeseries", + "title": "API replicas and requests in flight", + "id": 4, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 9 + }, + "targets": [ + { + "expr": "sum(up{job=\"messenger\"})", + "legendFormat": "healthy replicas", + "refId": "A" + }, + { + "expr": "sum(messenger_http_requests_in_flight)", + "legendFormat": "in flight", + "refId": "B" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + } + }, + { + "type": "timeseries", + "title": "API CPU usage", + "id": 5, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 17 + }, + "targets": [ + { + "expr": "sum(rate(process_cpu_seconds_total{job=\"messenger\"}[1m]))", + "legendFormat": "API", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "cores" + }, + "overrides": [] + } + }, + { + "type": "timeseries", + "title": "Nginx requests and connections", + "id": 6, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 17 + }, + "targets": [ + { + "expr": "sum(rate(nginx_http_requests_total[1m]))", + "legendFormat": "requests/s", + "refId": "A" + }, + { + "expr": "sum(nginx_connections_active)", + "legendFormat": "active connections", + "refId": "B" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + } + } + ] + }, + { + "type": "row", + "title": "PostgreSQL", + "id": 101, + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 25 + }, + "panels": [ + { + "type": "timeseries", + "title": "PostgreSQL availability and connections", + "id": 7, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 26 + }, + "targets": [ + { + "expr": "up{job=\"postgres\"}", + "legendFormat": "up", + "refId": "A" + }, + { + "expr": "sum(pg_stat_activity_count{datname=\"messenger\"})", + "legendFormat": "connections", + "refId": "B" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + } + }, + { + "type": "timeseries", + "title": "PostgreSQL transactions", + "id": 8, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 26 + }, + "targets": [ + { + "expr": "sum(rate(pg_stat_database_xact_commit{datname=\"messenger\"}[1m]))", + "legendFormat": "commit/s", + "refId": "A" + }, + { + "expr": "sum(rate(pg_stat_database_xact_rollback{datname=\"messenger\"}[1m]))", + "legendFormat": "rollback/s", + "refId": "B" + } + ], + "fieldConfig": { + "defaults": { + "unit": "ops" + }, + "overrides": [] + } + }, + { + "type": "timeseries", + "title": "PostgreSQL database size", + "id": 9, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 34 + }, + "targets": [ + { + "expr": "pg_database_size_bytes{datname=\"messenger\"}", + "legendFormat": "messenger", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "bytes" + }, + "overrides": [] + } + }, + { + "type": "timeseries", + "title": "Attachment jobs by status", + "id": 10, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 34 + }, + "targets": [ + { + "expr": "messenger_attachment_jobs_count", + "legendFormat": "{{status}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + } + }, + { + "type": "timeseries", + "title": "Oldest attachment job", + "id": 11, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 42 + }, + "targets": [ + { + "expr": "messenger_attachment_jobs_oldest_age_seconds", + "legendFormat": "{{status}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "s" + }, + "overrides": [] + } + }, + { + "type": "timeseries", + "title": "Pending outbox", + "id": 12, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 42 + }, + "targets": [ + { + "expr": "messenger_outbox_pending", + "legendFormat": "events", + "refId": "A" + }, + { + "expr": "messenger_outbox_oldest_pending_age_seconds", + "legendFormat": "oldest age, seconds", + "refId": "B" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byFrameRefID", + "options": "B" + }, + "properties": [ + { + "id": "unit", + "value": "s" + } + ] + } + ] + } + } + ] + }, + { + "type": "row", + "title": "Valkey and Redis", + "id": 102, + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 50 + }, + "panels": [ + { + "type": "timeseries", + "title": "Valkey and Redis availability", + "id": 13, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 51 + }, + "targets": [ + { + "expr": "up{job=\"valkey\"}", + "legendFormat": "Valkey", + "refId": "A" + }, + { + "expr": "up{job=\"redis-broker\"}", + "legendFormat": "Redis broker", + "refId": "B" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + } + }, + { + "type": "timeseries", + "title": "Valkey and Redis memory", + "id": 14, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 51 + }, + "targets": [ + { + "expr": "redis_memory_used_bytes{job=\"valkey\"}", + "legendFormat": "Valkey", + "refId": "A" + }, + { + "expr": "redis_memory_used_bytes{job=\"redis-broker\"}", + "legendFormat": "Redis broker", + "refId": "B" + } + ], + "fieldConfig": { + "defaults": { + "unit": "bytes" + }, + "overrides": [] + } + }, + { + "type": "timeseries", + "title": "Valkey cache hits and misses", + "id": 15, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 59 + }, + "targets": [ + { + "expr": "sum(rate(redis_keyspace_hits_total{job=\"valkey\"}[1m]))", + "legendFormat": "hits/s", + "refId": "A" + }, + { + "expr": "sum(rate(redis_keyspace_misses_total{job=\"valkey\"}[1m]))", + "legendFormat": "misses/s", + "refId": "B" + } + ], + "fieldConfig": { + "defaults": { + "unit": "ops" + }, + "overrides": [] + } + }, + { + "type": "timeseries", + "title": "Valkey and Redis commands", + "id": 16, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 59 + }, + "targets": [ + { + "expr": "sum(rate(redis_commands_processed_total{job=\"valkey\"}[1m]))", + "legendFormat": "Valkey commands/s", + "refId": "A" + }, + { + "expr": "sum(rate(redis_commands_processed_total{job=\"redis-broker\"}[1m]))", + "legendFormat": "Redis commands/s", + "refId": "B" + } + ], + "fieldConfig": { + "defaults": { + "unit": "ops" + }, + "overrides": [] + } + } + ] + }, + { + "type": "row", + "title": "Object storage", + "id": 103, + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 67 + }, + "panels": [ + { + "type": "timeseries", + "title": "MinIO availability", + "id": 17, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 68 + }, + "targets": [ + { + "expr": "up{job=\"minio\"}", + "legendFormat": "up", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + } + }, + { + "type": "timeseries", + "title": "MinIO objects and requests", + "id": 18, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 68 + }, + "targets": [ + { + "expr": "sum(minio_cluster_usage_object_total)", + "legendFormat": "objects", + "refId": "A" + }, + { + "expr": "sum(rate(minio_s3_requests_total[1m]))", + "legendFormat": "requests/s", + "refId": "B" + }, + { + "expr": "sum(rate(minio_s3_requests_errors_total[1m]))", + "legendFormat": "errors/s", + "refId": "C" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + } + }, + { + "type": "timeseries", + "title": "MinIO storage", + "id": 19, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 68 + }, + "targets": [ + { + "expr": "sum(minio_cluster_usage_total_bytes)", + "legendFormat": "used", + "refId": "A" + }, + { + "expr": "sum(minio_cluster_capacity_usable_free_bytes)", + "legendFormat": "free", + "refId": "B" + } + ], + "fieldConfig": { + "defaults": { + "unit": "bytes" + }, + "overrides": [] + } + } + ] + }, + { + "type": "row", + "title": "Attachment pipeline", + "id": 104, + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 76 + }, + "panels": [ + { + "type": "timeseries", + "title": "Pipeline availability and consumers", + "id": 20, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 77 + }, + "targets": [ + { + "expr": "sum(up{job=\"attachment-worker\"})", + "legendFormat": "workers up", + "refId": "A" + }, + { + "expr": "sum(up{job=\"outbox-dispatcher\"})", + "legendFormat": "dispatcher up", + "refId": "B" + }, + { + "expr": "max(redis_stream_group_consumers{job=\"redis-broker\",stream=\"attachment-jobs\",group=\"attachment-workers\"})", + "legendFormat": "active consumers", + "refId": "C" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + } + }, + { + "type": "timeseries", + "title": "Attachment upload results", + "id": 21, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 77 + }, + "targets": [ + { + "expr": "sum by (result) (rate(messenger_attachment_uploads_total[1m]))", + "legendFormat": "{{result}}", + "refId": "A" + }, + { + "expr": "sum(rate(messenger_attachment_upload_bytes_total[1m]))", + "legendFormat": "bytes/s", + "refId": "B" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byFrameRefID", + "options": "B" + }, + "properties": [ + { + "id": "unit", + "value": "Bps" + } + ] + } + ] + } + }, + { + "type": "timeseries", + "title": "Worker throughput by result", + "id": 22, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 85 + }, + "targets": [ + { + "expr": "sum by (result) (rate(messenger_attachment_worker_jobs_total[1m]))", + "legendFormat": "{{result}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "ops" + }, + "overrides": [] + } + }, + { + "type": "timeseries", + "title": "Worker processing latency p95", + "id": 23, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 85 + }, + "targets": [ + { + "expr": "histogram_quantile(0.95, sum by (le) (rate(messenger_attachment_worker_job_duration_seconds_bucket[5m])))", + "legendFormat": "p95", + "refId": "A" + }, + { + "expr": "histogram_quantile(0.95, sum by (le) (rate(messenger_attachment_upload_duration_seconds_bucket[5m])))", + "legendFormat": "upload p95", + "refId": "B" + } + ], + "fieldConfig": { + "defaults": { + "unit": "s" + }, + "overrides": [] + } + }, + { + "type": "timeseries", + "title": "Worker jobs in flight", + "id": 24, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 93 + }, + "targets": [ + { + "expr": "sum(messenger_attachment_worker_jobs_in_flight)", + "legendFormat": "jobs", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + } + }, + { + "type": "timeseries", + "title": "Retries and reclaimed deliveries", + "id": 25, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 93 + }, + "targets": [ + { + "expr": "sum(rate(messenger_attachment_worker_retries_total[1m]))", + "legendFormat": "retries/s", + "refId": "A" + }, + { + "expr": "sum(rate(messenger_attachment_worker_reclaimed_deliveries_total[1m]))", + "legendFormat": "reclaimed/s", + "refId": "B" + } + ], + "fieldConfig": { + "defaults": { + "unit": "ops" + }, + "overrides": [] + } + }, + { + "type": "timeseries", + "title": "Outbox dispatcher", + "id": 26, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 93 + }, + "targets": [ + { + "expr": "sum(rate(messenger_outbox_published_total[1m]))", + "legendFormat": "published/s", + "refId": "A" + }, + { + "expr": "sum(rate(messenger_outbox_publish_failures_total[1m]))", + "legendFormat": "failures/s", + "refId": "B" + }, + { + "expr": "sum(rate(messenger_outbox_recovered_leases_total[1m]))", + "legendFormat": "recovered/s", + "refId": "C" + } + ], + "fieldConfig": { + "defaults": { + "unit": "ops" + }, + "overrides": [] + } + }, + { + "type": "timeseries", + "title": "Redis Stream lag, PEL and DLQ", + "id": 27, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 101 + }, + "targets": [ + { + "expr": "max(redis_stream_group_lag{job=\"redis-broker\",stream=\"attachment-jobs\",group=\"attachment-workers\"})", + "legendFormat": "lag", + "refId": "A" + }, + { + "expr": "max(redis_stream_group_messages_pending{job=\"redis-broker\",stream=\"attachment-jobs\",group=\"attachment-workers\"})", + "legendFormat": "PEL", + "refId": "B" + }, + { + "expr": "max(redis_stream_length{job=\"redis-broker\",stream=\"attachment-jobs-failed\"})", + "legendFormat": "DLQ", + "refId": "C" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + } + }, + { + "type": "timeseries", + "title": "Redis Stream consumers", + "id": 28, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 101 + }, + "targets": [ + { + "expr": "max(redis_stream_group_consumers{job=\"redis-broker\",stream=\"attachment-jobs\",group=\"attachment-workers\"})", + "legendFormat": "consumers", + "refId": "A" + }, + { + "expr": "max(redis_stream_group_consumer_idle_seconds{job=\"redis-broker\",stream=\"attachment-jobs\",group=\"attachment-workers\"})", + "legendFormat": "max idle, seconds", + "refId": "B" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byFrameRefID", + "options": "B" + }, + "properties": [ + { + "id": "unit", + "value": "s" + } + ] + } + ] + } + } + ] + }, + { + "type": "row", + "title": "SLO and alerts", + "id": 105, + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 109 + }, + "panels": [ + { + "type": "timeseries", + "title": "Business API availability", + "id": 31, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 110 + }, + "targets": [ + { + "expr": "1 - (sum(rate(messenger_http_requests_total{route!~=\"/health/.*|/metrics\",status_class=\"5xx\"}[5m])) or vector(0)) / clamp_min((sum(rate(messenger_http_requests_total{route!~=\"/health/.*|/metrics\"}[5m])) or vector(0)), 0.000001)", + "legendFormat": "availability", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "percentunit" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "type": "timeseries", + "title": "API error-budget burn rate (99.5% SLO)", + "id": 32, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 110 + }, + "targets": [ + { + "expr": "(sum(rate(messenger_http_requests_total{route!~=\"/health/.*|/metrics\",status_class=\"5xx\"}[5m])) or vector(0)) / clamp_min((sum(rate(messenger_http_requests_total{route!~=\"/health/.*|/metrics\"}[5m])) or vector(0)), 0.000001) / 0.005", + "legendFormat": "5m burn", + "refId": "A" + }, + { + "expr": "(sum(rate(messenger_http_requests_total{route!~=\"/health/.*|/metrics\",status_class=\"5xx\"}[1h])) or vector(0)) / clamp_min((sum(rate(messenger_http_requests_total{route!~=\"/health/.*|/metrics\"}[1h])) or vector(0)), 0.000001) / 0.005", + "legendFormat": "1h burn", + "refId": "B" + }, + { + "expr": "(sum(rate(messenger_http_requests_total{route!~=\"/health/.*|/metrics\",status_class=\"5xx\"}[6h])) or vector(0)) / clamp_min((sum(rate(messenger_http_requests_total{route!~=\"/health/.*|/metrics\"}[6h])) or vector(0)), 0.000001) / 0.005", + "legendFormat": "6h burn", + "refId": "C" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "type": "timeseries", + "title": "Attachments ready within 30 seconds", + "id": 33, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 118 + }, + "targets": [ + { + "expr": "1 - (clamp_min((sum(rate(messenger_attachment_time_to_ready_seconds_bucket{le=\"+Inf\"}[5m])) or vector(0)) - (sum(rate(messenger_attachment_time_to_ready_seconds_bucket{le=\"30\"}[5m])) or vector(0)), 0) + (sum(rate(messenger_attachment_terminal_transitions_total{status=\"failed\"}[5m])) or vector(0))) / clamp_min((sum(rate(messenger_attachment_time_to_ready_seconds_bucket{le=\"+Inf\"}[5m])) or vector(0)) + (sum(rate(messenger_attachment_terminal_transitions_total{status=\"failed\"}[5m])) or vector(0)), 0.000001)", + "legendFormat": "ready in 30s", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "percentunit" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "type": "timeseries", + "title": "Attachment ready-time error-budget burn", + "id": 34, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 118 + }, + "targets": [ + { + "expr": "(clamp_min((sum(rate(messenger_attachment_time_to_ready_seconds_bucket{le=\"+Inf\"}[5m])) or vector(0)) - (sum(rate(messenger_attachment_time_to_ready_seconds_bucket{le=\"30\"}[5m])) or vector(0)), 0) + (sum(rate(messenger_attachment_terminal_transitions_total{status=\"failed\"}[5m])) or vector(0))) / clamp_min((sum(rate(messenger_attachment_time_to_ready_seconds_bucket{le=\"+Inf\"}[5m])) or vector(0)) + (sum(rate(messenger_attachment_terminal_transitions_total{status=\"failed\"}[5m])) or vector(0)), 0.000001) / 0.01", + "legendFormat": "5m burn", + "refId": "A" + }, + { + "expr": "(clamp_min((sum(rate(messenger_attachment_time_to_ready_seconds_bucket{le=\"+Inf\"}[1h])) or vector(0)) - (sum(rate(messenger_attachment_time_to_ready_seconds_bucket{le=\"30\"}[1h])) or vector(0)), 0) + (sum(rate(messenger_attachment_terminal_transitions_total{status=\"failed\"}[1h])) or vector(0))) / clamp_min((sum(rate(messenger_attachment_time_to_ready_seconds_bucket{le=\"+Inf\"}[1h])) or vector(0)) + (sum(rate(messenger_attachment_terminal_transitions_total{status=\"failed\"}[1h])) or vector(0)), 0.000001) / 0.01", + "legendFormat": "1h burn", + "refId": "B" + }, + { + "expr": "(clamp_min((sum(rate(messenger_attachment_time_to_ready_seconds_bucket{le=\"+Inf\"}[6h])) or vector(0)) - (sum(rate(messenger_attachment_time_to_ready_seconds_bucket{le=\"30\"}[6h])) or vector(0)), 0) + (sum(rate(messenger_attachment_terminal_transitions_total{status=\"failed\"}[6h])) or vector(0))) / clamp_min((sum(rate(messenger_attachment_time_to_ready_seconds_bucket{le=\"+Inf\"}[6h])) or vector(0)) + (sum(rate(messenger_attachment_terminal_transitions_total{status=\"failed\"}[6h])) or vector(0)), 0.000001) / 0.01", + "legendFormat": "6h burn", + "refId": "C" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "type": "timeseries", + "title": "SLO alert state", + "id": 35, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 126 + }, + "targets": [ + { + "expr": "max by (alertname, alertstate) (ALERTS{alertname=~\"MessengerAPIHighErrorBudgetBurn.*|AttachmentReadyHighErrorBudgetBurn.*|AttachmentQueueBacklogGrowing|AttachmentWorkerUnavailable\"})", + "legendFormat": "{{alertname}} {{alertstate}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + } + ] + }, + { + "type": "row", + "title": "Runtime resources", + "id": 107, + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 134 + }, + "panels": [ + { + "type": "timeseries", + "title": "Process CPU by service", + "id": 38, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 135 + }, + "targets": [ + { + "expr": "sum by (service_name) (rate(process_cpu_seconds_total[1m]))", + "legendFormat": "{{service_name}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "cores" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "type": "timeseries", + "title": "Go heap by service", + "id": 39, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 135 + }, + "targets": [ + { + "expr": "sum by (service_name) (go_memstats_heap_alloc_bytes)", + "legendFormat": "{{service_name}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "bytes" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "type": "timeseries", + "title": "Database pool used and wait rate", + "id": 40, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 143 + }, + "targets": [ + { + "expr": "sum by (service_name) (messenger_db_pool_connections{state=\"used\"})", + "legendFormat": "{{service_name}} used", + "refId": "A" + }, + { + "expr": "sum by (service_name) (rate(messenger_db_pool_waited_total[5m]))", + "legendFormat": "{{service_name}} waits/s", + "refId": "B" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "type": "timeseries", + "title": "Worker saturation", + "id": 41, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 143 + }, + "targets": [ + { + "expr": "sum(messenger_attachment_worker_jobs_in_flight)", + "legendFormat": "busy", + "refId": "A" + }, + { + "expr": "sum(messenger_attachment_worker_capacity)", + "legendFormat": "capacity", + "refId": "B" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + } + ] + }, + { + "type": "row", + "title": "Logs and traces", + "id": 106, + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 151 + }, + "panels": [ + { + "type": "logs", + "title": "Correlated application logs", + "id": 36, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 152 + }, + "datasource": { + "type": "loki", + "uid": "loki" + }, + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "loki" + }, + "editorMode": "code", + "expr": "{service_name=~\"messenger-.+\"} | json", + "queryType": "range", + "refId": "A" + } + ], + "options": { + "dedupStrategy": "none", + "enableLogDetails": true, + "prettifyLogMessage": false, + "showCommonLabels": false, + "showLabels": false, + "showTime": true, + "sortOrder": "Descending", + "wrapLogMessage": true + } + }, + { + "type": "traces", + "title": "Recent application traces (TraceQL)", + "id": 37, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 152 + }, + "datasource": { + "type": "tempo", + "uid": "tempo" + }, + "targets": [ + { + "datasource": { + "type": "tempo", + "uid": "tempo" + }, + "query": "{ resource.service.name =~ \"messenger-.*\" }", + "queryType": "traceql", + "refId": "A", + "limit": 50, + "tableType": "traces" + } + ], + "options": { + "showSpanFilters": true + } + }, + { + "type": "timeseries", + "title": "Trace-derived service call rate", + "id": 42, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 162 + }, + "targets": [ + { + "expr": "sum by (service, span_name) (rate(traces_spanmetrics_calls_total[5m]))", + "legendFormat": "{{service}} {{span_name}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "ops" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + } + }, + { + "type": "timeseries", + "title": "Collector delivery failures and queue pressure", + "id": 43, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 162 + }, + "targets": [ + { + "expr": "sum by (exporter) (rate(otelcol_exporter_send_failed_spans[5m]))", + "legendFormat": "{{exporter}} spans failed/s", + "refId": "A" + }, + { + "expr": "sum by (exporter) (rate(otelcol_exporter_send_failed_log_records[5m]))", + "legendFormat": "{{exporter}} logs failed/s", + "refId": "B" + }, + { + "expr": "max by (exporter) (otelcol_exporter_queue_size)", + "legendFormat": "{{exporter}} queue", + "refId": "C" + }, + { + "expr": "max by (exporter) (otelcol_exporter_queue_capacity)", + "legendFormat": "{{exporter}} capacity", + "refId": "D" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + } + } + ] + } + ], + "refresh": "5s", + "schemaVersion": 41, + "tags": [ + "highload", + "lab6", + "observability", + "slo" + ], + "templating": { + "list": [] + }, + "time": { + "from": "now-15m", + "to": "now" + }, + "timezone": "browser", + "title": "Messenger lab6 observability", + "uid": "messenger-lab6", + "version": 1, + "links": [ + { + "asDropdown": false, + "icon": "external link", + "includeVars": true, + "keepTime": true, + "tags": [], + "targetBlank": false, + "title": "Explore logs and traces", + "type": "link", + "url": "/explore" + }, + { + "asDropdown": false, + "icon": "external link", + "includeVars": false, + "keepTime": false, + "tags": [], + "targetBlank": true, + "title": "API runbook", + "type": "link", + "url": "https://gitea.privcloud.tech/dizi/mtusi2026lab6/src/branch/solution/bvt2303-plastinin/monitoring/runbooks/api-availability.md" + }, + { + "asDropdown": false, + "icon": "external link", + "includeVars": false, + "keepTime": false, + "tags": [], + "targetBlank": true, + "title": "Pipeline runbook", + "type": "link", + "url": "https://gitea.privcloud.tech/dizi/mtusi2026lab6/src/branch/solution/bvt2303-plastinin/monitoring/runbooks/attachment-pipeline.md" + } + ] +} diff --git a/monitoring/grafana/provisioning/alerting/.gitkeep b/monitoring/grafana/provisioning/alerting/.gitkeep new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/monitoring/grafana/provisioning/alerting/.gitkeep @@ -0,0 +1 @@ + diff --git a/monitoring/grafana/provisioning/dashboards/messenger.yml b/monitoring/grafana/provisioning/dashboards/messenger.yml new file mode 100644 index 0000000..fa7934c --- /dev/null +++ b/monitoring/grafana/provisioning/dashboards/messenger.yml @@ -0,0 +1,10 @@ +apiVersion: 1 + +providers: + - name: Messenger + folder: Highload + type: file + disableDeletion: true + editable: false + options: + path: /etc/grafana/dashboards diff --git a/monitoring/grafana/provisioning/datasources/observability.yml b/monitoring/grafana/provisioning/datasources/observability.yml new file mode 100644 index 0000000..741e6d2 --- /dev/null +++ b/monitoring/grafana/provisioning/datasources/observability.yml @@ -0,0 +1,76 @@ +apiVersion: 1 + +deleteDatasources: + - name: Prometheus + orgId: 1 + - name: Loki + orgId: 1 + - name: Tempo + orgId: 1 + +datasources: + - name: Prometheus + type: prometheus + uid: prometheus + access: proxy + url: http://prometheus:9090 + isDefault: true + editable: false + jsonData: + httpMethod: POST + exemplarTraceIdDestinations: + - name: trace_id + datasourceUid: tempo + + - name: Loki + type: loki + uid: loki + access: proxy + url: http://loki:3100 + editable: false + jsonData: + maxLines: 1000 + derivedFields: + - name: TraceID + matcherType: label + matcherRegex: trace_id + datasourceUid: tempo + url: '$${__value.raw}' + urlDisplayLabel: View trace + + - name: Tempo + type: tempo + uid: tempo + access: proxy + url: http://tempo:3200 + editable: false + jsonData: + nodeGraph: + enabled: true + search: + hide: false + serviceMap: + datasourceUid: prometheus + tracesToLogsV2: + datasourceUid: loki + spanStartTimeShift: -5m + spanEndTimeShift: 5m + filterByTraceID: true + filterBySpanID: false + customQuery: true + query: '{service_name="$${__span.tags["service.name"]}"} | trace_id = "$${__trace.traceId}"' + tags: + - key: service.name + value: service_name + tracesToMetrics: + datasourceUid: prometheus + spanStartTimeShift: -5m + spanEndTimeShift: 5m + tags: + - key: service.name + value: service + queries: + - name: Request rate + query: 'sum(rate(traces_spanmetrics_calls_total{$$__tags}[5m]))' + - name: Error rate + query: 'sum(rate(traces_spanmetrics_calls_total{$$__tags,span_status="STATUS_CODE_ERROR"}[5m]))' diff --git a/monitoring/grafana/provisioning/plugins/.gitkeep b/monitoring/grafana/provisioning/plugins/.gitkeep new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/monitoring/grafana/provisioning/plugins/.gitkeep @@ -0,0 +1 @@ + diff --git a/monitoring/postgres-queries.yml b/monitoring/postgres-queries.yml new file mode 100644 index 0000000..52fcfe1 --- /dev/null +++ b/monitoring/postgres-queries.yml @@ -0,0 +1,97 @@ +messenger_attachment_jobs: + query: | + WITH statuses(status) AS ( + VALUES ('queued'), ('processing'), ('failed') + ) + SELECT + statuses.status, + COUNT(j.id)::bigint AS count, + COALESCE(EXTRACT(EPOCH FROM (clock_timestamp() - MIN(j.created_at))), 0)::double precision AS oldest_age_seconds + FROM statuses + LEFT JOIN attachment_jobs j ON j.status = statuses.status + GROUP BY statuses.status + metrics: + - status: + usage: LABEL + description: Attachment job status. + - count: + usage: GAUGE + description: Attachment jobs in this status. + - oldest_age_seconds: + usage: GAUGE + description: Age of the oldest attachment job in this status. + +messenger_outbox: + query: | + SELECT + COUNT(*) FILTER (WHERE published_at IS NULL)::bigint AS pending, + COALESCE( + EXTRACT(EPOCH FROM (clock_timestamp() - MIN(created_at) FILTER (WHERE published_at IS NULL))), + 0 + )::double precision AS oldest_pending_age_seconds + FROM outbox_events + metrics: + - pending: + usage: GAUGE + description: Unpublished outbox events. + - oldest_pending_age_seconds: + usage: GAUGE + description: Age of the oldest unpublished outbox event. + +messenger_attachment_terminal_transitions: + query: | + WITH statuses(status) AS ( + VALUES ('ready'), ('failed') + ) + SELECT + statuses.status, + COUNT(events.job_id)::bigint AS total + FROM statuses + LEFT JOIN attachment_terminal_events events ON events.status = statuses.status + GROUP BY statuses.status + metrics: + - status: + usage: LABEL + description: First terminal status recorded for a durable attachment job. + - total: + usage: COUNTER + description: Durable attachment jobs whose first terminal outcome has this status. + +messenger_attachment_time_to_ready_seconds: + query: | + WITH bounds(le, upper_bound) AS ( + VALUES + ('0.05', 0.05::double precision), + ('0.1', 0.1::double precision), + ('0.25', 0.25::double precision), + ('0.5', 0.5::double precision), + ('1', 1::double precision), + ('2.5', 2.5::double precision), + ('5', 5::double precision), + ('10', 10::double precision), + ('15', 15::double precision), + ('20', 20::double precision), + ('30', 30::double precision), + ('60', 60::double precision), + ('+Inf', NULL::double precision) + ) + SELECT + bounds.le, + COUNT(events.job_id) FILTER ( + WHERE events.status = 'ready' + AND ( + bounds.upper_bound IS NULL + OR EXTRACT(EPOCH FROM (events.terminal_at - events.accepted_at)) <= bounds.upper_bound + ) + )::bigint AS bucket + FROM bounds + LEFT JOIN attachment_terminal_events events ON events.status = 'ready' + GROUP BY bounds.le, bounds.upper_bound + ORDER BY bounds.upper_bound NULLS LAST + metrics: + - le: + usage: LABEL + description: Upper bound of the durable time-to-ready bucket in seconds. + - bucket: + usage: COUNTER + description: Durable attachment jobs reaching ready within the bucket bound. diff --git a/monitoring/prometheus.yml b/monitoring/prometheus.yml new file mode 100644 index 0000000..70b5dfd --- /dev/null +++ b/monitoring/prometheus.yml @@ -0,0 +1,85 @@ +global: + scrape_interval: 2s + evaluation_interval: 5s + external_labels: + deployment_environment: lab + +rule_files: + - /etc/prometheus/rules/*.yml + +scrape_configs: + - job_name: messenger + dns_sd_configs: + - names: ["api"] + type: A + port: 8080 + refresh_interval: 2s + relabel_configs: + - target_label: service_name + replacement: messenger-api + metric_relabel_configs: + - source_labels: [__name__] + regex: "go_.*|process_.*" + action: keep + + - job_name: nginx + static_configs: + - targets: ["nginx-exporter:9113"] + + - job_name: postgres + static_configs: + - targets: ["postgres-exporter:9187"] + + - job_name: valkey + static_configs: + - targets: ["valkey-exporter:9121"] + + - job_name: redis-broker + static_configs: + - targets: ["redis-broker-exporter:9121"] + + - job_name: attachment-worker + dns_sd_configs: + - names: ["worker"] + type: A + port: 9091 + refresh_interval: 2s + relabel_configs: + - target_label: service_name + replacement: messenger-worker + metric_relabel_configs: + - source_labels: [__name__] + regex: "go_.*|process_.*" + action: keep + + - job_name: outbox-dispatcher + static_configs: + - targets: ["dispatcher:9091"] + relabel_configs: + - target_label: service_name + replacement: messenger-dispatcher + metric_relabel_configs: + - source_labels: [__name__] + regex: "go_.*|process_.*" + action: keep + + - job_name: minio + metrics_path: /minio/v2/metrics/cluster + static_configs: + - targets: ["s3:9000"] + + - job_name: otel-applications + static_configs: + - targets: ["otel-collector:8889"] + + - job_name: otel-collector + static_configs: + - targets: ["otel-collector:8888"] + + - job_name: tempo + static_configs: + - targets: ["tempo:3200"] + + - job_name: loki + static_configs: + - targets: ["loki:3100"] diff --git a/monitoring/rules/messenger.yml b/monitoring/rules/messenger.yml new file mode 100644 index 0000000..78a86dd --- /dev/null +++ b/monitoring/rules/messenger.yml @@ -0,0 +1,178 @@ +groups: + - name: messenger-api-slo + interval: 5s + rules: + - alert: MessengerAPIBusiness5xxHigh + expr: | + ( + (sum(rate(messenger_http_requests_total{route!~"/health/.*|/metrics",status_class="5xx"}[2m])) or vector(0)) + / + clamp_min((sum(rate(messenger_http_requests_total{route!~"/health/.*|/metrics"}[2m])) or vector(0)), 0.000001) + ) > 0.05 + for: 1m + labels: + severity: warning + service: messenger-api + slo: api-availability + annotations: + summary: Business API 5xx ratio is above 5% + description: More than 5% of business requests returned 5xx for one minute. Health and metrics routes are excluded. + runbook_url: https://gitea.privcloud.tech/dizi/mtusi2026lab6/src/branch/solution/bvt2303-plastinin/monitoring/runbooks/api-availability.md + + - alert: MessengerAPIHighErrorBudgetBurnFast + expr: | + ( + (sum(rate(messenger_http_requests_total{route!~"/health/.*|/metrics",status_class="5xx"}[5m])) or vector(0)) + / clamp_min((sum(rate(messenger_http_requests_total{route!~"/health/.*|/metrics"}[5m])) or vector(0)), 0.000001) + ) > 0.072 + and + ( + (sum(rate(messenger_http_requests_total{route!~"/health/.*|/metrics",status_class="5xx"}[1h])) or vector(0)) + / clamp_min((sum(rate(messenger_http_requests_total{route!~"/health/.*|/metrics"}[1h])) or vector(0)), 0.000001) + ) > 0.072 + for: 30s + labels: + severity: critical + service: messenger-api + slo: api-availability + annotations: + summary: API availability error budget is burning fast + description: The 99.5% API availability SLO burns above 14.4x in both 5m and 1h windows. + runbook_url: https://gitea.privcloud.tech/dizi/mtusi2026lab6/src/branch/solution/bvt2303-plastinin/monitoring/runbooks/api-availability.md + + - alert: MessengerAPIHighErrorBudgetBurnSlow + expr: | + ( + (sum(rate(messenger_http_requests_total{route!~"/health/.*|/metrics",status_class="5xx"}[30m])) or vector(0)) + / clamp_min((sum(rate(messenger_http_requests_total{route!~"/health/.*|/metrics"}[30m])) or vector(0)), 0.000001) + ) > 0.03 + and + ( + (sum(rate(messenger_http_requests_total{route!~"/health/.*|/metrics",status_class="5xx"}[6h])) or vector(0)) + / clamp_min((sum(rate(messenger_http_requests_total{route!~"/health/.*|/metrics"}[6h])) or vector(0)), 0.000001) + ) > 0.03 + for: 1m + labels: + severity: warning + service: messenger-api + slo: api-availability + annotations: + summary: API availability error budget is burning steadily + description: The 99.5% API availability SLO burns above 6x in both 30m and 6h windows. + runbook_url: https://gitea.privcloud.tech/dizi/mtusi2026lab6/src/branch/solution/bvt2303-plastinin/monitoring/runbooks/api-availability.md + + - name: messenger-attachment-slo + interval: 5s + rules: + - alert: AttachmentReadyHighErrorBudgetBurnFast + expr: | + ( + clamp_min( + (sum(rate(messenger_attachment_time_to_ready_seconds_bucket{le="+Inf"}[5m])) or vector(0)) + - (sum(rate(messenger_attachment_time_to_ready_seconds_bucket{le="30"}[5m])) or vector(0)), + 0 + ) + + (sum(rate(messenger_attachment_terminal_transitions_total{status="failed"}[5m])) or vector(0)) + ) + / clamp_min( + (sum(rate(messenger_attachment_time_to_ready_seconds_bucket{le="+Inf"}[5m])) or vector(0)) + + (sum(rate(messenger_attachment_terminal_transitions_total{status="failed"}[5m])) or vector(0)), + 0.000001 + ) > 0.144 + and + ( + clamp_min( + (sum(rate(messenger_attachment_time_to_ready_seconds_bucket{le="+Inf"}[1h])) or vector(0)) + - (sum(rate(messenger_attachment_time_to_ready_seconds_bucket{le="30"}[1h])) or vector(0)), + 0 + ) + + (sum(rate(messenger_attachment_terminal_transitions_total{status="failed"}[1h])) or vector(0)) + ) + / clamp_min( + (sum(rate(messenger_attachment_time_to_ready_seconds_bucket{le="+Inf"}[1h])) or vector(0)) + + (sum(rate(messenger_attachment_terminal_transitions_total{status="failed"}[1h])) or vector(0)), + 0.000001 + ) > 0.144 + for: 30s + labels: + severity: critical + service: messenger-worker + slo: attachment-ready + annotations: + summary: Attachment ready-time error budget is burning fast + description: The 99% within 30s ready-time SLO burns above 14.4x in both 5m and 1h windows. + runbook_url: https://gitea.privcloud.tech/dizi/mtusi2026lab6/src/branch/solution/bvt2303-plastinin/monitoring/runbooks/attachment-pipeline.md + + - alert: AttachmentReadyHighErrorBudgetBurnSlow + expr: | + ( + clamp_min( + (sum(rate(messenger_attachment_time_to_ready_seconds_bucket{le="+Inf"}[30m])) or vector(0)) + - (sum(rate(messenger_attachment_time_to_ready_seconds_bucket{le="30"}[30m])) or vector(0)), + 0 + ) + + (sum(rate(messenger_attachment_terminal_transitions_total{status="failed"}[30m])) or vector(0)) + ) + / clamp_min( + (sum(rate(messenger_attachment_time_to_ready_seconds_bucket{le="+Inf"}[30m])) or vector(0)) + + (sum(rate(messenger_attachment_terminal_transitions_total{status="failed"}[30m])) or vector(0)), + 0.000001 + ) > 0.06 + and + ( + clamp_min( + (sum(rate(messenger_attachment_time_to_ready_seconds_bucket{le="+Inf"}[6h])) or vector(0)) + - (sum(rate(messenger_attachment_time_to_ready_seconds_bucket{le="30"}[6h])) or vector(0)), + 0 + ) + + (sum(rate(messenger_attachment_terminal_transitions_total{status="failed"}[6h])) or vector(0)) + ) + / clamp_min( + (sum(rate(messenger_attachment_time_to_ready_seconds_bucket{le="+Inf"}[6h])) or vector(0)) + + (sum(rate(messenger_attachment_terminal_transitions_total{status="failed"}[6h])) or vector(0)), + 0.000001 + ) > 0.06 + for: 1m + labels: + severity: warning + service: messenger-worker + slo: attachment-ready + annotations: + summary: Attachment ready-time error budget is burning steadily + description: The 99% within 30s ready-time SLO burns above 6x in both 30m and 6h windows. + runbook_url: https://gitea.privcloud.tech/dizi/mtusi2026lab6/src/branch/solution/bvt2303-plastinin/monitoring/runbooks/attachment-pipeline.md + + - alert: AttachmentWorkerUnavailable + expr: (sum(up{job="attachment-worker"}) or vector(0)) < 1 + for: 30s + labels: + severity: critical + service: messenger-worker + annotations: + summary: No attachment worker is available + description: Prometheus has not observed a healthy worker target for 30 seconds. + runbook_url: https://gitea.privcloud.tech/dizi/mtusi2026lab6/src/branch/solution/bvt2303-plastinin/monitoring/runbooks/attachment-pipeline.md + + - alert: AttachmentQueueBacklogGrowing + expr: | + max(messenger_attachment_jobs_oldest_age_seconds{status="queued"}) > 30 + and max(deriv(messenger_attachment_jobs_oldest_age_seconds{status="queued"}[2m])) > 0 + for: 30s + labels: + severity: warning + service: messenger-worker + annotations: + summary: Attachment queue backlog is growing + description: The oldest queued attachment is older than 30 seconds and its age continues to increase. + runbook_url: https://gitea.privcloud.tech/dizi/mtusi2026lab6/src/branch/solution/bvt2303-plastinin/monitoring/runbooks/attachment-pipeline.md + + - alert: AttachmentRedisStreamLagHigh + expr: max(redis_stream_group_lag{job="redis-broker",stream="attachment-jobs",group="attachment-workers"}) > 20 + for: 1m + labels: + severity: warning + service: redis-broker + annotations: + summary: Redis Stream consumer lag is high + description: The attachment worker group has more than 20 messages waiting for delivery. + runbook_url: https://gitea.privcloud.tech/dizi/mtusi2026lab6/src/branch/solution/bvt2303-plastinin/monitoring/runbooks/attachment-pipeline.md diff --git a/monitoring/runbooks/api-availability.md b/monitoring/runbooks/api-availability.md new file mode 100644 index 0000000..2ff5453 --- /dev/null +++ b/monitoring/runbooks/api-availability.md @@ -0,0 +1,72 @@ +# API availability SLO + +## Цель и сигналы + +Объект контроля — бизнес-маршруты API за Nginx. Health endpoints исключены. Хорошими считаются ответы `2xx` и ожидаемые клиентские `4xx`; плохими — `5xx`. Цель — не менее 99,5% хороших ответов за скользящие 30 дней, error budget равен 0,5%. + +Основные сигналы: + +- `messenger_http_requests_total{method,route,status_class}` — число запросов; +- `messenger_http_request_duration_seconds_bucket{method,route}` — распределение server latency; +- `up{job="messenger"}` и `up{job="nginx"}` — доступность scrape targets; +- `messenger_db_pool_connections{state="used"}` и `rate(messenger_db_pool_waited_total[5m])` — использование пула и частота ожиданий свободного соединения; +- trace `messenger-api` — путь одного запроса по зависимостям; +- JSON-логи `messenger-api` с `request_id`, `trace_id` и `span_id`. + +`user_id`, `chat_id`, `message_id`, `attachment_id` и `request_id` не используются как labels метрик. + +## Диагностика: metrics → trace → logs + +1. В Grafana открыть dashboard **Messenger Lab 6 / API** и установить общий временной диапазон инцидента. +2. Проверить error ratio, RPS, p95/p99 и `up`. Рост `5xx` при нормальном RPS подтверждает влияние на SLI; отсутствие трафика не считается успешностью. +3. Сравнить DB pool used/waiting и состояние PostgreSQL. Рост ожидания пула указывает на зависимость, а не на CPU handler. +4. В панели ошибок выбрать exemplar или перейти в Explore → Tempo. Если exemplar отсутствует, выполнить TraceQL: + + ```text + { resource.service.name = "messenger-api" && span.http.route != "/health/ready" && status = error } + ``` + +5. В trace найти первый error span. Проверить `http.route`, `http.request.method`, `http.response.status_code`, `db.system`, duration и resource attributes. Полный URL, токен и тело запроса в span отсутствуют. +6. Скопировать `trace_id`, перейти по trace-to-logs или выполнить в Explore → Loki: + + ```text + {service_name="messenger-api"} | json | trace_id="" + ``` + +7. Сверить `request_id`, событие, severity и error class. Текст сообщений, `Authorization` и пароли в логах отсутствуют. + +## Проверка alert + +Состояние правил видно в Grafana Alerting и в Prometheus: + +```bash +curl -s http://127.0.0.1:9090/api/v1/alerts +``` + +Контролируемый отказ PostgreSQL: + +```bash +ENV_FILE=lab6.env.example scripts/incident-database.sh +``` + +Сценарий сохраняет baseline, нагрузку, состояние деградации, `alert-firing.json`, восстановление и `alert-resolved.json` в `evidence/runtime/`. Trap запускает PostgreSQL и API даже при ошибке сценария. Volumes не удаляются. + +Ожидаемая последовательность: + +```text +нормальный error ratio +→ PostgreSQL остановлен +→ 5xx растёт +→ MessengerAPIBusiness5xxHigh и MessengerAPIHighErrorBudgetBurnFast firing +→ PostgreSQL восстановлен +→ успешные ответы возвращаются +→ оперативный alert resolved; burn-rate хранит историю длинного окна +``` + +## Действия + +1. Если `up{job="nginx"}=0`, восстановить ingress и проверить его логи. +2. Если API targets недоступны, восстановить реплики по одной и проверить readiness. +3. Если API работает, а DB spans завершаются ошибкой или pool waiting растёт, восстановить PostgreSQL и не увеличивать число API-реплик до устранения причины. +4. После восстановления проверить успешный smoke, отсутствие firing alerts и возвращение latency к baseline. +5. Не удалять volumes и не сбрасывать очередь ради закрытия alert. diff --git a/monitoring/runbooks/attachment-pipeline.md b/monitoring/runbooks/attachment-pipeline.md new file mode 100644 index 0000000..4c7155f --- /dev/null +++ b/monitoring/runbooks/attachment-pipeline.md @@ -0,0 +1,86 @@ +# Attachment pipeline SLO + +## Цель и сигналы + +Объект контроля — первый terminal outcome каждой уникальной durable job. Цель — не менее 99% jobs впервые завершаются `ready` не позднее 30 секунд после acceptance за скользящие 30 дней. Первый `failed` и первое завершение `ready` позже 30 секунд расходуют error budget; replay не переписывает уже записанный outcome. + +Основные сигналы: + +- `messenger_attachment_worker_jobs_total{job_type,result}` — исходы доставок worker; +- `messenger_attachment_terminal_transitions_total{status}` — первые terminal outcomes из append-only PostgreSQL ledger; +- `messenger_attachment_worker_job_duration_seconds_bucket{job_type,result}` — время обработки; +- `messenger_attachment_time_to_ready_seconds_bucket` — end-to-end время; +- `messenger_attachment_jobs_count{status}` — состояния jobs в PostgreSQL; +- `messenger_attachment_jobs_oldest_age_seconds{status}` — возраст старейшей job; +- `messenger_attachment_worker_jobs_in_flight` и `messenger_attachment_worker_capacity` — насыщение worker; +- `messenger_outbox_pending` — задержка до публикации; +- Redis Stream lag и PEL — доставка и неподтверждённые сообщения. + +Queue depth не заменяет processing latency: очередь может быть короткой, но содержать одну очень старую job. + +## Диагностика: metrics → trace → logs + +1. В Grafana открыть dashboard **Messenger Lab 6 / Attachment pipeline** и выбрать диапазон инцидента. +2. Определить место задержки: + - растёт `messenger_outbox_pending` — проблема до публикации в broker; + - растут stream lag и `queued` — worker не успевает или недоступен; + - растут PEL и `processing` — delivery получена, но не завершена; + - растёт `failed` — постоянные ошибки или исчерпаны retry; + - busy равен capacity — worker насыщен. +3. В Explore → Tempo найти upload по route или worker consumer span: + + ```text + { resource.service.name = "messenger-api" && span.http.route = "/api/v1/chats/{chat_id}/attachments" } + ``` + + ```text + { resource.service.name = "messenger-worker" && name = "attachment.process" } + ``` + +4. В trace пройти цепочку `HTTP upload → outbox publish → Redis consume → S3 variants → PostgreSQL ready`. Проверить producer/consumer связь и retry spans. Технический `attachment_id` допустим только как span/log attribute синтетического стенда, но не как metric label. +5. Перейти из выбранного span в Loki или выполнить: + + ```text + {service_name="messenger-worker"} | json | trace_id="" + ``` + +6. Сверить в логе `event`, `job_type`, `attempt`, `result`, `trace_id` и `span_id`; в span проверить `job.outcome`. Payload envelope, изображение и credentials не логируются. + +## Проверка alert + +Короткая деградация: + +```bash +ENV_FILE=lab6.env.example scripts/incident-worker.sh short +``` + +Длительная деградация: + +```bash +ENV_FILE=lab6.env.example scripts/incident-worker.sh long +``` + +Ожидаемая последовательность: + +```text +worker up, backlog около нуля +→ worker остановлен +→ queued и oldest age растут +→ short: AttachmentWorkerUnavailable firing +→ long: AttachmentQueueBacklogGrowing firing +→ worker запущен +→ backlog дренируется +→ все принятые attachments достигают ready +→ alert resolved +``` + +Каждый сценарий восстанавливает worker через trap и сохраняет evidence без session token и raw payload. + +## Действия + +1. Если worker `up=0`, восстановить worker и наблюдать скорость дренирования. +2. Если worker доступен, но busy равно capacity, проверить S3/DB spans до масштабирования concurrency. +3. Если растёт outbox, проверить dispatcher и PostgreSQL lease; Redis worker здесь ещё не причина. +4. Если растёт PEL, проверить heartbeat, lease expiry и reclaim. +5. Для `failed` определить permanent/transient error по логу и trace. Replay выполнять только после устранения причины. +6. После восстановления проверить backlog, oldest age, `failed`, firing alerts и terminal state каждого синтетического attachment. diff --git a/nginx/nginx.conf b/nginx/nginx.conf new file mode 100644 index 0000000..d62dcd6 --- /dev/null +++ b/nginx/nginx.conf @@ -0,0 +1,57 @@ +user nginx; +worker_processes auto; +error_log /dev/stderr notice; +pid /var/run/nginx.pid; + +events { + worker_connections 1024; +} + +http { + resolver 127.0.0.11 valid=2s ipv6=off; + + log_format structured escape=json + '{"time":"$time_iso8601","request_id":"$http_x_request_id",' + '"method":"$request_method","uri":"$request_uri","status":$status,' + '"request_time":$request_time,"upstream":"$upstream_addr",' + '"upstream_status":"$upstream_status","upstream_time":"$upstream_response_time"}'; + + access_log /dev/stdout structured; + + upstream messenger { + zone messenger 64k; + include /etc/nginx/conf.d/upstream.conf; + keepalive 32; + } + + server { + listen 8080; + client_max_body_size 11m; + + location / { + proxy_pass http://messenger; + proxy_http_version 1.1; + proxy_set_header Connection ""; + proxy_set_header Host $host; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_request_buffering off; + + proxy_connect_timeout 1s; + proxy_send_timeout 10s; + proxy_read_timeout 10s; + proxy_next_upstream error timeout http_502 http_503 http_504; + proxy_next_upstream_tries 2; + proxy_next_upstream_timeout 3s; + } + } + + server { + listen 8081; + access_log off; + + location = /stub_status { + stub_status; + } + } +} diff --git a/nginx/upstream.conf b/nginx/upstream.conf new file mode 100644 index 0000000..f6131cb --- /dev/null +++ b/nginx/upstream.conf @@ -0,0 +1 @@ +server api:8080 resolve max_fails=1 fail_timeout=3s; diff --git a/scripts/incident-database.sh b/scripts/incident-database.sh new file mode 100755 index 0000000..bc3c6e0 --- /dev/null +++ b/scripts/incident-database.sh @@ -0,0 +1,91 @@ +#!/bin/sh + +set -eu + +env_file=${ENV_FILE:-lab6.env.example} +base_url=${BASE_URL:-http://127.0.0.1:8080} +prometheus_url=${PROMETHEUS_URL:-http://127.0.0.1:9090} +tempo_url=${TEMPO_URL:-http://127.0.0.1:3200} +loki_url=${LOKI_URL:-http://127.0.0.1:3100} +evidence_root=${EVIDENCE_DIR:-evidence/runtime} +outage_seconds=${DB_OUTAGE_SECONDS:-75} +require_alerts=${REQUIRE_ALERT_TRANSITIONS:-1} +api_alert_name=${API_ALERT_NAME:-MessengerAPIBusiness5xxHigh} +burn_alert_name=${API_BURN_ALERT_NAME:-MessengerAPIHighErrorBudgetBurnFast} +run_id=$(date -u +%Y%m%dT%H%M%SZ) +output_dir="$evidence_root/database-$run_id" +mkdir -p "$output_dir" +load_pid= + +compose() { + docker compose --env-file "$env_file" "$@" +} + +restore() { + if [ -n "$load_pid" ]; then + kill "$load_pid" >/dev/null 2>&1 || true + fi + compose up -d postgres migrate api nginx >/dev/null 2>&1 || true +} + +trap restore EXIT INT TERM + +python3 scripts/lab6-observe.py --prometheus-url "$prometheus_url" --phase baseline --output "$output_dir/baseline.json" >/dev/null +python3 scripts/lab6-traffic.py \ + --base-url "$base_url" \ + --mode messages \ + --duration "$((outage_seconds + 20))" \ + --concurrency 4 \ + --rate "${DB_LOAD_RATE:-20}" \ + --started-file "$output_dir/load-started" \ + --output "$output_dir/traffic.json" >/dev/null & +load_pid=$! +remaining=60 +while [ ! -f "$output_dir/load-started" ] && [ "$remaining" -gt 0 ]; do + sleep 0.5 + remaining=$((remaining - 1)) +done +test -f "$output_dir/load-started" +sleep 2 +started_at=$(date -u +%Y-%m-%dT%H:%M:%SZ) +compose stop postgres >/dev/null +sleep "$outage_seconds" +if [ "$require_alerts" = "1" ]; then + python3 scripts/lab6-alert.py --prometheus-url "$prometheus_url" --name "$api_alert_name" --state firing --timeout "${ALERT_WAIT_SECONDS:-90}" --output "$output_dir/alert-firing.json" >/dev/null + python3 scripts/lab6-alert.py --prometheus-url "$prometheus_url" --name "$burn_alert_name" --state firing --timeout "${ALERT_WAIT_SECONDS:-90}" --output "$output_dir/burn-alert-firing.json" >/dev/null +fi +python3 scripts/lab6-correlation.py --mode database --tempo-url "$tempo_url" --loki-url "$loki_url" --output "$output_dir/database-correlation.json" --require >/dev/null +python3 scripts/lab6-observe.py --prometheus-url "$prometheus_url" --phase degraded --output "$output_dir/degraded.json" >/dev/null +recovery_started_at=$(date -u +%Y-%m-%dT%H:%M:%SZ) +compose up -d --wait --wait-timeout 90 postgres migrate api nginx >/dev/null +wait "$load_pid" || true +load_pid= +if [ "$require_alerts" = "1" ]; then + python3 scripts/lab6-alert.py --prometheus-url "$prometheus_url" --name "$api_alert_name" --state resolved --timeout "${ALERT_RESOLVE_SECONDS:-180}" --output "$output_dir/alert-resolved.json" >/dev/null +fi +sleep "${RECOVERY_OBSERVATION_SECONDS:-20}" +python3 scripts/lab6-observe.py --prometheus-url "$prometheus_url" --phase recovered --output "$output_dir/recovered.json" >/dev/null +recovered_at=$(date -u +%Y-%m-%dT%H:%M:%SZ) +python3 - "$output_dir/summary.json" "$started_at" "$recovery_started_at" "$recovered_at" "$outage_seconds" <<'PY' +import datetime +import json +import pathlib +import sys + +output, started, recovery_started, recovered, duration = sys.argv[1:] +parse = lambda value: datetime.datetime.fromisoformat(value.replace("Z", "+00:00")) +report = { + "schema_version": 1, + "incident": "postgres_unavailable", + "started_at": started, + "recovery_started_at": recovery_started, + "recovered_at": recovered, + "planned_injection_seconds": int(duration), + "injected_duration_seconds": int((parse(recovery_started) - parse(started)).total_seconds()), + "recovery_observation_seconds": int((parse(recovered) - parse(recovery_started)).total_seconds()), + "artifacts": ["baseline.json", "degraded.json", "recovered.json", "traffic.json", "database-correlation.json", "alert-firing.json", "alert-resolved.json", "burn-alert-firing.json"], +} +pathlib.Path(output).write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") +PY +trap - EXIT INT TERM +printf '%s\n' "$output_dir" diff --git a/scripts/incident-telemetry.sh b/scripts/incident-telemetry.sh new file mode 100755 index 0000000..1f0cc50 --- /dev/null +++ b/scripts/incident-telemetry.sh @@ -0,0 +1,73 @@ +#!/bin/sh + +set -eu + +env_file=${ENV_FILE:-lab6.env.example} +base_url=${BASE_URL:-http://127.0.0.1:8080} +prometheus_url=${PROMETHEUS_URL:-http://127.0.0.1:9090} +evidence_root=${EVIDENCE_DIR:-evidence/runtime} +outage_seconds=${TELEMETRY_OUTAGE_SECONDS:-30} +run_id=$(date -u +%Y%m%dT%H%M%SZ) +output_dir="$evidence_root/telemetry-$run_id" +mkdir -p "$output_dir" +load_pid= + +compose() { + docker compose --env-file "$env_file" "$@" +} + +restore() { + if [ -n "$load_pid" ]; then + kill "$load_pid" >/dev/null 2>&1 || true + fi + compose up -d tempo loki >/dev/null 2>&1 || true +} + +trap restore EXIT INT TERM + +python3 scripts/lab6-observe.py --prometheus-url "$prometheus_url" --phase baseline --output "$output_dir/baseline.json" >/dev/null +started_at=$(date -u +%Y-%m-%dT%H:%M:%SZ) +compose stop tempo loki >/dev/null +python3 scripts/lab6-traffic.py \ + --base-url "$base_url" \ + --mode mixed \ + --duration "$outage_seconds" \ + --concurrency 4 \ + --rate "${TELEMETRY_LOAD_RATE:-10}" \ + --wait-ready \ + --ready-timeout 120 \ + --output "$output_dir/traffic.json" >/dev/null & +load_pid=$! +sleep "$outage_seconds" +wait "$load_pid" +load_pid= +health_code=$(curl --silent --output /dev/null --write-out '%{http_code}' "$base_url/health/ready") +python3 scripts/lab6-observe.py --prometheus-url "$prometheus_url" --phase degraded --output "$output_dir/degraded.json" >/dev/null +recovery_started_at=$(date -u +%Y-%m-%dT%H:%M:%SZ) +compose up -d --wait --wait-timeout 90 tempo loki >/dev/null +sleep "${RECOVERY_OBSERVATION_SECONDS:-20}" +python3 scripts/lab6-observe.py --prometheus-url "$prometheus_url" --phase recovered --output "$output_dir/recovered.json" >/dev/null +recovered_at=$(date -u +%Y-%m-%dT%H:%M:%SZ) +python3 - "$output_dir/summary.json" "$started_at" "$recovery_started_at" "$recovered_at" "$outage_seconds" "$health_code" <<'PY' +import datetime +import json +import pathlib +import sys + +output, started, recovery_started, recovered, duration, health_code = sys.argv[1:] +parse = lambda value: datetime.datetime.fromisoformat(value.replace("Z", "+00:00")) +report = { + "schema_version": 1, + "incident": "telemetry_backends_unavailable", + "started_at": started, + "recovery_started_at": recovery_started, + "recovered_at": recovered, + "injected_duration_seconds": int(duration), + "api_ready_http_status_during_outage": int(health_code), + "recovery_observation_seconds": int((parse(recovered) - parse(recovery_started)).total_seconds()), + "artifacts": ["baseline.json", "degraded.json", "recovered.json", "traffic.json"], +} +pathlib.Path(output).write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") +PY +trap - EXIT INT TERM +printf '%s\n' "$output_dir" diff --git a/scripts/incident-worker.sh b/scripts/incident-worker.sh new file mode 100755 index 0000000..016df05 --- /dev/null +++ b/scripts/incident-worker.sh @@ -0,0 +1,107 @@ +#!/bin/sh + +set -eu + +mode=${1:-short} +env_file=${ENV_FILE:-lab6.env.example} +base_url=${BASE_URL:-http://127.0.0.1:8080} +prometheus_url=${PROMETHEUS_URL:-http://127.0.0.1:9090} +evidence_root=${EVIDENCE_DIR:-evidence/runtime} +require_alerts=${REQUIRE_ALERT_TRANSITIONS:-1} +case "$mode" in + short) + incident_seconds=${SHORT_WORKER_OUTAGE_SECONDS:-75} + load_seconds=${SHORT_WORKER_LOAD_SECONDS:-60} + requests=${SHORT_WORKER_REQUESTS:-300} + default_alert=AttachmentWorkerUnavailable + ;; + long) + incident_seconds=${LONG_WORKER_OUTAGE_SECONDS:-180} + load_seconds=${LONG_WORKER_LOAD_SECONDS:-120} + requests=${LONG_WORKER_REQUESTS:-1000} + default_alert=AttachmentQueueBacklogGrowing + ;; + *) + echo "usage: $0 [short|long]" >&2 + exit 2 + ;; +esac +worker_alert_name=${WORKER_ALERT_NAME:-$default_alert} + +run_id=$(date -u +%Y%m%dT%H%M%SZ) +output_dir="$evidence_root/worker-$mode-$run_id" +mkdir -p "$output_dir" +load_pid= + +compose() { + docker compose --env-file "$env_file" "$@" +} + +restore() { + if [ -n "$load_pid" ]; then + kill "$load_pid" >/dev/null 2>&1 || true + fi + compose up -d worker >/dev/null 2>&1 || true +} + +trap restore EXIT INT TERM + +python3 scripts/lab6-observe.py --prometheus-url "$prometheus_url" --phase baseline --output "$output_dir/baseline.json" >/dev/null +started_at=$(date -u +%Y-%m-%dT%H:%M:%SZ) +compose stop worker >/dev/null +python3 scripts/lab6-traffic.py \ + --base-url "$base_url" \ + --mode attachments \ + --duration "$load_seconds" \ + --requests "$requests" \ + --concurrency 4 \ + --rate "${WORKER_LOAD_RATE:-5}" \ + --wait-ready \ + --ready-timeout 480 \ + --output "$output_dir/traffic.json" >/dev/null & +load_pid=$! +sleep "$incident_seconds" +if [ "$require_alerts" = "1" ]; then + python3 scripts/lab6-alert.py --prometheus-url "$prometheus_url" --name "$worker_alert_name" --state firing --timeout "${ALERT_WAIT_SECONDS:-90}" --output "$output_dir/alert-firing.json" >/dev/null +fi +python3 scripts/lab6-observe.py --prometheus-url "$prometheus_url" --phase degraded --output "$output_dir/degraded.json" >/dev/null +recovery_started_at=$(date -u +%Y-%m-%dT%H:%M:%SZ) +compose up -d --wait --wait-timeout 90 worker >/dev/null +wait "$load_pid" +load_pid= +if [ "$require_alerts" = "1" ]; then + python3 scripts/lab6-alert.py --prometheus-url "$prometheus_url" --name "$worker_alert_name" --state resolved --timeout "${ALERT_RESOLVE_SECONDS:-180}" --output "$output_dir/alert-resolved.json" >/dev/null + if [ "$mode" = "long" ]; then + python3 scripts/lab6-alert.py --prometheus-url "$prometheus_url" --name AttachmentReadyHighErrorBudgetBurnFast --state firing --timeout "${ALERT_WAIT_SECONDS:-90}" --output "$output_dir/burn-fast-firing.json" >/dev/null + python3 scripts/lab6-alert.py --prometheus-url "$prometheus_url" --name AttachmentReadyHighErrorBudgetBurnSlow --state firing --timeout "${ALERT_WAIT_SECONDS:-90}" --output "$output_dir/burn-slow-firing.json" >/dev/null + fi +fi +sleep "${RECOVERY_OBSERVATION_SECONDS:-20}" +python3 scripts/lab6-observe.py --prometheus-url "$prometheus_url" --phase recovered --output "$output_dir/recovered.json" >/dev/null +recovered_at=$(date -u +%Y-%m-%dT%H:%M:%SZ) +python3 - "$output_dir/summary.json" "$mode" "$started_at" "$recovery_started_at" "$recovered_at" "$incident_seconds" <<'PY' +import datetime +import json +import pathlib +import sys + +output, mode, started, recovery_started, recovered, duration = sys.argv[1:] +parse = lambda value: datetime.datetime.fromisoformat(value.replace("Z", "+00:00")) +report = { + "schema_version": 1, + "incident": "worker_unavailable", + "profile": mode, + "started_at": started, + "recovery_started_at": recovery_started, + "recovered_at": recovered, + "planned_injection_seconds": int(duration), + "injected_duration_seconds": int((parse(recovery_started) - parse(started)).total_seconds()), + "recovery_observation_seconds": int((parse(recovered) - parse(recovery_started)).total_seconds()), + "artifacts": ["baseline.json", "degraded.json", "recovered.json", "traffic.json", "alert-firing.json", "alert-resolved.json"], +} +if mode == "long": + report["artifacts"].extend(["burn-fast-firing.json", "burn-slow-firing.json"]) +pathlib.Path(output).write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") +PY +trap - EXIT INT TERM +printf '%s\n' "$output_dir" diff --git a/scripts/lab6-alert.py b/scripts/lab6-alert.py new file mode 100755 index 0000000..1e75ce9 --- /dev/null +++ b/scripts/lab6-alert.py @@ -0,0 +1,67 @@ +#!/usr/bin/env python3 + +import argparse +import json +from pathlib import Path +import time +import urllib.error +import urllib.request + + +def fetch(prometheus_url): + url = prometheus_url.rstrip("/") + "/api/v1/alerts" + with urllib.request.urlopen(url, timeout=10) as response: + payload = json.loads(response.read(4 << 20).decode("utf-8")) + return payload.get("data", {}).get("alerts", []) + + +def matching(alerts, name): + return [alert for alert in alerts if alert.get("labels", {}).get("alertname") == name] + + +def main(): + parser = argparse.ArgumentParser(description="Wait for a Prometheus alert transition") + parser.add_argument("--prometheus-url", default="http://127.0.0.1:9090") + parser.add_argument("--name", required=True) + parser.add_argument("--state", choices=("firing", "resolved"), required=True) + parser.add_argument("--timeout", type=float, default=120) + parser.add_argument("--output", required=True) + args = parser.parse_args() + + deadline = time.monotonic() + args.timeout + last = [] + error = None + observed = False + while time.monotonic() < deadline: + try: + last = matching(fetch(args.prometheus_url), args.name) + error = None + if args.state == "firing" and any(item.get("state") == "firing" for item in last): + observed = True + break + if args.state == "resolved" and not any(item.get("state") in ("pending", "firing") for item in last): + observed = True + break + except (urllib.error.URLError, json.JSONDecodeError) as current_error: + error = str(current_error) + time.sleep(2) + + report = { + "schema_version": 1, + "alert": args.name, + "expected_state": args.state, + "observed": observed, + "observed_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), + "last_states": sorted({item.get("state", "") for item in last}), + "error": error, + } + destination = Path(args.output) + destination.parent.mkdir(parents=True, exist_ok=True) + destination.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + print(json.dumps(report, ensure_ascii=False, indent=2)) + if not observed: + raise SystemExit(1) + + +if __name__ == "__main__": + main() diff --git a/scripts/lab6-correlation.py b/scripts/lab6-correlation.py new file mode 100755 index 0000000..9b981a7 --- /dev/null +++ b/scripts/lab6-correlation.py @@ -0,0 +1,148 @@ +#!/usr/bin/env python3 + +import argparse +import json +from pathlib import Path +import re +import time +import urllib.error +import urllib.parse +import urllib.request + +TRACE_ID = re.compile(r'(?i)\b[0-9a-f]{32}\b') +SERVICES = ("messenger-api", "messenger-worker", "messenger-dispatcher") + + +def get_json(url): + with urllib.request.urlopen(url, timeout=10) as response: + return json.loads(response.read(8 << 20).decode("utf-8")) + + +def tempo_query(base_url, query): + url = base_url.rstrip("/") + "/api/search?" + urllib.parse.urlencode({"q": query, "limit": 100}) + payload = get_json(url) + return {item.get("traceID") for item in payload.get("traces", []) if item.get("traceID")} + + +def extract_trace_ids(payload): + result = set() + for stream in payload.get("data", {}).get("result", []): + for value in stream.get("values", []): + candidates = [stream.get("stream", {}), value[1] if len(value) > 1 else ""] + if len(value) > 2: + candidates.append(value[2]) + for candidate in candidates: + if isinstance(candidate, dict): + trace_id = candidate.get("trace_id") or candidate.get("traceid") + if trace_id: + result.add(str(trace_id)) + candidate = json.dumps(candidate) + result.update(match.group(0).lower() for match in TRACE_ID.finditer(str(candidate))) + return result + + +def loki_logs(base_url, service, start_ns): + query = '{service_name="' + service + '"}' + url = base_url.rstrip("/") + "/loki/api/v1/query_range?" + urllib.parse.urlencode({ + "query": query, + "start": str(start_ns), + "limit": 5000, + "direction": "backward", + }) + payload = get_json(url) + count = sum(len(stream.get("values", [])) for stream in payload.get("data", {}).get("result", [])) + return count, extract_trace_ids(payload) + + +def capture(tempo_url, loki_url, lookback_seconds): + start_ns = int((time.time() - lookback_seconds) * 1_000_000_000) + traces = {service: tempo_query(tempo_url, '{ resource.service.name = "' + service + '" }') for service in SERVICES} + logs = {} + log_trace_ids = {} + for service in SERVICES: + count, ids = loki_logs(loki_url, service, start_ns) + logs[service] = count + log_trace_ids[service] = ids + api_worker = traces["messenger-api"] & traces["messenger-worker"] + all_trace_ids = set().union(*traces.values()) + all_log_trace_ids = set().union(*log_trace_ids.values()) + return { + "traces_by_service": {service: len(ids) for service, ids in traces.items()}, + "logs_by_service": logs, + "api_worker_shared_traces": len(api_worker), + "traces_with_correlated_logs": len(all_trace_ids & all_log_trace_ids), + } + + +def capture_database(tempo_url, loki_url, lookback_seconds): + start_ns = int((time.time() - lookback_seconds) * 1_000_000_000) + traces = tempo_query( + tempo_url, + '{ resource.service.name = "messenger-api" && span.db.system = "postgresql" && status = error }', + ) + log_count, log_trace_ids = loki_logs(loki_url, "messenger-api", start_ns) + return { + "database_error_traces": len(traces), + "api_logs": log_count, + "database_error_traces_with_logs": len(traces & log_trace_ids), + } + + +def complete(report): + return ( + report["traces_by_service"]["messenger-api"] > 0 + and report["traces_by_service"]["messenger-worker"] > 0 + and report["logs_by_service"]["messenger-api"] > 0 + and report["logs_by_service"]["messenger-worker"] > 0 + and report["api_worker_shared_traces"] > 0 + and report["traces_with_correlated_logs"] > 0 + ) + + +def database_complete(report): + return report["database_error_traces"] > 0 and report["database_error_traces_with_logs"] > 0 + + +def main(): + parser = argparse.ArgumentParser(description="Verify trace and log correlation in Tempo and Loki") + parser.add_argument("--tempo-url", default="http://127.0.0.1:3200") + parser.add_argument("--loki-url", default="http://127.0.0.1:3100") + parser.add_argument("--lookback", type=float, default=900) + parser.add_argument("--wait", type=float, default=45) + parser.add_argument("--output", required=True) + parser.add_argument("--require", action="store_true") + parser.add_argument("--mode", choices=("baseline", "database"), default="baseline") + args = parser.parse_args() + + deadline = time.monotonic() + args.wait + error = None + report = None + while time.monotonic() < deadline: + try: + report = capture_database(args.tempo_url, args.loki_url, args.lookback) if args.mode == "database" else capture(args.tempo_url, args.loki_url, args.lookback) + error = None + if database_complete(report) if args.mode == "database" else complete(report): + break + except (urllib.error.URLError, json.JSONDecodeError, KeyError) as current_error: + error = str(current_error) + time.sleep(2) + + result = { + "schema_version": 1, + "captured_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), + "lookback_seconds": args.lookback, + "signals": report, + "mode": args.mode, + "complete": bool(report and (database_complete(report) if args.mode == "database" else complete(report))), + "error": error, + } + destination = Path(args.output) + destination.parent.mkdir(parents=True, exist_ok=True) + destination.write_text(json.dumps(result, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + print(json.dumps(result, ensure_ascii=False, indent=2)) + if args.require and not result["complete"]: + raise SystemExit(1) + + +if __name__ == "__main__": + main() diff --git a/scripts/lab6-observe.py b/scripts/lab6-observe.py new file mode 100644 index 0000000..6ff2489 --- /dev/null +++ b/scripts/lab6-observe.py @@ -0,0 +1,91 @@ +#!/usr/bin/env python3 + +import argparse +import json +from pathlib import Path +import time +import urllib.error +import urllib.parse +import urllib.request + +QUERIES = { + "api_error_ratio_1m": 'sum(rate(messenger_http_requests_total{status_class="5xx",route!~"/health/.*"}[1m])) / clamp_min(sum(rate(messenger_http_requests_total{route!~"/health/.*"}[1m])), 1)', + "api_request_rate_1m": 'sum(rate(messenger_http_requests_total{route!~"/health/.*"}[1m]))', + "queued_jobs": 'sum(messenger_attachment_jobs_count{status="queued"})', + "processing_jobs": 'sum(messenger_attachment_jobs_count{status="processing"})', + "oldest_queued_job_seconds": 'max(messenger_attachment_jobs_oldest_age_seconds{status="queued"})', + "worker_jobs_in_flight": 'sum(messenger_attachment_worker_jobs_in_flight)', + "worker_capacity": 'sum(messenger_attachment_worker_capacity)', + "pending_outbox": 'sum(messenger_outbox_pending)', + "worker_up": 'min(up{job="attachment-worker"})', + "collector_queue_size": 'max by (exporter) (otelcol_exporter_queue_size)', + "collector_queue_capacity": 'max by (exporter) (otelcol_exporter_queue_capacity)', + "collector_export_failures_total": 'sum by (exporter) (otelcol_exporter_send_failed_spans) or sum by (exporter) (otelcol_exporter_send_failed_log_records) or on (exporter) (0 * max by (exporter) (otelcol_exporter_queue_capacity))', + "collector_dropped_records_total": 'sum by (exporter) (otelcol_exporter_enqueue_failed_spans) or sum by (exporter) (otelcol_exporter_enqueue_failed_log_records) or on (exporter) (0 * max by (exporter) (otelcol_exporter_queue_capacity))', + "collector_memory_bytes": 'max(otelcol_process_memory_rss) or max(process_resident_memory_bytes{job="otel-collector"})', + "collector_memory_peak_bytes": 'max(max_over_time(otelcol_process_memory_rss[2m])) or max(max_over_time(process_resident_memory_bytes{job="otel-collector"}[2m]))', +} + + +def get_json(url): + try: + with urllib.request.urlopen(url, timeout=10) as response: + return json.loads(response.read(4 << 20).decode("utf-8")), None + except (urllib.error.URLError, json.JSONDecodeError) as error: + return None, str(error) + + +def query(prometheus_url, expression): + url = prometheus_url.rstrip("/") + "/api/v1/query?" + urllib.parse.urlencode({"query": expression}) + payload, error = get_json(url) + if error: + return {"error": error} + if payload.get("status") != "success": + return {"error": payload.get("error", "query failed")} + rows = payload.get("data", {}).get("result", []) + return { + "series": [ + {"labels": row.get("metric", {}), "value": row.get("value", [None, None])[1]} + for row in rows + ] + } + + +def alerts(prometheus_url): + payload, error = get_json(prometheus_url.rstrip("/") + "/api/v1/alerts") + if error: + return {"error": error, "items": []} + items = [] + for alert in payload.get("data", {}).get("alerts", []): + labels = alert.get("labels", {}) + items.append({ + "name": labels.get("alertname", ""), + "state": alert.get("state", ""), + "severity": labels.get("severity", ""), + "active_at": alert.get("activeAt", ""), + }) + return {"items": sorted(items, key=lambda item: (item["name"], item["state"]))} + + +def main(): + parser = argparse.ArgumentParser(description="Capture a bounded Prometheus snapshot for Lab 6 evidence") + parser.add_argument("--prometheus-url", default="http://127.0.0.1:9090") + parser.add_argument("--phase", required=True) + parser.add_argument("--output", required=True) + args = parser.parse_args() + + report = { + "schema_version": 1, + "captured_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), + "phase": args.phase, + "queries": {name: query(args.prometheus_url, expression) for name, expression in QUERIES.items()}, + "alerts": alerts(args.prometheus_url), + } + destination = Path(args.output) + destination.parent.mkdir(parents=True, exist_ok=True) + destination.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + print(json.dumps(report, ensure_ascii=False, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/scripts/lab6-traffic.py b/scripts/lab6-traffic.py new file mode 100755 index 0000000..244d53b --- /dev/null +++ b/scripts/lab6-traffic.py @@ -0,0 +1,261 @@ +#!/usr/bin/env python3 + +import argparse +import base64 +from concurrent.futures import ThreadPoolExecutor +import json +import math +from pathlib import Path +import secrets +import struct +import threading +import time +import urllib.error +import urllib.parse +import urllib.request +import zlib + +MAX_RESPONSE_BYTES = 1 << 20 + + +def png_fixture(width=96, height=64): + rows = [] + for y in range(height): + pixels = bytearray() + for x in range(width): + pixels.extend(((x * 3) % 256, (y * 5) % 256, (x + y) % 256)) + rows.append(b"\x00" + bytes(pixels)) + raw = b"".join(rows) + + def chunk(name, payload): + return struct.pack(">I", len(payload)) + name + payload + struct.pack(">I", zlib.crc32(name + payload) & 0xFFFFFFFF) + + header = struct.pack(">IIBBBBB", width, height, 8, 2, 0, 0, 0) + return b"\x89PNG\r\n\x1a\n" + chunk(b"IHDR", header) + chunk(b"IDAT", zlib.compress(raw, 6)) + chunk(b"IEND", b"") + + +def request(method, url, body=None, headers=None, timeout=10): + req = urllib.request.Request(url, data=body, method=method, headers=headers or {}) + try: + with urllib.request.urlopen(req, timeout=timeout) as response: + payload = response.read(MAX_RESPONSE_BYTES) + return response.status, payload + except urllib.error.HTTPError as error: + return error.code, error.read(MAX_RESPONSE_BYTES) + except urllib.error.URLError: + return 0, b"" + + +def decode_json(payload): + return json.loads(payload.decode("utf-8")) + + +def json_request(method, url, value=None, token=None, headers=None): + request_headers = {"Accept": "application/json"} + body = None + if value is not None: + body = json.dumps(value, separators=(",", ":")).encode("utf-8") + request_headers["Content-Type"] = "application/json" + if token: + request_headers["Authorization"] = "Bearer " + token + if headers: + request_headers.update(headers) + status, payload = request(method, url, body, request_headers) + return status, decode_json(payload) if payload else {} + + +def bootstrap(base_url): + suffix = secrets.token_hex(6) + password = "Lab6-" + secrets.token_urlsafe(18) + users = [] + for prefix in ("telemetry_a_", "telemetry_b_"): + username = prefix + suffix + status, user = json_request("POST", base_url + "/api/v1/users", { + "username": username, + "display_name": "Synthetic telemetry load", + "password": password, + }) + if status != 201: + raise RuntimeError("register user returned HTTP %d" % status) + users.append((username, user["id"])) + + credentials = base64.b64encode((users[0][0] + ":" + password).encode("utf-8")).decode("ascii") + status, payload = request("POST", base_url + "/api/v1/auth/sessions", headers={ + "Authorization": "Basic " + credentials, + "Accept": "application/json", + }) + if status != 201: + raise RuntimeError("create session returned HTTP %d" % status) + token = decode_json(payload)["token"] + + status, chat = json_request("POST", base_url + "/api/v1/chats", { + "title": "Lab 6 synthetic telemetry", + "member_ids": [users[0][1], users[1][1]], + }, token) + if status != 201: + raise RuntimeError("create chat returned HTTP %d" % status) + return token, chat["id"] + + +class Results: + def __init__(self): + self.lock = threading.Lock() + self.latencies = [] + self.statuses = {} + self.attachment_ids = [] + + def record(self, status, elapsed, attachment_id=None): + with self.lock: + self.latencies.append(elapsed) + key = str(status) + self.statuses[key] = self.statuses.get(key, 0) + 1 + if attachment_id: + self.attachment_ids.append(attachment_id) + + +def percentile(values, quantile): + if not values: + return None + ordered = sorted(values) + index = max(0, math.ceil(quantile * len(ordered)) - 1) + return round(ordered[index] * 1000, 3) + + +def run_request(base_url, token, chat_id, mode, sequence, fixture): + if mode == "mixed": + mode = "attachments" if sequence % 3 == 0 else "messages" + started = time.monotonic() + if mode == "messages": + status, _ = json_request("POST", base_url + "/api/v1/chats/" + chat_id + "/messages", { + "text": "synthetic telemetry event", + "attachment_ids": [], + }, token, {"Idempotency-Key": secrets.token_hex(16)}) + return status, time.monotonic() - started, None + + upload_url = base_url + "/api/v1/chats/" + chat_id + "/attachments?" + urllib.parse.urlencode({ + "filename": "synthetic.png", + }) + status, payload = request("POST", upload_url, fixture, { + "Authorization": "Bearer " + token, + "Content-Type": "image/png", + "Accept": "application/json", + }) + attachment_id = None + if status == 202: + attachment_id = decode_json(payload).get("id") + return status, time.monotonic() - started, attachment_id + + +def wait_for_attachments(base_url, token, attachment_ids, timeout): + remaining = set(attachment_ids) + states = {"ready": 0, "failed": 0, "timeout": 0} + deadline = time.monotonic() + timeout + while remaining and time.monotonic() < deadline: + for attachment_id in list(remaining): + status, metadata = json_request("GET", base_url + "/api/v1/attachments/" + attachment_id, token=token) + if status != 200: + continue + state = metadata.get("status") + if state in ("ready", "failed"): + states[state] += 1 + remaining.remove(attachment_id) + if remaining: + time.sleep(0.25) + states["timeout"] = len(remaining) + return states + + +def main(): + parser = argparse.ArgumentParser(description="Bounded synthetic load for Lab 6 telemetry and incidents") + parser.add_argument("--base-url", default="http://127.0.0.1:8080") + parser.add_argument("--mode", choices=("messages", "attachments", "mixed"), default="mixed") + parser.add_argument("--duration", type=float, default=30) + parser.add_argument("--requests", type=int, default=0, help="Maximum requests; zero means duration only") + parser.add_argument("--concurrency", type=int, default=4) + parser.add_argument("--rate", type=float, default=0, help="Approximate aggregate request rate; zero is unthrottled") + parser.add_argument("--wait-ready", action="store_true") + parser.add_argument("--ready-timeout", type=float, default=120) + parser.add_argument("--output") + parser.add_argument("--started-file", help="Touch after synthetic session and chat are ready") + args = parser.parse_args() + + if args.duration <= 0 or args.duration > 1800: + parser.error("duration must be in (0, 1800]") + if args.requests < 0: + parser.error("requests must be non-negative") + if args.concurrency < 1 or args.concurrency > 64: + parser.error("concurrency must be in [1, 64]") + if args.rate < 0 or args.rate > 1000: + parser.error("rate must be in [0, 1000]") + + base_url = args.base_url.rstrip("/") + token, chat_id = bootstrap(base_url) + if args.started_file: + started_file = Path(args.started_file) + started_file.parent.mkdir(parents=True, exist_ok=True) + started_file.touch() + fixture = png_fixture() + results = Results() + next_sequence = 0 + sequence_lock = threading.Lock() + deadline = time.monotonic() + args.duration + + def worker(): + nonlocal next_sequence + while time.monotonic() < deadline: + with sequence_lock: + if args.requests and next_sequence >= args.requests: + return + sequence = next_sequence + next_sequence += 1 + status, elapsed, attachment_id = run_request(base_url, token, chat_id, args.mode, sequence, fixture) + results.record(status, elapsed, attachment_id) + if args.rate: + delay = args.concurrency / args.rate - elapsed + if delay > 0: + time.sleep(delay) + + started = time.monotonic() + with ThreadPoolExecutor(max_workers=args.concurrency) as pool: + futures = [pool.submit(worker) for _ in range(args.concurrency)] + for future in futures: + future.result() + elapsed = time.monotonic() - started + + attachment_states = None + if args.wait_ready and results.attachment_ids: + attachment_states = wait_for_attachments(base_url, token, results.attachment_ids, args.ready_timeout) + + total = len(results.latencies) + successes = sum(count for status, count in results.statuses.items() if status.startswith("2")) + report = { + "schema_version": 1, + "workload": args.mode, + "started_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime(time.time() - elapsed)), + "duration_seconds": round(elapsed, 3), + "concurrency": args.concurrency, + "requested_rate": args.rate or None, + "requests": total, + "successful_requests": successes, + "request_rate": round(total / elapsed, 3) if elapsed else None, + "status_counts": dict(sorted(results.statuses.items())), + "latency_ms": { + "p50": percentile(results.latencies, 0.50), + "p95": percentile(results.latencies, 0.95), + "p99": percentile(results.latencies, 0.99), + }, + "attachments_accepted": len(results.attachment_ids), + "attachment_terminal_states": attachment_states, + } + encoded = json.dumps(report, ensure_ascii=False, indent=2) + "\n" + if args.output: + destination = Path(args.output) + destination.parent.mkdir(parents=True, exist_ok=True) + destination.write_text(encoded, encoding="utf-8") + print(encoded, end="") + if not total: + raise SystemExit("no requests were sent") + + +if __name__ == "__main__": + main() diff --git a/scripts/restart.sh b/scripts/restart.sh new file mode 100755 index 0000000..d587251 --- /dev/null +++ b/scripts/restart.sh @@ -0,0 +1,9 @@ +#!/bin/sh + +set -eu + +docker compose stop nginx api worker dispatcher redis-broker valkey postgres s3 +docker compose start postgres valkey redis-broker s3 +docker compose run --rm s3-bootstrap +docker compose run --rm migrate +docker compose up -d --wait --wait-timeout 90 diff --git a/scripts/rolling-update.sh b/scripts/rolling-update.sh new file mode 100755 index 0000000..695e3f8 --- /dev/null +++ b/scripts/rolling-update.sh @@ -0,0 +1,223 @@ +#!/bin/sh + +set -eu + +if [ "$#" -ne 1 ] || [ ! -f "$1" ]; then + printf 'usage: %s \n' "$0" >&2 + exit 2 +fi + +manifest=$1 +deploy_dir=${DEPLOY_STATE_DIR:-"$HOME/.local/state/mtusi2026lab5"} +current_manifest=$deploy_dir/current-images.env +active_containers=$deploy_dir/active-api-containers +previous_containers=$deploy_dir/previous-api-containers +upstream_file=$deploy_dir/upstream.conf +upstream_backup=$deploy_dir/upstream.previous.conf +candidate_file=$deploy_dir/candidate-api-containers +replicas=${API_REPLICAS:-2} +project=${COMPOSE_PROJECT_NAME:-mtusi2026lab5} +switched=false +nginx_created=false +upstream_changed=false +old_pool="" + +mkdir -p "$deploy_dir" +: > "$candidate_file" +export NGINX_UPSTREAM_FILE=$upstream_file + +load_manifest() { + set -a + # shellcheck disable=SC1090 + . "$1" + set +a +} + +compose() { + docker compose -f compose.yaml -f deploy/compose.images.yaml -f deploy/compose.runtime.yaml "$@" +} + +container_name() { + name=$(docker inspect --format '{{.Name}}' "$1") + printf '%s\n' "${name#/}" +} + +running_container() { + [ "$(docker inspect --format '{{.State.Running}}' "$1" 2>/dev/null || true)" = true ] +} + +write_upstream() { + output=$1 + containers=$2 + : > "$output" + for container in $containers; do + name=$(container_name "$container") + printf 'server %s:8080 resolve max_fails=1 fail_timeout=3s;\n' "$name" >> "$output" + done +} + +wait_for_containers() { + containers=$1 + deadline=$(($(date +%s) + 120)) + + while [ "$(date +%s)" -lt "$deadline" ]; do + count=0 + healthy=0 + for container in $containers; do + count=$((count + 1)) + status=$(docker inspect --format '{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}' "$container") + if [ "$status" = healthy ]; then + healthy=$((healthy + 1)) + fi + done + if [ "$count" -eq "$replicas" ] && [ "$healthy" -eq "$replicas" ]; then + return 0 + fi + sleep 1 + done + + printf 'candidate API pool did not become healthy: expected %s\n' "$replicas" >&2 + return 1 +} + +smoke_check() { + compose exec -T nginx wget -q -O /dev/null http://127.0.0.1:8080/health/ready +} + +remove_containers() { + containers=$1 + for container in $containers; do + docker rm -f "$container" >/dev/null 2>&1 || true + done +} + +restore_upstream() { + if [ -s "$upstream_backup" ]; then + cp "$upstream_backup" "$upstream_file" + compose exec -T nginx nginx -t + compose exec -T nginx nginx -s reload + fi +} + +rollback() { + status=$? + trap - EXIT INT TERM + candidates=$(cat "$candidate_file" 2>/dev/null || true) + + if [ "$upstream_changed" = true ] && [ -n "$old_pool" ]; then + printf '%s\n' 'Deployment failed; switching Nginx back to the previous API pool.' >&2 + restore_upstream || true + if [ "$switched" = true ]; then + smoke_check || true + fi + elif [ "$nginx_created" = true ]; then + compose stop nginx >/dev/null 2>&1 || true + fi + remove_containers "$candidates" + exit "$status" +} + +trap rollback EXIT +trap 'exit 130' INT +trap 'exit 143' TERM + +load_manifest "$manifest" +new_api_image=$API_IMAGE + +if [ -f "$previous_containers" ]; then + remove_containers "$(cat "$previous_containers")" + rm -f "$previous_containers" +fi + +if [ -s "$active_containers" ]; then + for container in $(cat "$active_containers"); do + if running_container "$container"; then + old_pool="$old_pool $container" + fi + done +fi +if [ -z "$old_pool" ]; then + old_pool=$(compose ps -q api) +fi + +if [ -s "$upstream_file" ]; then + : +elif [ -n "$old_pool" ]; then + write_upstream "$upstream_file" "$old_pool" +else + cp nginx/upstream.conf "$upstream_file" +fi +cp "$upstream_file" "$upstream_backup" + +compose pull api nginx swagger prometheus grafana s3 s3-bootstrap +compose up -d --no-build --wait --wait-timeout 120 postgres valkey redis-broker s3 +compose run --rm --no-deps s3-bootstrap +compose run --rm --no-deps migrate + +nginx_container=$(compose ps -q nginx) +if [ -n "$nginx_container" ] && ! docker inspect --format '{{range .Mounts}}{{println .Destination}}{{end}}' "$nginx_container" | grep -qx /etc/nginx/conf.d/upstream.conf; then + compose up -d --no-build --no-deps --force-recreate nginx + nginx_container=$(compose ps -q nginx) +fi +if [ -n "$nginx_container" ]; then + running_nginx_image=$(docker inspect --format '{{.Config.Image}}' "$nginx_container") + if [ "$running_nginx_image" != "$NGINX_IMAGE" ]; then + printf 'verified Nginx image differs from the running load balancer; deploy it in a dedicated maintenance operation\n' >&2 + exit 1 + fi +fi + +API_IMAGE=$new_api_image +export API_IMAGE +short_digest=$(printf '%s' "$new_api_image" | sed 's/.*sha256://; s/[^0-9a-f].*//' | cut -c1-12) +if [ -z "$short_digest" ]; then + short_digest=$(date +%s) +fi +generation=${DEPLOY_GENERATION:-"$(date +%s)-$$"} + +candidate_pool="" +index=1 +while [ "$index" -le "$replicas" ]; do + name="${project}-api-${short_digest}-${generation}-${index}" + container=$(compose run -d --no-deps --name "$name" api) + candidate_pool="$candidate_pool $container" + printf '%s\n' "$container" >> "$candidate_file" + index=$((index + 1)) +done +wait_for_containers "$candidate_pool" + +write_upstream "$upstream_file.next" "$candidate_pool" +cp "$upstream_file.next" "$upstream_file" +upstream_changed=true + +if [ -n "$(compose ps -q nginx)" ]; then + compose exec -T nginx nginx -t + switched=true + compose exec -T nginx nginx -s reload +else + nginx_created=true + compose up -d --no-build --no-deps nginx + switched=true +fi +smoke_check + +application_network="${project}_application" +for container in $candidate_pool; do + docker network disconnect "$application_network" "$container" + docker network connect --alias api "$application_network" "$container" +done +smoke_check + +compose up -d --no-build --no-deps dispatcher worker swagger postgres-exporter valkey-exporter redis-broker-exporter nginx-exporter prometheus grafana + +printf '%s\n' "$candidate_pool" > "$active_containers" +cp "$manifest" "$current_manifest" +rm -f "$candidate_file" "$upstream_file.next" +trap - EXIT INT TERM + +if [ -n "$old_pool" ]; then + for container in $old_pool; do + docker stop --time 6 "$container" >/dev/null || true + done + printf '%s\n' "$old_pool" > "$previous_containers" +fi diff --git a/scripts/stop-one.sh b/scripts/stop-one.sh new file mode 100755 index 0000000..5327251 --- /dev/null +++ b/scripts/stop-one.sh @@ -0,0 +1,16 @@ +#!/bin/sh + +set -eu + +: "${TARGET_INSTANCE_ID:?TARGET_INSTANCE_ID is required}" + +for container in $(docker compose ps -q api); do + instance=$(docker inspect --format '{{.Config.Hostname}}' "$container") + if [ "$instance" = "$TARGET_INSTANCE_ID" ]; then + docker stop --time 6 "$container" >/dev/null + exit 0 + fi +done + +printf '%s\n' "API instance not found: $TARGET_INSTANCE_ID" >&2 +exit 1 diff --git a/scripts/test-lab5-acceptance.sh b/scripts/test-lab5-acceptance.sh new file mode 100755 index 0000000..57f27ca --- /dev/null +++ b/scripts/test-lab5-acceptance.sh @@ -0,0 +1,18 @@ +#!/bin/sh + +set -eu + +state_file=$(mktemp) + +restore_pipeline() { + rm -f "$state_file" + docker compose up -d dispatcher worker >/dev/null +} + +trap restore_pipeline EXIT INT TERM + +docker compose stop worker dispatcher >/dev/null +docker compose --profile experiments run --rm lab5-experiment /app/lab5experiment api-produce > "$state_file" +docker compose restart api redis-broker >/dev/null +docker compose up -d --wait --wait-timeout 90 api nginx redis-broker dispatcher worker >/dev/null +docker compose --profile experiments run --rm --no-deps -T lab5-experiment /app/lab5experiment api-verify < "$state_file" diff --git a/scripts/test-lab5.sh b/scripts/test-lab5.sh new file mode 100755 index 0000000..dd6433c --- /dev/null +++ b/scripts/test-lab5.sh @@ -0,0 +1,11 @@ +#!/bin/sh + +set -eu + +restore_pipeline() { + docker compose up -d dispatcher worker >/dev/null +} + +trap restore_pipeline EXIT INT TERM +docker compose stop worker dispatcher >/dev/null +docker compose --profile experiments run --rm lab5-experiment diff --git a/scripts/test-lab6.sh b/scripts/test-lab6.sh new file mode 100755 index 0000000..c2ffb45 --- /dev/null +++ b/scripts/test-lab6.sh @@ -0,0 +1,73 @@ +#!/bin/sh + +set -eu + +profile=${1:-full} +case "$profile" in + smoke|full) ;; + *) + echo "usage: $0 [smoke|full]" >&2 + exit 2 + ;; +esac + +env_file=${ENV_FILE:-lab6.env.example} +base_url=${BASE_URL:-http://127.0.0.1:8080} +prometheus_url=${PROMETHEUS_URL:-http://127.0.0.1:9090} +tempo_url=${TEMPO_URL:-http://127.0.0.1:3200} +loki_url=${LOKI_URL:-http://127.0.0.1:3100} +run_id=$(date -u +%Y%m%dT%H%M%SZ) +output_dir=${EVIDENCE_DIR:-evidence/runtime/lab6-$run_id} +mkdir -p "$output_dir" + +compose() { + docker compose --env-file "$env_file" "$@" +} + +restore() { + compose up -d postgres redis-broker s3 valkey api nginx dispatcher worker otel-collector prometheus loki tempo grafana >/dev/null 2>&1 || true +} + +trap restore EXIT INT TERM + +test -f "$env_file" +compose config --quiet +compose up -d --build --wait --wait-timeout 180 +python3 scripts/lab6-traffic.py \ + --base-url "$base_url" \ + --mode mixed \ + --duration "${BASELINE_LOAD_SECONDS:-15}" \ + --requests "${BASELINE_REQUESTS:-120}" \ + --concurrency 4 \ + --rate "${BASELINE_LOAD_RATE:-10}" \ + --wait-ready \ + --ready-timeout 120 \ + --output "$output_dir/baseline-traffic.json" >/dev/null +python3 scripts/lab6-observe.py \ + --prometheus-url "$prometheus_url" \ + --phase baseline \ + --output "$output_dir/baseline-observability.json" >/dev/null +python3 scripts/lab6-correlation.py \ + --tempo-url "$tempo_url" \ + --loki-url "$loki_url" \ + --output "$output_dir/baseline-correlation.json" \ + --require >/dev/null + +if [ "$profile" = "full" ]; then + ENV_FILE="$env_file" BASE_URL="$base_url" PROMETHEUS_URL="$prometheus_url" TEMPO_URL="$tempo_url" LOKI_URL="$loki_url" EVIDENCE_DIR="$output_dir/incidents" scripts/incident-database.sh + ENV_FILE="$env_file" BASE_URL="$base_url" PROMETHEUS_URL="$prometheus_url" EVIDENCE_DIR="$output_dir/incidents" scripts/incident-worker.sh long + ENV_FILE="$env_file" BASE_URL="$base_url" PROMETHEUS_URL="$prometheus_url" EVIDENCE_DIR="$output_dir/incidents" scripts/incident-worker.sh short + ENV_FILE="$env_file" BASE_URL="$base_url" PROMETHEUS_URL="$prometheus_url" EVIDENCE_DIR="$output_dir/incidents" scripts/incident-telemetry.sh +fi + +arguments= +if [ "$profile" = "full" ]; then + arguments="--require-incidents" + if [ "${REQUIRE_ALERT_TRANSITIONS:-1}" = "1" ]; then + arguments="$arguments --require-alerts" + fi +fi +# shellcheck disable=SC2086 +python3 scripts/validate-lab6-evidence.py "$output_dir" $arguments +trap - EXIT INT TERM +printf '%s\n' "$output_dir" diff --git a/scripts/validate-lab6-evidence.py b/scripts/validate-lab6-evidence.py new file mode 100755 index 0000000..643a6fb --- /dev/null +++ b/scripts/validate-lab6-evidence.py @@ -0,0 +1,202 @@ +#!/usr/bin/env python3 + +import argparse +import json +import math +from pathlib import Path + + +def load(path): + return json.loads(path.read_text(encoding="utf-8")) + + +def check(condition, message, errors): + if not condition: + errors.append(message) + + +def query_scalar(snapshot, name, source, errors): + query = snapshot.get("queries", {}).get(name, {}) + check("error" not in query, source + " query failed: " + name, errors) + series = query.get("series", []) + check(len(series) == 1, source + " query must return one series: " + name, errors) + if len(series) != 1: + return None + try: + value = float(series[0]["value"]) + except (KeyError, TypeError, ValueError): + check(False, source + " query returned a non-numeric value: " + name, errors) + return None + check(math.isfinite(value), source + " query returned a non-finite value: " + name, errors) + return value if math.isfinite(value) else None + + +def query_exporters(snapshot, name, source, errors): + expected = {"otlp_grpc/tempo", "otlp_http/loki"} + query = snapshot.get("queries", {}).get(name, {}) + check("error" not in query, source + " query failed: " + name, errors) + values = {} + for series in query.get("series", []): + exporter = series.get("labels", {}).get("exporter") + check(exporter in expected, source + " query returned unexpected exporter: " + name, errors) + check(exporter not in values, source + " query returned duplicate exporter: " + name, errors) + try: + value = float(series["value"]) + except (KeyError, TypeError, ValueError): + check(False, source + " query returned a non-numeric value: " + name, errors) + continue + check(math.isfinite(value), source + " query returned a non-finite value: " + name, errors) + if exporter in expected and exporter not in values and math.isfinite(value): + values[exporter] = value + check(set(values) == expected, source + " query must return the exact exporter set: " + name, errors) + return values if set(values) == expected else None + + +def validate_telemetry_outage(directory, errors): + phase_names = ("baseline", "degraded", "recovered") + exporter_metric_names = ( + "collector_queue_size", + "collector_queue_capacity", + "collector_export_failures_total", + "collector_dropped_records_total", + ) + scalar_metric_names = ( + "collector_memory_bytes", + "collector_memory_peak_bytes", + ) + phases = {} + for phase in phase_names: + path = directory / (phase + ".json") + check(path.exists(), str(path) + " is missing", errors) + if not path.exists(): + return + snapshot = load(path) + phases[phase] = { + **{name: query_exporters(snapshot, name, path.name, errors) for name in exporter_metric_names}, + **{name: query_scalar(snapshot, name, path.name, errors) for name in scalar_metric_names}, + } + if any(value is None for values in phases.values() for value in values.values()): + return + + for phase, values in phases.items(): + for exporter in ("otlp_grpc/tempo", "otlp_http/loki"): + capacity = values["collector_queue_capacity"][exporter] + size = values["collector_queue_size"][exporter] + check(capacity > 0, phase + " collector queue capacity is not positive: " + exporter, errors) + check(0 <= size <= capacity, phase + " collector queue size is outside capacity: " + exporter, errors) + check(values["collector_memory_bytes"] > 0, phase + " collector memory is not positive", errors) + check(values["collector_memory_peak_bytes"] > 0, phase + " collector peak memory is not positive", errors) + + baseline = phases["baseline"] + degraded = phases["degraded"] + recovered = phases["recovered"] + for exporter in ("otlp_grpc/tempo", "otlp_http/loki"): + for name in ("collector_export_failures_total", "collector_dropped_records_total"): + check(degraded[name][exporter] >= baseline[name][exporter], name + " decreased during outage: " + exporter, errors) + check(recovered[name][exporter] >= degraded[name][exporter], name + " decreased after recovery: " + exporter, errors) + outage_observed = ( + degraded["collector_queue_size"][exporter] > baseline["collector_queue_size"][exporter] + or recovered["collector_export_failures_total"][exporter] > baseline["collector_export_failures_total"][exporter] + or recovered["collector_dropped_records_total"][exporter] > baseline["collector_dropped_records_total"][exporter] + ) + check(outage_observed, "telemetry outage was not observed for exporter: " + exporter, errors) + drained_limit = max( + baseline["collector_queue_size"][exporter], + recovered["collector_queue_capacity"][exporter] * 0.05, + ) + check( + recovered["collector_queue_size"][exporter] <= drained_limit, + "collector queue did not drain after recovery: " + exporter, + errors, + ) + memory_limit = max(baseline["collector_memory_peak_bytes"] * 1.5, baseline["collector_memory_peak_bytes"] + 64 * 1024 * 1024) + check(degraded["collector_memory_peak_bytes"] <= memory_limit, "collector peak memory grew beyond the outage bound", errors) + check(recovered["collector_memory_peak_bytes"] <= memory_limit, "collector peak memory remained above the recovery bound", errors) + + +def main(): + parser = argparse.ArgumentParser(description="Validate generated Lab 6 incident evidence") + parser.add_argument("directory") + parser.add_argument("--require-incidents", action="store_true") + parser.add_argument("--require-alerts", action="store_true") + args = parser.parse_args() + + root = Path(args.directory) + errors = [] + baseline = root / "baseline-traffic.json" + check(baseline.exists(), "baseline-traffic.json is missing", errors) + if baseline.exists(): + report = load(baseline) + check(report.get("requests", 0) > 0, "baseline sent no requests", errors) + check(report.get("successful_requests", 0) > 0, "baseline had no successful requests", errors) + + correlation = root / "baseline-correlation.json" + check(correlation.exists(), "baseline-correlation.json is missing", errors) + if correlation.exists(): + check(load(correlation).get("complete") is True, "metrics/log/trace correlation is incomplete", errors) + + observability = root / "baseline-observability.json" + check(observability.exists(), "baseline-observability.json is missing", errors) + if observability.exists(): + firing = [item.get("name") for item in load(observability).get("alerts", {}).get("items", []) if item.get("state") == "firing"] + check(not firing, "baseline has firing alerts: " + ", ".join(firing), errors) + + summaries = list(root.rglob("summary.json")) + if args.require_incidents: + names = {load(path).get("incident") for path in summaries} + for required in ("worker_unavailable", "postgres_unavailable", "telemetry_backends_unavailable"): + check(required in names, "incident evidence is missing: " + required, errors) + + for summary_path in summaries: + summary = load(summary_path) + incident = summary.get("incident") + traffic_path = summary_path.parent / "traffic.json" + check(traffic_path.exists(), str(traffic_path) + " is missing", errors) + if not traffic_path.exists(): + continue + traffic = load(traffic_path) + check(traffic.get("requests", 0) > 0, incident + " sent no traffic", errors) + if incident == "worker_unavailable": + terminal = traffic.get("attachment_terminal_states") or {} + check(traffic.get("attachments_accepted", 0) > 0, incident + " accepted no attachments", errors) + check(terminal.get("ready", 0) == traffic.get("attachments_accepted", -1), incident + " did not drain every accepted attachment", errors) + check(terminal.get("failed", 0) == 0, incident + " produced failed attachments", errors) + check(terminal.get("timeout", 0) == 0, incident + " left attachment jobs non-terminal", errors) + elif incident == "postgres_unavailable": + statuses = traffic.get("status_counts", {}) + failures = sum(count for status, count in statuses.items() if not status.startswith("2")) + check(failures > 0, incident + " produced no observable failed requests", errors) + correlation_path = summary_path.parent / "database-correlation.json" + check(correlation_path.exists(), str(correlation_path) + " is missing", errors) + if correlation_path.exists(): + check(load(correlation_path).get("complete") is True, "database error trace/log correlation is incomplete", errors) + elif incident == "telemetry_backends_unavailable": + terminal = traffic.get("attachment_terminal_states") or {} + check(traffic.get("successful_requests", 0) > 0, incident + " had no successful business requests", errors) + check(traffic.get("attachments_accepted", 0) > 0, incident + " accepted no attachments", errors) + check(terminal.get("ready", 0) == traffic.get("attachments_accepted", -1), incident + " did not finish every attachment", errors) + check(terminal.get("failed", 0) == 0 and terminal.get("timeout", 0) == 0, incident + " left unsuccessful attachments", errors) + check(summary.get("api_ready_http_status_during_outage") == 200, incident + " broke API readiness", errors) + validate_telemetry_outage(summary_path.parent, errors) + + if args.require_alerts and incident in ("worker_unavailable", "postgres_unavailable"): + for name in ("alert-firing.json", "alert-resolved.json"): + path = summary_path.parent / name + check(path.exists(), str(path) + " is missing", errors) + if path.exists(): + check(load(path).get("observed") is True, str(path) + " did not observe the transition", errors) + if incident == "worker_unavailable" and summary.get("profile") == "long": + for name in ("burn-fast-firing.json", "burn-slow-firing.json"): + path = summary_path.parent / name + check(path.exists(), str(path) + " is missing", errors) + if path.exists(): + check(load(path).get("observed") is True, str(path) + " did not observe attachment burn", errors) + + result = {"valid": not errors, "errors": errors, "incident_summaries": len(summaries)} + print(json.dumps(result, ensure_ascii=False, indent=2)) + if errors: + raise SystemExit(1) + + +if __name__ == "__main__": + main()