From ce035c85282e75513ea36cb429f4b8d775398d62 Mon Sep 17 00:00:00 2001 From: Grbisba Date: Fri, 13 Nov 2026 11:22:22 +0300 Subject: [PATCH 1/2] =?UTF-8?q?=D0=9F=D0=B5=D1=80=D0=B5=D0=BD=D0=B5=D1=81?= =?UTF-8?q?=20=D0=B8=D0=B7=D0=BC=D0=B5=D0=BD=D0=B5=D0=BD=D0=B8=D1=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .env.example | 45 ++- .gitignore | 1 + Dockerfile | 32 ++ Makefile | 89 ++++- README.md | 34 +- compose.yaml | 182 +++++++++++ infra/haproxy.cfg | 39 +++ scripts/bootstrap-storage.py | 25 ++ scripts/bootstrap.sh | 23 ++ scripts/compose.sh | 4 + scripts/generate-types.py | 17 + scripts/restart.sh | 7 + scripts/rolling-update.py | 45 +++ scripts/rolling-update.sh | 4 + scripts/stop-one.sh | 17 + scripts/test-storage.sh | 11 + service/cmd/app/main.go | 150 +++++++++ service/cmd/app/storage.go | 175 ++++++++++ service/cmd/benchmark/main.go | 260 +++++++++++++++ service/cmd/benchmark/main_test.go | 42 +++ service/go.mod | 66 ++++ service/go.sum | 147 +++++++++ .../adapter/controller/http/attachment.go | 1 + .../controller/http/attachmentHandler.go | 91 ++++++ .../internal/adapter/controller/http/auth.go | 88 +++++ .../internal/adapter/controller/http/chat.go | 287 +++++++++++++++++ .../adapter/controller/http/controller.go | 102 ++++++ .../adapter/controller/http/handler.go | 5 + .../adapter/controller/http/health.go | 60 ++++ .../adapter/controller/http/router.go | 1 + .../internal/adapter/controller/http/user.go | 113 +++++++ .../adapter/repository/cache/outside/chat.go | 78 +++++ .../repository/storage/attachment_scan.go | 34 ++ .../adapter/repository/storage/attachments.go | 71 ++++ .../adapter/repository/storage/chats.go | 91 ++++++ .../repository/storage/idempotency_test.go | 303 ++++++++++++++++++ .../adapter/repository/storage/messages.go | 153 +++++++++ .../repository/storage/refresh_tokens.go | 193 +++++++++++ .../adapter/repository/storage/users.go | 103 ++++++ service/internal/config/abstractions.go | 7 + service/internal/config/application.go | 63 ++++ service/internal/config/attachment.go | 58 ++++ service/internal/config/config.go | 110 +++++++ service/internal/config/cookies.go | 65 ++++ service/internal/config/jwt.go | 63 ++++ service/internal/config/postgres.go | 110 +++++++ service/internal/config/redis.go | 72 +++++ service/internal/config/server.go | 69 ++++ service/internal/config/validation.go | 11 + service/internal/dto/attachment.go | 1 + service/internal/dto/auth.go | 126 ++++++++ service/internal/dto/chat.go | 31 ++ service/internal/dto/message.go | 38 +++ service/internal/dto/page.go | 25 ++ service/internal/dto/user.go | 30 ++ service/internal/gen/contracts/types.gen.go | 226 +++++++++++++ service/internal/pkg/apperr/code.go | 274 ++++++++++++++++ service/internal/pkg/apperr/code_string.go | 37 +++ service/internal/pkg/apperr/error.go | 123 +++++++ service/internal/pkg/apperr/meta.go | 42 +++ service/internal/pkg/apperr/metakey.go | 19 ++ service/internal/pkg/apperr/postgres.go | 81 +++++ service/internal/pkg/apperr/redis.go | 170 ++++++++++ .../internal/pkg/auth/jwtprovider/claims.go | 145 +++++++++ .../internal/pkg/auth/jwtprovider/cookie.go | 84 +++++ .../internal/pkg/auth/jwtprovider/error.go | 18 ++ .../pkg/auth/jwtprovider/interface.go | 26 ++ .../internal/pkg/auth/jwtprovider/manager.go | 199 ++++++++++++ .../internal/pkg/auth/jwtprovider/provider.go | 187 +++++++++++ .../internal/pkg/auth/jwtprovider/readers.go | 35 ++ service/internal/pkg/cache/redis.go | 43 +++ service/internal/pkg/encrypting/argon2id.go | 71 ++++ service/internal/pkg/encrypting/encryptor.go | 72 +++++ service/internal/pkg/encrypting/error.go | 8 + service/internal/pkg/encrypting/interface.go | 12 + service/internal/pkg/encrypting/token_test.go | 43 +++ service/internal/pkg/migrator/tern.go | 164 ++++++++++ service/internal/pkg/objectstore/s3.go | 69 ++++ .../internal/pkg/postgres/after-connect.go | 25 ++ service/internal/pkg/postgres/config.go | 7 + service/internal/pkg/postgres/err.go | 10 + service/internal/pkg/postgres/interfaces.go | 15 + service/internal/pkg/postgres/postgres.go | 95 ++++++ service/internal/pkg/postgres/txmanager/tx.go | 59 ++++ .../internal/pkg/rest/framework/framework.go | 257 +++++++++++++++ service/internal/pkg/rest/headers/headers.go | 8 + service/internal/pkg/rest/middleware/auth.go | 75 +++++ .../internal/pkg/rest/middleware/auth_test.go | 73 +++++ .../internal/pkg/rest/middleware/instance.go | 12 + .../internal/pkg/rest/middleware/logging.go | 34 ++ .../pkg/rest/middleware/middleware.go | 5 + .../pkg/rest/middleware/request-id.go | 22 ++ .../internal/pkg/rest/middleware/timeout.go | 23 ++ service/internal/pkg/rest/router/router.go | 131 ++++++++ .../pkg/types/authprovider/provider.go | 82 +++++ .../pkg/types/environment/environment.go | 77 +++++ service/internal/pkg/types/error.go | 25 ++ service/internal/pkg/types/role/role.go | 105 ++++++ service/internal/pkg/validator/validator.go | 29 ++ .../internal/service/attachment/reconcile.go | 205 ++++++++++++ .../internal/service/attachment/service.go | 162 ++++++++++ service/internal/service/auth/api.go | 15 + service/internal/service/auth/interface.go | 21 ++ service/internal/service/auth/svc.go | 267 +++++++++++++++ service/internal/service/chat/api.go | 15 + service/internal/service/chat/interface.go | 28 ++ service/internal/service/chat/svc.go | 286 +++++++++++++++++ service/internal/service/user/api.go | 15 + service/internal/service/user/interface.go | 15 + service/internal/service/user/svc.go | 67 ++++ service/migrations/001_create_users_table.sql | 18 ++ .../002_create_refresh_tokens_table.sql | 19 ++ service/migrations/003_create_chats_table.sql | 14 + .../004_create_chat_members_table.sql | 15 + .../migrations/005_create_messages_table.sql | 18 ++ .../migrations/006_message_idempotency.sql | 14 + service/migrations/007_attachments.sql | 44 +++ service/migrations/embed.go | 8 + service/tests/api_test.go | 285 ++++++++++++++++ service/tests/attachments_test.go | 204 ++++++++++++ service/tests/concurrency_test.go | 161 ++++++++++ service/tests/idempotency_test.go | 104 ++++++ 122 files changed, 9357 insertions(+), 28 deletions(-) create mode 100644 Dockerfile create mode 100644 compose.yaml create mode 100644 infra/haproxy.cfg create mode 100644 scripts/bootstrap-storage.py create mode 100755 scripts/bootstrap.sh create mode 100755 scripts/compose.sh create mode 100644 scripts/generate-types.py create mode 100755 scripts/restart.sh create mode 100644 scripts/rolling-update.py create mode 100755 scripts/rolling-update.sh create mode 100755 scripts/stop-one.sh create mode 100755 scripts/test-storage.sh create mode 100644 service/cmd/app/main.go create mode 100644 service/cmd/app/storage.go create mode 100644 service/cmd/benchmark/main.go create mode 100644 service/cmd/benchmark/main_test.go create mode 100644 service/go.mod create mode 100644 service/go.sum create mode 100644 service/internal/adapter/controller/http/attachment.go create mode 100644 service/internal/adapter/controller/http/attachmentHandler.go create mode 100644 service/internal/adapter/controller/http/auth.go create mode 100644 service/internal/adapter/controller/http/chat.go create mode 100644 service/internal/adapter/controller/http/controller.go create mode 100644 service/internal/adapter/controller/http/handler.go create mode 100644 service/internal/adapter/controller/http/health.go create mode 100644 service/internal/adapter/controller/http/router.go create mode 100644 service/internal/adapter/controller/http/user.go create mode 100644 service/internal/adapter/repository/cache/outside/chat.go create mode 100644 service/internal/adapter/repository/storage/attachment_scan.go create mode 100644 service/internal/adapter/repository/storage/attachments.go create mode 100644 service/internal/adapter/repository/storage/chats.go create mode 100644 service/internal/adapter/repository/storage/idempotency_test.go create mode 100644 service/internal/adapter/repository/storage/messages.go create mode 100644 service/internal/adapter/repository/storage/refresh_tokens.go create mode 100644 service/internal/adapter/repository/storage/users.go create mode 100644 service/internal/config/abstractions.go create mode 100644 service/internal/config/application.go create mode 100644 service/internal/config/attachment.go create mode 100644 service/internal/config/config.go create mode 100644 service/internal/config/cookies.go create mode 100644 service/internal/config/jwt.go create mode 100644 service/internal/config/postgres.go create mode 100644 service/internal/config/redis.go create mode 100644 service/internal/config/server.go create mode 100644 service/internal/config/validation.go create mode 100644 service/internal/dto/attachment.go create mode 100644 service/internal/dto/auth.go create mode 100644 service/internal/dto/chat.go create mode 100644 service/internal/dto/message.go create mode 100644 service/internal/dto/page.go create mode 100644 service/internal/dto/user.go create mode 100644 service/internal/gen/contracts/types.gen.go create mode 100644 service/internal/pkg/apperr/code.go create mode 100644 service/internal/pkg/apperr/code_string.go create mode 100644 service/internal/pkg/apperr/error.go create mode 100644 service/internal/pkg/apperr/meta.go create mode 100644 service/internal/pkg/apperr/metakey.go create mode 100644 service/internal/pkg/apperr/postgres.go create mode 100644 service/internal/pkg/apperr/redis.go create mode 100644 service/internal/pkg/auth/jwtprovider/claims.go create mode 100644 service/internal/pkg/auth/jwtprovider/cookie.go create mode 100644 service/internal/pkg/auth/jwtprovider/error.go create mode 100644 service/internal/pkg/auth/jwtprovider/interface.go create mode 100644 service/internal/pkg/auth/jwtprovider/manager.go create mode 100644 service/internal/pkg/auth/jwtprovider/provider.go create mode 100644 service/internal/pkg/auth/jwtprovider/readers.go create mode 100644 service/internal/pkg/cache/redis.go create mode 100644 service/internal/pkg/encrypting/argon2id.go create mode 100644 service/internal/pkg/encrypting/encryptor.go create mode 100644 service/internal/pkg/encrypting/error.go create mode 100644 service/internal/pkg/encrypting/interface.go create mode 100644 service/internal/pkg/encrypting/token_test.go create mode 100644 service/internal/pkg/migrator/tern.go create mode 100644 service/internal/pkg/objectstore/s3.go create mode 100644 service/internal/pkg/postgres/after-connect.go create mode 100644 service/internal/pkg/postgres/config.go create mode 100644 service/internal/pkg/postgres/err.go create mode 100644 service/internal/pkg/postgres/interfaces.go create mode 100644 service/internal/pkg/postgres/postgres.go create mode 100644 service/internal/pkg/postgres/txmanager/tx.go create mode 100644 service/internal/pkg/rest/framework/framework.go create mode 100644 service/internal/pkg/rest/headers/headers.go create mode 100644 service/internal/pkg/rest/middleware/auth.go create mode 100644 service/internal/pkg/rest/middleware/auth_test.go create mode 100644 service/internal/pkg/rest/middleware/instance.go create mode 100644 service/internal/pkg/rest/middleware/logging.go create mode 100644 service/internal/pkg/rest/middleware/middleware.go create mode 100644 service/internal/pkg/rest/middleware/request-id.go create mode 100644 service/internal/pkg/rest/middleware/timeout.go create mode 100644 service/internal/pkg/rest/router/router.go create mode 100644 service/internal/pkg/types/authprovider/provider.go create mode 100644 service/internal/pkg/types/environment/environment.go create mode 100644 service/internal/pkg/types/error.go create mode 100644 service/internal/pkg/types/role/role.go create mode 100644 service/internal/pkg/validator/validator.go create mode 100644 service/internal/service/attachment/reconcile.go create mode 100644 service/internal/service/attachment/service.go create mode 100644 service/internal/service/auth/api.go create mode 100644 service/internal/service/auth/interface.go create mode 100644 service/internal/service/auth/svc.go create mode 100644 service/internal/service/chat/api.go create mode 100644 service/internal/service/chat/interface.go create mode 100644 service/internal/service/chat/svc.go create mode 100644 service/internal/service/user/api.go create mode 100644 service/internal/service/user/interface.go create mode 100644 service/internal/service/user/svc.go create mode 100644 service/migrations/001_create_users_table.sql create mode 100644 service/migrations/002_create_refresh_tokens_table.sql create mode 100644 service/migrations/003_create_chats_table.sql create mode 100644 service/migrations/004_create_chat_members_table.sql create mode 100644 service/migrations/005_create_messages_table.sql create mode 100644 service/migrations/006_message_idempotency.sql create mode 100644 service/migrations/007_attachments.sql create mode 100644 service/migrations/embed.go create mode 100644 service/tests/api_test.go create mode 100644 service/tests/attachments_test.go create mode 100644 service/tests/concurrency_test.go create mode 100644 service/tests/idempotency_test.go diff --git a/.env.example b/.env.example index 119384b5..ecbfeb2a 100644 --- a/.env.example +++ b/.env.example @@ -1,23 +1,36 @@ -# Copy to .env for your LOCAL synthetic test environment; never commit .env. -# These are logical interface suggestions; map actual framework names in REPORT.md. -APP_HOST=0.0.0.0 -APP_PORT=8080 -APP_ENV=lab -SESSION_TTL_SECONDS=3600 -# Choose credentials locally. File paths contain no secret values. -DATABASE_URL_FILE=/run/secrets/database_url -# Optional on 3, required use on 4 in lab2; Redis OR Valkey: -SESSION_STORE=postgres -CACHE_URL_FILE=/run/secrets/cache_url +# scripts/bootstrap.sh creates ignored .env.lab5 with random synthetic credentials. +# This example documents actual Compose inputs; it contains no live secrets. +POSTGRES_PASSWORD=replace-with-local-postgres-password +REDIS_PASSWORD=replace-with-local-redis-password +HOST_PORT=18085 API_REPLICAS=2 -IDEMPOTENCY_TTL_SECONDS=86400 -S3_ENDPOINT=http://s3:9000 -S3_REGION=us-east-1 +API_CPUS=1 +API_MEMORY=512m +POSTGRES_MAX_CONNECTIONS=8 +APPLICATION_LOG_LEVEL=warn +SESSION_TTL_SECONDS=3600 +API_IMAGE=mtusi-messenger:lab5 +# Session TTL defaults to 3600; use >=300 for smoke/persistence. +# Message idempotency retention is fixed at 24 hours; API never runs migrations. + +# S3 credentials are generated locally by scripts/bootstrap-storage.py. +# For a custom bucket, also export S3_BUCKET before make up: the credential +# bootstrap reads the process environment, while Compose reads .env.lab5. S3_BUCKET=messenger-attachments -S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id -S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key +# Runtime values fixed in compose.yaml (not configurable through this file): +# S3_ENDPOINT=http://s3:9000 +# S3_REGION=us-east-1 +# S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access +# S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret MAX_UPLOAD_BYTES=10485760 MAX_IMAGE_PIXELS=40000000 +# Bounded attachment request lifetime; grace must exceed twice this duration. +UPLOAD_TIMEOUT=120s +RECONCILE_GRACE_PERIOD=1h +# SeaweedFS image and persistent bucket bootstrap are pinned in compose.yaml. +# API has four upload slots and one decoder per replica, GOMEMLIMIT=384MiB. + +# Planned lab5 async pipeline settings; the ported lab4 application does not use these yet. BROKER_URL_FILE=/run/secrets/broker_url WORKER_CONCURRENCY=2 JOB_MAX_ATTEMPTS=5 diff --git a/.gitignore b/.gitignore index b5a71896..73a022b1 100644 --- a/.gitignore +++ b/.gitignore @@ -30,3 +30,4 @@ coverage/ .vscode/ # Keep small, sanitized evidence summaries in evidence/ under version control. # Inspect git diff --cached: ignore rules are not a secret scanner. +/.ignore/ diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..38980df9 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,32 @@ +# syntax=docker/dockerfile:1 + +FROM --platform=$BUILDPLATFORM golang:1.27.2-alpine3.23@sha256:9e45f0eb4a63ed37ad6e604950407558b7c738e34e015ae77a5d4ad369cde079 AS build + +WORKDIR /src + +COPY service/go.mod service/go.sum ./ +RUN go mod download + +COPY service/ ./ + +ARG TARGETOS +ARG TARGETARCH +RUN CGO_ENABLED=0 GOOS=$TARGETOS GOARCH=$TARGETARCH \ + go build -mod=readonly -trimpath -ldflags="-s -w" -o /out/messenger ./cmd/app + +FROM alpine:3.23.6@sha256:85fe1e81d6758c208f3e1eed4338a1997e19d4be002d4dd32d3100c9a8c010a0 + +COPY --from=build --chmod=0555 /out/messenger /usr/local/bin/messenger + +ENV APP_HOST=0.0.0.0 \ + APP_PORT=8080 \ + APP_SHUTDOWN_TIMEOUT=8s + +USER 65532:65532 +EXPOSE 8080 +STOPSIGNAL SIGTERM + +HEALTHCHECK --interval=10s --timeout=3s --start-period=5s --retries=3 \ + CMD wget -q -O /dev/null "http://127.0.0.1:${APP_PORT}/health/ready" || exit 1 + +ENTRYPOINT ["/usr/local/bin/messenger"] diff --git a/Makefile b/Makefile index 5d86d171..db8c42a0 100644 --- a/Makefile +++ b/Makefile @@ -1,13 +1,24 @@ +SHELL := /bin/bash +GO ?= go PYTHON ?= python3 -BASE_URL ?= http://localhost:8080 +BASE_URL ?= http://localhost:18085 CA_FILE ?= ACTION_SCRIPT ?= TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",) +OPENAPI_SPEC := contracts/openapi.json +GEN_OUTPUT := service/internal/gen/contracts/types.gen.go +GEN_PACKAGE := contracts +OAPI_CODEGEN ?= $(shell $(GO) env GOPATH)/bin/oapi-codegen -.PHONY: help check validate test load test-persistence test-failover +.PHONY: help check validate test load test-persistence test-failover openapi-generate certs key-gen create-dir gen-private-key gen-public-key gen-key gen-crt help: - @echo "check: offline structure; validate: full OpenAPI (requirements-dev.txt); test: running API; load: synthetic writes" + @echo "up/down: lab5 stand (ported lab4 application); build/vet: Go checks" + @echo "test: lab5 API smoke (requires async implementation); test-go/test-storage: existing Go checks" + @echo "restart/restore/rolling-update: infrastructure actions; benchmark/load: synthetic traffic" + @echo "check/validate: lab5 contracts; test-persistence/test-failover: explicit ACTION_SCRIPT hooks" + @echo "reconcile/reconcile-apply: storage consistency" + @echo "openapi-generate: generate types from lab5 contract; key-gen/certs: local keys" check: $(PYTHON) scripts/check_contract.py @@ -28,3 +39,75 @@ test-persistence: test-failover: @test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable stop-one hook"; exit 2) $(PYTHON) tests/resilience.py --mode failover --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS) + + +openapi-generate: + @mkdir -p $(dir $(GEN_OUTPUT)) + @echo "Generating Go types from $(OPENAPI_SPEC)..." + @set -o pipefail; \ + $(PYTHON) scripts/generate-types.py | \ + $(OAPI_CODEGEN) -generate types -package $(GEN_PACKAGE) -o $(GEN_OUTPUT) /dev/stdin + +key-gen: gen-public-key + +create-dir: + @mkdir -p certs +gen-private-key: | create-dir + @openssl genrsa -out certs/private_key.pem 2048 +gen-public-key: gen-private-key + @openssl rsa -in certs/private_key.pem -pubout -outform PEM -out certs/public_key.pem + +certs: gen-crt + +gen-key: | create-dir + @openssl genrsa -out ./certs/server.key 2048 +gen-crt: gen-key + @openssl req -new -x509 -key ./certs/server.key -out ./certs/server.crt -days 365 + + +.PHONY: up down test-go +up: + ./scripts/bootstrap.sh + ./scripts/compose.sh up -d --build --wait --wait-timeout 120 + +down: + ./scripts/compose.sh down + +test-go: + cd service && TEST_BASE_URL="$(BASE_URL)" $(GO) test -race -count=1 -v ./... + +RATE ?= 100 +DURATION ?= 30s +BENCH_ARGS ?= + +.PHONY: build vet restart restore rolling-update benchmark test-storage +build: + cd service && $(GO) build ./... + @mkdir -p bin + cd service && $(GO) build -o ../bin/benchmark ./cmd/benchmark + +vet: + cd service && $(GO) vet ./... + +restart: + ./scripts/restart.sh + +restore: + ./scripts/compose.sh up -d --wait --wait-timeout 90 --no-recreate api lb + +rolling-update: + ./scripts/rolling-update.sh + +benchmark: build + ./bin/benchmark -base-url "$(BASE_URL)" -rate "$(RATE)" -duration "$(DURATION)" $(BENCH_ARGS) + +test-storage: + ./scripts/test-storage.sh + +.PHONY: reconcile reconcile-apply + +reconcile: + ./scripts/compose.sh run --rm -T operator storage reconcile + +reconcile-apply: + ./scripts/compose.sh run --rm -T operator storage reconcile --apply diff --git a/README.md b/README.md index 009d6997..3d3d0d1b 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ Отделить приём файла от тяжёлой обработки, корректно переживать повторную доставку и остановку worker. -Это **skeleton**, а не готовый сервер. Здесь есть задание, контракт и публичные проверки. Реализацию приложения, Dockerfile и требуемую инфраструктуру пишет студент. Язык и framework свободные; UI не обязателен. +В репозиторий перенесена рабочая основа lab4: Go-приложение, миграции, Dockerfile и стенд с PostgreSQL, Redis, S3 (SeaweedFS) и HAProxy. Задание, контракт и публичные проверки относятся к lab5; очередь и worker ещё предстоит реализовать. Язык и framework свободные; UI не обязателен. [Карта курса](COURSE.md) · [Процесс fork/PR](CONTRIBUTING.md) · [OpenAPI](contracts/openapi.json) · [Семантика API](contracts/README.md) · [Проверки](tests/README.md) · [Отчёт](REPORT.md) · [Примеры JSON](contracts/examples.json) · [Памятка преподавателю](MAINTAINERS.md) @@ -48,21 +48,37 @@ ## Как начать и проверить ```sh -# Работает прямо в skeleton, Python 3.9+; приложение не запускается: +# Проверка структуры и контрактов, Python 3.9+; приложение не запускается: make check -# После реализации запустите свой сервер/стенд по REPORT.md. -# По умолчанию тест использует http://localhost:8080. +# Сборка приложения (Go 1.27+) и статический анализ: +make build +make vet + +# Создание локальных настроек/ключей и запуск стенда (Docker Compose): +make up +curl --fail http://localhost:18085/health/ready + +# Публичные проверки lab5; полностью заработают после реализации async pipeline: make test + +# Остановка стенда с сохранением данных: +make down ``` -`.env.example`, `compose.example.yaml` и `contracts/infrastructure.json` описывают настройки и роли компонентов. **compose.example.yaml — список ролей с пустым services, не запускаемый стенд.** Реализуйте свой `compose.yaml`/`stack.yaml`, закрепите версии образов, заполните локальный `.env` и опишите bootstrap/миграции. Все зависимости поднимаются из этого репозитория; ссылка «у меня БД уже установлена» не заменяет воспроизводимость. +`make up` выполняет `scripts/bootstrap.sh`: создаёт игнорируемые `.env.lab5` и `secrets/lab5/` с новыми локальными паролями и ключами, сохраняя существующие при повторном запуске. Пароли и ключи из lab4 не копируются. `.env.example` документирует настройки стенда; параметры будущей очереди пока не используются. Для смены порта измените `HOST_PORT` в `.env.lab5` и передавайте соответствующий `BASE_URL` командам Makefile. + +Compose-проект называется `mtusi2026lab5`, использует собственные volumes и по умолчанию доступен на `http://localhost:18085`, поэтому может работать одновременно с lab4. PostgreSQL мигрируется отдельным одноразовым сервисом, S3 bucket инициализируется перед запуском API. Ключи `make key-gen` и `make certs` создаются отдельно в `certs/`; для Compose ключи создаёт bootstrap. + +**Это перенос основы, а не готовая lab5:** upload пока синхронный и возвращает `201`, брокера и worker нет. Контракт lab5 требует `202`, фоновой обработки и вариантов изображения, поэтому публичный smoke сейчас не должен проходить полностью. Существующие Go-проверки перенесены вместе с приложением и доступны через `make test-go` и `make test-storage`. `make openapi-generate` использует контракт lab5; при переносе сохранены сгенерированные типы lab4. + +`compose.yaml` — запускаемый стенд. **compose.example.yaml — список ролей с пустым services, не запускаемый стенд.** `contracts/infrastructure.json` задаёт требования lab5; расширьте стенд брокером и worker при реализации задания. Все зависимости поднимаются из этого репозитория; ссылка «у меня БД уже установлена» не заменяет воспроизводимость. ```sh -# Compose-вариант после реализации: -docker compose --env-file .env config --quiet -docker compose --env-file .env up -d --build -# Дождитесь /health/ready; затем make test. +# Прямое управление тем же стендом: +./scripts/bootstrap.sh +./scripts/compose.sh config --quiet +./scripts/compose.sh up -d --build --wait --wait-timeout 120 # Для Swarm укажите эквивалентные build/push/deploy-команды в REPORT.md. ``` diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 00000000..217cce67 --- /dev/null +++ b/compose.yaml @@ -0,0 +1,182 @@ +name: mtusi2026lab5 +x-api: &api + image: ${API_IMAGE:-mtusi-messenger:lab5} + build: . + restart: unless-stopped + cpus: ${API_CPUS:-1} + mem_limit: ${API_MEMORY:-512m} + environment: &environment + APP_ENABLED: "true" + APP_HOST: 0.0.0.0 + APP_PORT: 8080 + APP_READ_TIMEOUT: 125s + APP_WRITE_TIMEOUT: 125s + APP_IDLE_TIMEOUT: 60s + APP_SHUTDOWN_TIMEOUT: 8s + APPLICATION_LOG_LEVEL: ${APPLICATION_LOG_LEVEL:-warn} + APPLICATION_ENVIRONMENT: prod + APPLICATION_SHUTDOWN_TIMEOUT: 8s + POSTGRES_HOST: postgres + POSTGRES_PORT: 5432 + POSTGRES_USER: messenger + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?Run scripts/bootstrap.sh} + POSTGRES_DATABASE: messenger + POSTGRES_VERSION_TABLE: versions + POSTGRES_RUN_MIGRATIONS: "false" + POSTGRES_MAX_CONNECTIONS: ${POSTGRES_MAX_CONNECTIONS:-8} + POSTGRES_CONNECT_TIMEOUT: 500ms + POSTGRES_QUERY_TIMEOUT: 2s + REDIS_HOST: redis + REDIS_PORT: 6379 + REDIS_DB: 0 + REDIS_USERNAME: default + REDIS_PASSWORD: ${REDIS_PASSWORD:?Run scripts/bootstrap.sh} + SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600} + JWT_ACCESS_TOKEN_LIFETIME: 15m + JWT_PRIVATE_KEY_PATH: /run/certs/private_key.pem + JWT_PUBLIC_KEY_PATH: /run/certs/public_key.pem + S3_ENDPOINT: http://s3:9000 + S3_REGION: us-east-1 + S3_BUCKET: ${S3_BUCKET:-messenger-attachments} + S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access + S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret + MAX_UPLOAD_BYTES: ${MAX_UPLOAD_BYTES:-10485760} + MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS:-40000000} + UPLOAD_TIMEOUT: ${UPLOAD_TIMEOUT:-120s} + RECONCILE_GRACE_PERIOD: ${RECONCILE_GRACE_PERIOD:-1h} + GOMEMLIMIT: 384MiB + networks: [default, storage] + volumes: + - ./secrets/lab5/private_key.pem:/run/certs/private_key.pem:ro + - ./secrets/lab5/public_key.pem:/run/certs/public_key.pem:ro + - ./secrets/lab5/s3_api_access:/run/secrets/s3_access:ro + - ./secrets/lab5/s3_api_secret:/run/secrets/s3_secret:ro + stop_grace_period: 10s +services: + api: + <<: *api + deploy: + replicas: ${API_REPLICAS:-2} + depends_on: + migrate: + condition: service_completed_successfully + s3-init: + condition: service_completed_successfully + migrate: + <<: *api + # The migration command uses only PostgreSQL: no signing keys or Redis credentials. + volumes: [] + environment: + <<: *environment + REDIS_PASSWORD: "" + restart: "no" + command: [migrate] + cpus: 0.5 + mem_limit: 128m + healthcheck: + disable: true + depends_on: + postgres: + condition: service_healthy + lb: + image: ${HAPROXY_IMAGE:-haproxy:3.2-alpine@sha256:96ec3c75a9264ba94e733eeacb44051ed6e7b79bb77a73c2c36f236b6423f1f0} + restart: unless-stopped + cpus: 0.5 + mem_limit: 128m + ports: + - "127.0.0.1:${HOST_PORT:-18085}:8080" + volumes: + - ./infra/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro + depends_on: + api: + condition: service_started + healthcheck: + test: [CMD, wget, -q, -O, /dev/null, http://127.0.0.1:8080/health/ready] + interval: 2s + timeout: 1s + retries: 5 + stop_grace_period: 10s + postgres: + image: postgres:17.11-alpine3.23@sha256:05aa77262db9277fde6374bb2f901c9604b8a9373029e1a7993440c6aae48736 + restart: unless-stopped + cpus: 2 + mem_limit: 1g + environment: + POSTGRES_USER: messenger + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?Run scripts/bootstrap.sh} + POSTGRES_DB: messenger + volumes: + - postgres_data:/var/lib/postgresql/data + healthcheck: + test: [CMD-SHELL, 'pg_isready -U messenger -d messenger'] + interval: 2s + timeout: 1s + retries: 30 + stop_grace_period: 30s + redis: + image: redis:8.2.11-alpine3.22@sha256:489a769e0a54dc245896132a15c72d2bc7296719954da99398f43a3877e5f8eb + restart: unless-stopped + cpus: 0.25 + mem_limit: 128m + environment: + REDIS_PASSWORD: ${REDIS_PASSWORD:?Run scripts/bootstrap.sh} + command: [sh, -c, 'exec docker-entrypoint.sh redis-server --appendonly yes --maxmemory 64mb --maxmemory-policy allkeys-lru --requirepass "$$REDIS_PASSWORD"'] + volumes: + - redis_data:/data + healthcheck: + test: [CMD-SHELL, 'REDISCLI_AUTH="$$REDIS_PASSWORD" redis-cli --no-auth-warning ping | grep -qx PONG'] + interval: 2s + timeout: 1s + retries: 20 + stop_grace_period: 10s + s3: + image: chrislusf/seaweedfs:4.52@sha256:57877e32bb90073b37dc109b438ba66e84fd5ed9ced5f5c04f56444ef243ce59 + restart: unless-stopped + cpus: 1 + mem_limit: 512m + networks: [storage] + command: [server, '-dir=/data', '-ip=s3', '-ip.bind=0.0.0.0', '-volume.max=4', '-master.volumeSizeLimitMB=128', '-filer', '-s3', '-s3.port=9000', '-s3.config=/etc/seaweedfs/s3.json'] + volumes: + - s3_data:/data + - ./secrets/lab5/s3.json:/etc/seaweedfs/s3.json:ro + healthcheck: + test: [CMD, wget, -q, -O, /dev/null, http://127.0.0.1:9333/cluster/status] + interval: 2s + timeout: 1s + retries: 30 + stop_grace_period: 30s + s3-init: + <<: *api + restart: "no" + cpus: 0.25 + mem_limit: 128m + command: [storage, bootstrap] + volumes: + - ./secrets/lab5/s3_admin_access:/run/secrets/s3_access:ro + - ./secrets/lab5/s3_admin_secret:/run/secrets/s3_secret:ro + healthcheck: + disable: true + depends_on: + s3: + condition: service_healthy + operator: + <<: *api + profiles: [tools] + restart: "no" + command: [storage, reconcile] + volumes: + - ./secrets/lab5/s3_admin_access:/run/secrets/s3_access:ro + - ./secrets/lab5/s3_admin_secret:/run/secrets/s3_secret:ro + - ./tests/fixtures:/fixtures:ro + healthcheck: + disable: true + depends_on: {} + +volumes: + postgres_data: + redis_data: + s3_data: + +networks: + storage: + internal: true diff --git a/infra/haproxy.cfg b/infra/haproxy.cfg new file mode 100644 index 00000000..b0626d61 --- /dev/null +++ b/infra/haproxy.cfg @@ -0,0 +1,39 @@ +global + maxconn 4096 + stats socket /tmp/haproxy.sock mode 600 level admin + stats socket ipv4@127.0.0.1:9999 level admin + +defaults + mode http + timeout connect 500ms + timeout client 125s + timeout server 125s + timeout check 500ms + retries 0 + +resolvers docker + nameserver dns 127.0.0.11:53 + resolve_retries 2 + timeout resolve 1s + timeout retry 1s + hold valid 1s + hold obsolete 1s + hold nx 1s + +frontend public + bind :8080 + http-request deny deny_status 404 if { path_beg /internal } + default_backend api + +backend api + balance roundrobin + http-reuse always + option httpchk + http-check send meth GET uri /health/ready ver HTTP/1.1 hdr Host api + http-check expect status 200 + server-template replica 1-4 api:8080 check inter 1s fall 2 rise 1 resolvers docker init-addr none resolve-prefer ipv4 + +listen statistics + bind :8404 + stats enable + stats uri /stats diff --git a/scripts/bootstrap-storage.py b/scripts/bootstrap-storage.py new file mode 100644 index 00000000..52dc7c8f --- /dev/null +++ b/scripts/bootstrap-storage.py @@ -0,0 +1,25 @@ +#!/usr/bin/env python3 +"""Generate local synthetic storage credentials without rotating existing keys.""" +import json +import os +from pathlib import Path +import secrets + +root=Path(__file__).resolve().parents[1]/'secrets'/'lab5' +root.mkdir(parents=True,exist_ok=True) +for name in ('s3_admin_access','s3_admin_secret','s3_api_access','s3_api_secret'): + p=root/name + if not p.exists(): + p.write_text(secrets.token_hex(24)+'\n') + p.chmod(0o444) # Non-root containers must read these local synthetic secrets. +bucket=os.environ.get('S3_BUCKET','messenger-attachments') +identities=[] +for role,actions in [('admin',['Admin','Read','Write','List','Tagging']),('api',['Read:'+bucket,'Write:'+bucket,'List:'+bucket])]: + identities.append({'name':role,'credentials':[{'accessKey':(root/('s3_'+role+'_access')).read_text().strip(),'secretKey':(root/('s3_'+role+'_secret')).read_text().strip()}],'actions':actions}) +p=root/'s3.json' +content=json.dumps({'identities':identities})+'\n' +if not p.exists() or p.read_text()!=content: + if p.exists(): p.chmod(0o600) + p.write_text(content) +p.chmod(0o444) +print('Local storage credentials ready; existing keys preserved.') diff --git a/scripts/bootstrap.sh b/scripts/bootstrap.sh new file mode 100755 index 00000000..4c09f781 --- /dev/null +++ b/scripts/bootstrap.sh @@ -0,0 +1,23 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +umask 077 +mkdir -p secrets/lab5 +if [ ! -f .env.lab5 ]; then + { printf 'POSTGRES_PASSWORD='; openssl rand -hex 24 + printf 'REDIS_PASSWORD='; openssl rand -hex 24 + printf 'HOST_PORT=18085\nAPI_REPLICAS=2\n' + } > .env.lab5 +fi +if [ ! -f secrets/lab5/private_key.pem ]; then + openssl genrsa -out secrets/lab5/private_key.pem 2048 2>/dev/null +fi +if [ ! -f secrets/lab5/public_key.pem ]; then + openssl rsa -in secrets/lab5/private_key.pem -pubout -out secrets/lab5/public_key.pem 2>/dev/null +fi +# Only synthetic stand keys, readable by the non-root application inside containers. +chmod 755 secrets secrets/lab5 +chmod 444 secrets/lab5/*.pem +printf '%s\n' 'Local lab5 settings and synthetic keys ready; existing files preserved.' + +python3 scripts/bootstrap-storage.py diff --git a/scripts/compose.sh b/scripts/compose.sh new file mode 100755 index 00000000..a2190cef --- /dev/null +++ b/scripts/compose.sh @@ -0,0 +1,4 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +exec docker compose --env-file .env.lab5 "$@" diff --git a/scripts/generate-types.py b/scripts/generate-types.py new file mode 100644 index 00000000..b823e116 --- /dev/null +++ b/scripts/generate-types.py @@ -0,0 +1,17 @@ +#!/usr/bin/env python3 +"""Convert a temporary generation view to the oapi-codegen 3.0 subset. +Runtime handlers enforce anyOf and required fields; normative contract is untouched. +""" +import json +from pathlib import Path +s=json.loads(Path('contracts/openapi.json').read_text()) +for schema in s['components']['schemas'].values(): + schema.pop('anyOf',None) + for prop in schema.get('properties',{}).values(): + if prop.get('type')==['string','null']: + prop.update(type='string',nullable=True) +# Preserve pointers so handlers can distinguish missing fields from empty values. +for name,field in [('CreateUser','password'),('CreateMessage','text')]: + s['components']['schemas'][name]['properties'][field]['x-go-type-skip-optional-pointer']=False + s['components']['schemas'][name]['required'].remove(field) +print(json.dumps(s)) diff --git a/scripts/restart.sh b/scripts/restart.sh new file mode 100755 index 00000000..24699183 --- /dev/null +++ b/scripts/restart.sh @@ -0,0 +1,7 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +./scripts/compose.sh stop api lb +./scripts/compose.sh restart postgres redis s3 +./scripts/compose.sh run --rm -T s3-init +./scripts/compose.sh up -d --wait --wait-timeout 90 api lb diff --git a/scripts/rolling-update.py b/scripts/rolling-update.py new file mode 100644 index 00000000..1db08f17 --- /dev/null +++ b/scripts/rolling-update.py @@ -0,0 +1,45 @@ +#!/usr/bin/env python3 +"""Drain and replace API processes individually, retaining other container IDs.""" +import json,os,subprocess,time +from pathlib import Path +ROOT=Path(__file__).resolve().parents[1] +def run(args,input=None): + p=subprocess.run(args,cwd=ROOT,input=input,text=True,capture_output=True,timeout=100) + if p.returncode:raise RuntimeError(p.stderr.strip()[:300]) + return p.stdout.strip() +def runtime(lb,cmd):return run(['docker','exec','-i',lb,'nc','-w','1','127.0.0.1','9999'],cmd+'\n') +def ids():return run(['./scripts/compose.sh','ps','-q','api']).split() +def states(lb): + lines=runtime(lb,'show servers state').splitlines();head=lines[1].lstrip('# ').split() + return [dict(zip(head,l.split())) for l in lines[2:] if l.strip()] +def main(): + initial=ids() + if len(initial)<2:raise SystemExit('rolling update requires at least two running API replicas') + lb=run(['./scripts/compose.sh','ps','-q','lb']) + for old in initial: + info=json.loads(run(['docker','inspect',old]))[0] + ip=next(iter(info['NetworkSettings']['Networks'].values()))['IPAddress'] + backend=next(s['srv_name'] for s in states(lb) if s['srv_addr']==ip and s['be_name']=='api') + runtime(lb,f'set server api/{backend} state drain') + try: + deadline=time.monotonic()+10 + while True: + # Existing transactions get a chance to finish before SIGTERM. + import csv,io + raw=run(['docker','exec',lb,'wget','-qO-','http://127.0.0.1:8404/stats;csv']) + row=next(r for r in csv.DictReader(io.StringIO(raw)) if r['# pxname']=='api' and r['svname']==backend) + if int(row['scur'])==0:break + if time.monotonic()>deadline:raise RuntimeError('backend did not drain') + time.sleep(.1) + before=set(ids());run(['docker','stop','--time','8',old]);run(['docker','rm',old]) + run(['./scripts/compose.sh','up','-d','--no-deps','--no-recreate','--scale',f'api={len(initial)}','api']) + after=set(ids()) + if not (before-{old}).issubset(after):raise RuntimeError('another replica was replaced unexpectedly') + new=(after-before).pop();deadline=time.monotonic()+60 + while run(['docker','inspect','--format','{{.State.Health.Status}}',new])!='healthy': + if time.monotonic()>deadline:raise RuntimeError('replacement failed readiness') + time.sleep(.5) + print(json.dumps({'old_container':old,'new_container':new,'name':info['Name'],'drained_backend':backend}),flush=True) + finally:runtime(lb,f'set server api/{backend} state ready') + time.sleep(3) # allow DNS refresh and a successful active readiness check +if __name__=='__main__':main() diff --git a/scripts/rolling-update.sh b/scripts/rolling-update.sh new file mode 100755 index 00000000..c56d6281 --- /dev/null +++ b/scripts/rolling-update.sh @@ -0,0 +1,4 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +exec python3 scripts/rolling-update.py diff --git a/scripts/stop-one.sh b/scripts/stop-one.sh new file mode 100755 index 00000000..bba32904 --- /dev/null +++ b/scripts/stop-one.sh @@ -0,0 +1,17 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +: "${TARGET_INSTANCE_ID:?missing target alias}" +for id in $(./scripts/compose.sh ps -q api); do + hostname=$(docker inspect --format '{{.Config.Hostname}}' "$id") + case "$TARGET_INSTANCE_ID" in + "$hostname"-*) + current=$(docker exec "$id" wget -T 2 -qO- http://127.0.0.1:8080/internal/metrics | python3 -c 'import json,sys; print(json.load(sys.stdin)["instance_id"])') + [ "$current" = "$TARGET_INSTANCE_ID" ] || { printf '%s\n' 'Target process was already replaced' >&2; exit 1; } + docker stop --time 8 "$id" >/dev/null + exit 0 ;; + + esac +done +printf '%s\n' 'Target does not match a running lab5 API container' >&2 +exit 1 diff --git a/scripts/test-storage.sh b/scripts/test-storage.sh new file mode 100755 index 00000000..9cb83079 --- /dev/null +++ b/scripts/test-storage.sh @@ -0,0 +1,11 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +mkdir -p bin +(cd service && GOOS=linux GOARCH=$(docker info --format '{{.Architecture}}' | sed 's/aarch64/arm64/;s/x86_64/amd64/') CGO_ENABLED=0 go test -c -o ../bin/storage.test ./internal/adapter/repository/storage) +database="lab5_atomic_$(date +%s)_$$" +./scripts/compose.sh exec -T postgres psql -U messenger -d messenger -v ON_ERROR_STOP=1 -c "CREATE DATABASE $database" >/dev/null +trap './scripts/compose.sh exec -T postgres psql -U messenger -d messenger -c "DROP DATABASE IF EXISTS $database WITH (FORCE)" >/dev/null' EXIT +id=$(./scripts/compose.sh ps -q api | head -n 1) +docker cp bin/storage.test "$id:/tmp/storage.test" >/dev/null +docker exec -e "TEST_DATABASE_NAME=$database" "$id" /tmp/storage.test -test.v diff --git a/service/cmd/app/main.go b/service/cmd/app/main.go new file mode 100644 index 00000000..804126da --- /dev/null +++ b/service/cmd/app/main.go @@ -0,0 +1,150 @@ +package main + +import ( + "context" + "fmt" + "log/slog" + "os" + "os/signal" + "syscall" + "time" + + "encoding/json" + "gitea.privcloud.tech/mtusi2026/internal/adapter/controller/http" + "gitea.privcloud.tech/mtusi2026/internal/adapter/repository/cache/outside" + "gitea.privcloud.tech/mtusi2026/internal/adapter/repository/storage" + "gitea.privcloud.tech/mtusi2026/internal/config" + "gitea.privcloud.tech/mtusi2026/internal/pkg/auth/jwtprovider" + "gitea.privcloud.tech/mtusi2026/internal/pkg/cache" + "gitea.privcloud.tech/mtusi2026/internal/pkg/encrypting" + "gitea.privcloud.tech/mtusi2026/internal/pkg/migrator" + "gitea.privcloud.tech/mtusi2026/internal/pkg/objectstore" + "gitea.privcloud.tech/mtusi2026/internal/pkg/postgres" + "gitea.privcloud.tech/mtusi2026/internal/pkg/postgres/txmanager" + "gitea.privcloud.tech/mtusi2026/internal/pkg/rest/middleware" + "gitea.privcloud.tech/mtusi2026/internal/pkg/rest/router" + "gitea.privcloud.tech/mtusi2026/internal/pkg/validator" + "gitea.privcloud.tech/mtusi2026/internal/service/attachment" + "gitea.privcloud.tech/mtusi2026/internal/service/auth" + "gitea.privcloud.tech/mtusi2026/internal/service/chat" + "gitea.privcloud.tech/mtusi2026/internal/service/user" + "github.com/google/uuid" + stdhttp "net/http" +) + +func main() { + time.Local = time.UTC + if err := run(); err != nil { + slog.Error("application stopped", "error", err) + os.Exit(1) + } +} + +func run() error { + cfg, err := config.New() + if err != nil { + return fmt.Errorf("load server config: %w", err) + } + + if len(os.Args) > 1 && os.Args[1] == "migrate" { + cfg.Postgres.RunMigrations = true + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + defer cancel() + return migrator.New(cfg).Start(ctx) + } + attachmentConfig, err := config.LoadAttachment() + if err != nil { + return err + } + if len(os.Args) > 1 && os.Args[1] == "storage" { + return runStorage(cfg, attachmentConfig, os.Args[2:]) + } + + // API processes never run migrations; the one-shot migrator owns schema changes. + cfg.Postgres.RunMigrations = false + var level slog.Level + if err := level.UnmarshalText([]byte(cfg.Application.LogLevel)); err != nil { + return err + } + slog.SetDefault(slog.New(slog.NewJSONHandler(os.Stdout, &slog.HandlerOptions{Level: level}))) + hostname, _ := os.Hostname() + instanceID := hostname + "-" + uuid.NewString() + slog.Info("instance_started", "instance_id", instanceID) + pgx := postgres.New(cfg.Postgres) + rx := cache.New(cfg) + postgresMigrator := migrator.New(cfg) + defer pgx.Close() + defer rx.Close() + tm := txmanager.New(pgx) + provider := jwtprovider.New(cfg) + encryptor := encrypting.New() + + chatsRepo := storage.NewChatsRepo(pgx) + messageRepo := storage.NewMessagesRepo(pgx) + refreshTokensRepo := storage.NewRefreshTokensRepo(pgx) + userRepo := storage.NewUsersRepo(pgx) + + chatCache := outside.NewChatCache(rx) + + authSvc := auth.New(provider, tm, encryptor, userRepo, refreshTokensRepo) + userSvc := user.New(encryptor, userRepo) + chatSvc := chat.New(tm, chatsRepo, messageRepo, chatCache) + objectStore := objectstore.New(attachmentConfig) + attachmentSvc := attachment.New(&storage.AttachmentsRepo{Pool: pgx}, objectStore, attachmentConfig, chatSvc) + + httpRouter := router.New( + middleware.NewAuthMiddleware(authSvc), + middleware.Instance(instanceID), + middleware.RequestIDMiddleware(), + middleware.LoggingMiddleware(), + middleware.RequestTimeout(cfg.Postgres.QueryTimeout, attachmentConfig.Timeout), + ) + httpRouter.Get("/internal/metrics", func(w stdhttp.ResponseWriter, r *stdhttp.Request) { + stat := pgx.Stat() + _ = json.NewEncoder(w).Encode(map[string]any{"instance_id": instanceID, "acquire_count": stat.AcquireCount(), "acquire_duration_ns": stat.AcquireDuration().Nanoseconds(), "empty_acquire_count": stat.EmptyAcquireCount(), "acquired_conns": stat.AcquiredConns(), "total_conns": stat.TotalConns(), "max_conns": stat.MaxConns()}) + }) + server := http.New(cfg, httpRouter) + requestValidator := validator.NewValidatorImpl() + registerRoutes( + http.NewAuthHandler(httpRouter, authSvc), + http.NewHealthHandler(httpRouter, server, pgx, objectStore), + http.NewChatHandler(httpRouter, provider, requestValidator, chatSvc), + http.NewUserHandler(httpRouter, provider, requestValidator, userSvc), + http.NewAttachmentHandler(httpRouter, provider, attachmentSvc), + ) + + ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) + defer stop() + + if err = postgresMigrator.Start(ctx); err != nil { + return fmt.Errorf("start postgres migrator: %w", err) + } + if err = server.Start(ctx); err != nil { + return fmt.Errorf("start HTTP server: %w", err) + } + if err = rx.Start(ctx); err != nil { + slog.Warn("redis unavailable at startup; continuing without cache", "error", err) + } + slog.Info("HTTP server started", "address", cfg.Server.BindAddress()) + + <-ctx.Done() + shutdownCtx, cancel := context.WithTimeout(context.Background(), cfg.Server.ShutdownTimeout) + defer cancel() + if err = server.Stop(shutdownCtx); err != nil { + return fmt.Errorf("stop HTTP server: %w", err) + } + if err = postgresMigrator.Close(shutdownCtx); err != nil { + return fmt.Errorf("close postgres migrator: %w", err) + } + if err = rx.Close(); err != nil { + return fmt.Errorf("close redis: %w", err) + } + pgx.Close() + return nil +} + +func registerRoutes(handlers ...http.Handler) { + for _, handler := range handlers { + handler.RegisterRoutes() + } +} diff --git a/service/cmd/app/storage.go b/service/cmd/app/storage.go new file mode 100644 index 00000000..daeb6a1d --- /dev/null +++ b/service/cmd/app/storage.go @@ -0,0 +1,175 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "flag" + "fmt" + "log/slog" + "net/http" + "os" + "strings" + "time" + + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/aws/smithy-go" + "github.com/google/uuid" + + "gitea.privcloud.tech/mtusi2026/internal/adapter/repository/storage" + "gitea.privcloud.tech/mtusi2026/internal/config" + "gitea.privcloud.tech/mtusi2026/internal/pkg/objectstore" + "gitea.privcloud.tech/mtusi2026/internal/pkg/postgres" + "gitea.privcloud.tech/mtusi2026/internal/service/attachment" +) + +func runStorage(cfg *config.Config, c *config.Attachment, args []string) error { + slog.SetDefault(slog.New(slog.NewJSONHandler(os.Stderr, &slog.HandlerOptions{Level: slog.LevelWarn}))) + if len(args) == 0 { + return fmt.Errorf("storage command required") + } + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute) + defer cancel() + store := objectstore.New(c) + if args[0] == "bootstrap" { + var err error + for i := 0; i < 60; i++ { + _, err = store.Client.CreateBucket(ctx, &s3.CreateBucketInput{Bucket: &c.Bucket}) + var api smithy.APIError + if err == nil || (errors.As(err, &api) && (api.ErrorCode() == "BucketAlreadyOwnedByYou" || api.ErrorCode() == "BucketAlreadyExists")) { + if store.Ping(ctx) == nil { + // HEAD alone can succeed before the restarted master/volume write path. + // This constant key is outside the application's immutable originals prefix. + key := "bootstrap/write-check" + if store.Put(ctx, key, "application/octet-stream", strings.NewReader("ready"), 5) == nil && store.Delete(ctx, key) == nil { + return json.NewEncoder(os.Stdout).Encode(map[string]any{"bucket_ready": true, "write_checked": true}) + } + } + } + time.Sleep(time.Second) + } + return fmt.Errorf("storage bootstrap failed: %w", err) + } + if args[0] == "private-check" { + // Unsigned requests must reach the S3 server and receive an auth failure, + // rather than passing because a network error hid a public object. + fs := flag.NewFlagSet("private-check", flag.ContinueOnError) + key := fs.String("key", "", "known object key") + if e := fs.Parse(args[1:]); e != nil { + return e + } + client := http.Client{Timeout: 5 * time.Second, CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }} + resp, e := client.Get(c.Endpoint + "/" + c.Bucket + "/" + *key) + if e != nil { + return e + } + defer resp.Body.Close() + if resp.StatusCode != 401 && resp.StatusCode != 403 { + return fmt.Errorf("unsigned GET returned %d", resp.StatusCode) + } + _, e = store.Client.CreateBucket(ctx, &s3.CreateBucketInput{Bucket: new("forbidden-" + uuid.NewString())}) + var api smithy.APIError + if !errors.As(e, &api) || api.ErrorCode() != "AccessDenied" { + return fmt.Errorf("API key bucket creation was not denied") + } + return json.NewEncoder(os.Stdout).Encode(map[string]any{"anonymous_status": resp.StatusCode, "api_create_bucket_denied": true}) + } + if args[0] == "object-check" { + fs := flag.NewFlagSet("object-check", flag.ContinueOnError) + key := fs.String("key", "", "synthetic object key") + if e := fs.Parse(args[1:]); e != nil { + return e + } + head, e := store.Head(ctx, *key) + if objectstore.Missing(e) { + return json.NewEncoder(os.Stdout).Encode(map[string]any{"exists": false}) + } + if e != nil { + return e + } + return json.NewEncoder(os.Stdout).Encode(map[string]any{"exists": true, "last_modified": head.LastModified, "size_bytes": head.ContentLength}) + } + + pool := postgres.New(cfg.Postgres) + defer pool.Close() + svc := attachment.New(&storage.AttachmentsRepo{Pool: pool}, store, c, nil) + switch args[0] { + case "reconcile": + fs := flag.NewFlagSet("reconcile", flag.ContinueOnError) + apply := fs.Bool("apply", false, "apply safe changes") + if e := fs.Parse(args[1:]); e != nil { + return e + } + result, e := svc.Reconcile(ctx, *apply) + _ = json.NewEncoder(os.Stdout).Encode(result) + return e + case "restore": + fs := flag.NewFlagSet("restore", flag.ContinueOnError) + id := fs.String("id", "", "attachment id") + source := fs.String("source", "", "original file") + if e := fs.Parse(args[1:]); e != nil { + return e + } + uid, e := uuid.Parse(*id) + if e != nil { + return e + } + if e = svc.Restore(ctx, uid, *source); e != nil { + return e + } + return json.NewEncoder(os.Stdout).Encode(map[string]any{"restored": uid}) + case "seed": + fs := flag.NewFlagSet("seed", flag.ContinueOnError) + count := fs.Int("count", 1, "synthetic orphan count") + keyOverride := fs.String("key", "", "explicit key for a single synthetic object") + if e := fs.Parse(args[1:]); e != nil { + return e + } + if *count < 1 || *count > 10000 { + return fmt.Errorf("count must be 1..10000") + } + keys := []string{} + for i := 0; i < *count; i++ { + key := attachment.Prefix + uuid.NewString() + if *keyOverride != "" { + if *count != 1 { + return fmt.Errorf("explicit key requires count=1") + } + key = *keyOverride + } + if e := store.Put(ctx, key, "image/png", strings.NewReader("synthetic-orphan"), 16); e != nil { + return e + } + if *count == 1 { + keys = append(keys, key) + } + } + return json.NewEncoder(os.Stdout).Encode(map[string]any{"seeded": *count, "keys": keys}) + case "remove-object": + fs := flag.NewFlagSet("remove-object", flag.ContinueOnError) + id := fs.String("id", "", "synthetic attachment id") + if e := fs.Parse(args[1:]); e != nil { + return e + } + uid, e := uuid.Parse(*id) + if e != nil { + return e + } + a, e := svc.Repo.Get(ctx, uid) + if e != nil { + return e + } + conn, unlock, e := svc.Repo.Lock(ctx, a.Key) + _ = conn + if e != nil { + return e + } + defer unlock() + if e = store.Delete(ctx, a.Key); e != nil { + return e + } + return json.NewEncoder(os.Stdout).Encode(map[string]any{"removed_object": uid}) + default: + return fmt.Errorf("unknown storage command") + } +} diff --git a/service/cmd/benchmark/main.go b/service/cmd/benchmark/main.go new file mode 100644 index 00000000..6952953c --- /dev/null +++ b/service/cmd/benchmark/main.go @@ -0,0 +1,260 @@ +// Command benchmark generates paced lab5 traffic through the public load balancer. +package main + +import ( + "bytes" + "encoding/json" + "flag" + "fmt" + "io" + "math" + "net/http" + "os" + "slices" + "strings" + "sync" + "sync/atomic" + "time" + + "github.com/google/uuid" +) + +type client struct { + base, token string + http *http.Client +} + +func (c *client) call(method, path string, body any, key string) (map[string]any, int, error) { + var data io.Reader + if body != nil { + b, e := json.Marshal(body) + if e != nil { + return nil, 0, e + } + data = bytes.NewReader(b) + } + req, e := http.NewRequest(method, c.base+path, data) + if e != nil { + return nil, 0, e + } + if body != nil { + req.Header.Set("Content-Type", "application/json") + } + if c.token != "" { + req.Header.Set("Authorization", "Bearer "+c.token) + } + if key != "" { + req.Header.Set("Idempotency-Key", key) + } + // net/http otherwise treats Idempotency-Key + GetBody as permission to retry POST. + if method == http.MethodPost { + req.GetBody = nil + } + resp, e := c.http.Do(req) + if e != nil { + return nil, 0, e + } + defer resp.Body.Close() + b, e := io.ReadAll(io.LimitReader(resp.Body, 1<<20)) + if e != nil { + return nil, resp.StatusCode, e + } + if resp.StatusCode != 200 && resp.StatusCode != 201 { + return nil, resp.StatusCode, fmt.Errorf("HTTP %d", resp.StatusCode) + } + var out map[string]any + e = json.Unmarshal(b, &out) + return out, resp.StatusCode, e +} +func (c *client) setup(seed int) (string, error) { + username := "bench_" + strings.ReplaceAll(uuid.NewString(), "-", "")[:20] + password := "Synthetic-benchmark-123" + user, _, e := c.call("POST", "/api/v1/users", map[string]string{"username": username, "display_name": "Benchmark", "password": password}, "") + if e != nil { + return "", e + } + req, _ := http.NewRequest("POST", c.base+"/api/v1/auth/sessions", nil) + req.SetBasicAuth(username, password) + resp, e := c.http.Do(req) + if e != nil { + return "", e + } + var session struct { + Token string `json:"token"` + } + e = json.NewDecoder(resp.Body).Decode(&session) + resp.Body.Close() + if e != nil || resp.StatusCode != 201 || session.Token == "" { + return "", fmt.Errorf("login status %d: %v", resp.StatusCode, e) + } + c.token = session.Token + chat, _, e := c.call("POST", "/api/v1/chats", map[string]any{"title": "Benchmark", "member_ids": []any{user["id"]}}, "") + if e != nil { + return "", e + } + path := "/api/v1/chats/" + chat["id"].(string) + "/messages" + // Bounded concurrent setup is excluded from the measured interval. + var group sync.WaitGroup + slots := make(chan struct{}, 8) + errs := make(chan error, seed) + for i := range seed { + slots <- struct{}{} + group.Go(func() { + defer func() { <-slots }() + _, _, e := c.call("POST", path, map[string]string{"text": fmt.Sprintf("seed-%d", i)}, uuid.NewString()) + if e != nil { + errs <- e + } + }) + } + group.Wait() + close(errs) + for e := range errs { + if e != nil { + return "", e + } + } + return path, nil +} + +type result struct { + MeasurementUnix float64 `json:"measurement_started_unix"` + Label string `json:"label"` + TargetRate int `json:"target_rps"` + Duration float64 `json:"duration_seconds"` + Elapsed float64 `json:"elapsed_including_drain_seconds"` + Scheduled int `json:"scheduled"` + Dropped int `json:"generator_dropped"` + Success int `json:"successful_requests"` + Errors int `json:"failed_requests"` + ReadSuccess int `json:"read_success"` + WriteSuccess int `json:"write_success"` + SuccessfulRPS float64 `json:"successful_rps"` + ErrorRatio float64 `json:"error_ratio_including_drops"` + Latency map[string]float64 `json:"successful_latency_ms"` + Statuses map[int]int `json:"statuses"` + PeakInFlight int64 `json:"peak_in_flight"` + Note string `json:"note"` +} + +func phase(c *client, path string, rate int, duration time.Duration, maxInFlight int, label string) result { + out := result{Label: label, TargetRate: rate, Duration: duration.Seconds(), Statuses: map[int]int{}, Latency: map[string]float64{}, Note: "Paced open-loop, 10ms dispatch batches; 90% first-page GET, 10% POST; no POST retries. Percentiles of successes include scheduler delay; failures and capacity drops reported separately. RPS denominator includes drain."} + start := time.Now() + out.MeasurementUnix = float64(start.UnixNano()) / 1e9 + end := start.Add(duration) + slots := make(chan struct{}, maxInFlight) + var group sync.WaitGroup + var mu sync.Mutex + var active atomic.Int64 + var peak atomic.Int64 + latencies := make([]float64, 0, rate*int(duration.Seconds())) + index := 0 + for tick := 0; ; tick++ { + now := time.Now() + if !now.Before(end) { + break + } + target := start.Add(time.Duration(tick) * 10 * time.Millisecond) + if target.After(now) { + time.Sleep(time.Until(target)) + } + until := min(int(time.Since(start).Seconds()*float64(rate))+1, int(duration.Seconds()*float64(rate))) + for index < until { + i := index + index++ + out.Scheduled++ + scheduled := start.Add(time.Duration(float64(i) / float64(rate) * float64(time.Second))) + select { + case slots <- struct{}{}: + group.Go(func() { + n := active.Add(1) + for old := peak.Load(); n > old; old = peak.Load() { + if peak.CompareAndSwap(old, n) { + break + } + } + defer func() { active.Add(-1); <-slots }() + method, p := "GET", path+"?limit=50" + var body any + key := "" + if i%10 == 0 { + method = "POST" + p = path + key = uuid.NewString() + body = map[string]string{"text": key} + } + _, status, err := c.call(method, p, body, key) + latency := float64(time.Since(scheduled).Microseconds()) / 1000 + mu.Lock() + defer mu.Unlock() + out.Statuses[status]++ + if err != nil { + out.Errors++ + } else { + out.Success++ + if method == "GET" { + out.ReadSuccess++ + } else { + out.WriteSuccess++ + } + latencies = append(latencies, max(0, latency)) + } + }) + default: + out.Dropped++ + } + } + } + group.Wait() + out.Elapsed = time.Since(start).Seconds() + out.PeakInFlight = peak.Load() + out.SuccessfulRPS = float64(out.Success) / out.Elapsed + if out.Scheduled > 0 { + out.ErrorRatio = float64(out.Errors+out.Dropped) / float64(out.Scheduled) + } + slices.Sort(latencies) + for name, p := range map[string]float64{"p50": .5, "p95": .95, "p99": .99} { + if len(latencies) > 0 { + out.Latency[name] = latencies[max(0, int(math.Ceil(p*float64(len(latencies))))-1)] + } + } + return out +} +func main() { + base := flag.String("base-url", "http://localhost:18085", "public LB URL") + rate := flag.Int("rate", 100, "scheduled requests per second") + duration := flag.Duration("duration", 30*time.Second, "measured time") + warmup := flag.Duration("warmup", 5*time.Second, "warmup excluded from results") + seed := flag.Int("seed", 1000, "initial messages") + maxFlight := flag.Int("max-in-flight", 256, "bounded in-flight limit") + label := flag.String("label", "manual", "case label") + output := flag.String("output", "", "JSON destination") + flag.Parse() + if *rate < 1 || *rate > 100000 || *duration <= 0 || *warmup < 0 || *maxFlight < 1 || *seed < 0 { + fmt.Fprintln(os.Stderr, "invalid benchmark settings") + os.Exit(2) + } + c := &client{base: *base, http: &http.Client{Timeout: 2 * time.Second, Transport: &http.Transport{MaxIdleConns: 512, MaxIdleConnsPerHost: 512, MaxConnsPerHost: *maxFlight, IdleConnTimeout: 60 * time.Second}}} + path, e := c.setup(*seed) + if e != nil { + fmt.Fprintln(os.Stderr, "setup:", e) + os.Exit(1) + } + if *warmup > 0 { + phase(c, path, *rate, *warmup, *maxFlight, "warmup") + } + row := phase(c, path, *rate, *duration, *maxFlight, *label) + b, e := json.MarshalIndent(row, "", " ") + if e != nil { + panic(e) + } + if *output != "" { + if e = os.WriteFile(*output, append(b, '\n'), 0644); e != nil { + panic(e) + } + } + fmt.Println(string(b)) + if row.Success == 0 { + os.Exit(1) + } +} diff --git a/service/cmd/benchmark/main_test.go b/service/cmd/benchmark/main_test.go new file mode 100644 index 00000000..63e10344 --- /dev/null +++ b/service/cmd/benchmark/main_test.go @@ -0,0 +1,42 @@ +package main + +import ( + "io" + "net/http" + "net/http/httptest" + "sync/atomic" + "testing" + "time" +) + +func TestLostPostResponseIsNotRetried(t *testing.T) { + var posts atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method == http.MethodGet { + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{}`)) + return + } + posts.Add(1) + _, _ = io.Copy(io.Discard, r.Body) + conn, _, err := w.(http.Hijacker).Hijack() + if err != nil { + t.Error(err) + return + } + _ = conn.Close() + })) + defer server.Close() + transport := &http.Transport{MaxIdleConnsPerHost: 1} + defer transport.CloseIdleConnections() + c := &client{base: server.URL, http: &http.Client{Timeout: time.Second, Transport: transport}} + if _, _, err := c.call(http.MethodGet, "/warm", nil, ""); err != nil { + t.Fatal(err) + } + if _, _, err := c.call(http.MethodPost, "/message", map[string]string{"text": "uncertain result"}, "same-key"); err == nil { + t.Fatal("lost response must be reported as an error") + } + if posts.Load() != 1 { + t.Fatalf("transport sent %d POSTs, want exactly 1", posts.Load()) + } +} diff --git a/service/go.mod b/service/go.mod new file mode 100644 index 00000000..b816a855 --- /dev/null +++ b/service/go.mod @@ -0,0 +1,66 @@ +module gitea.privcloud.tech/mtusi2026 + +go 1.27.0 + +require ( + github.com/aws/aws-sdk-go-v2 v1.47.8 + github.com/aws/aws-sdk-go-v2/credentials v1.20.16 + github.com/aws/aws-sdk-go-v2/service/s3 v1.114.9 + github.com/aws/smithy-go v1.28.8 + github.com/go-chi/chi/v5 v5.5.0 + github.com/go-playground/validator/v10 v10.30.5 + github.com/golang-jwt/jwt/v5 v5.3.1 + github.com/google/uuid v1.6.0 + github.com/jackc/pgerrcode v0.0.0-20250907135507-afb5586c32a6 + github.com/jackc/pgx/v5 v5.11.0 + github.com/jackc/tern/v2 v2.4.3 + github.com/mcosta74/pgx-slog v0.4.1 + github.com/oapi-codegen/runtime v1.4.2 + github.com/redis/go-redis/v9 v9.23.0 + github.com/spf13/viper v1.21.0 + github.com/vgarvardt/pgx-google-uuid/v5 v5.6.0 + go.uber.org/multierr v1.11.0 + golang.org/x/crypto v0.57.0 + golang.org/x/sync v0.23.0 + google.golang.org/grpc v1.85.0 +) + +require ( + dario.cat/mergo v1.0.2 // indirect + github.com/Masterminds/goutils v1.1.1 // indirect + github.com/Masterminds/semver/v3 v3.5.0 // indirect + github.com/Masterminds/sprig/v3 v3.3.0 // indirect + github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.26 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.11 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.11 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.11 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.24 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.12 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.11 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.11 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/fsnotify/fsnotify v1.9.0 // indirect + github.com/gabriel-vasile/mimetype v1.4.15 // indirect + github.com/go-playground/locales v0.14.1 // indirect + github.com/go-playground/universal-translator v0.18.1 // indirect + github.com/go-viper/mapstructure/v2 v2.4.0 // indirect + github.com/huandu/xstrings v1.5.0 // indirect + github.com/jackc/pgpassfile v1.0.0 // indirect + github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect + github.com/jackc/puddle/v2 v2.2.2 // indirect + github.com/leodido/go-urn v1.5.0 // indirect + github.com/mitchellh/copystructure v1.2.0 // indirect + github.com/mitchellh/reflectwalk v1.0.2 // indirect + github.com/pelletier/go-toml/v2 v2.4.3-0.20260624044523-35f78d5908f1 // indirect + github.com/sagikazarmark/locafero v0.11.0 // indirect + github.com/shopspring/decimal v1.4.0 // indirect + github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8 // indirect + github.com/spf13/afero v1.15.0 // indirect + github.com/spf13/cast v1.10.0 // indirect + github.com/spf13/pflag v1.0.10 // indirect + github.com/subosito/gotenv v1.6.0 // indirect + go.uber.org/atomic v1.12.0 // indirect + go.yaml.in/yaml/v3 v3.0.5 // indirect + golang.org/x/sys v0.48.0 // indirect + golang.org/x/text v0.42.0 // indirect +) diff --git a/service/go.sum b/service/go.sum new file mode 100644 index 00000000..9c564cfe --- /dev/null +++ b/service/go.sum @@ -0,0 +1,147 @@ +dario.cat/mergo v1.0.2 h1:85+piFYR1tMbRrLcDwR18y4UKJ3aH1Tbzi24VRW1TK8= +dario.cat/mergo v1.0.2/go.mod h1:E/hbnu0NxMFBjpMIE34DRGLWqDy0g5FuKDhCb31ngxA= +github.com/Masterminds/goutils v1.1.1 h1:5nUrii3FMTL5diU80unEVvNevw1nH4+ZV4DSLVJLSYI= +github.com/Masterminds/goutils v1.1.1/go.mod h1:8cTjp+g8YejhMuvIA5y2vz3BpJxksy863GQaJW2MFNU= +github.com/Masterminds/semver/v3 v3.5.0 h1:kQceYJfbupGfZOKZQg0kou0DgAKhzDg2NZPAwZ/2OOE= +github.com/Masterminds/semver/v3 v3.5.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM= +github.com/Masterminds/sprig/v3 v3.3.0 h1:mQh0Yrg1XPo6vjYXgtf5OtijNAKJRNcTdOOGZe3tPhs= +github.com/Masterminds/sprig/v3 v3.3.0/go.mod h1:Zy1iXRYNqNLUolqCpL4uhk6SHUMAOSCzdgBfDb35Lz0= +github.com/aws/aws-sdk-go-v2 v1.47.8 h1:fuClo7Un+Bi0EDTnSIPWxVEoJcze+wkGzNA4Vo3jRcQ= +github.com/aws/aws-sdk-go-v2 v1.47.8/go.mod h1:I8lFSH4MGrZiw2m7miCrRYHsgpC1+HW8uiVGOiPYPcw= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.26 h1:SJNA4XaM8Z09adeTHf/vME93jzl44lJV70pSawIBXac= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.26/go.mod h1:l0ME5pd2QytH8UZzETcrt90x9Bm+3TyBVeWfVEcZ82c= +github.com/aws/aws-sdk-go-v2/credentials v1.20.16 h1:cqRWTY4fcctJGqZedMC1Khu4h2VsgRODUOIZwn39n3Y= +github.com/aws/aws-sdk-go-v2/credentials v1.20.16/go.mod h1:x8JgCOAnbnEVQOLVX6lOtsyTrG2EHgiRNqzku9jiVxA= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.11 h1:LTxHSJ/rwTFfaXqi5mlKKHJOW+shFB+HQ7NrCa4OQ48= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.11/go.mod h1:GF0bVKZVQX4UeUtPmTuWZj3gw0eRCdkVhW0xFnSRLmU= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.11 h1:DGUM0yztI3Zz5kePulT/mMP/gVZFGTTISUiCcqSQtT0= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.11/go.mod h1:kbcdJgN9cgSzZlUqsMveDl9IfexJqOzCihXjQXTNaPk= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.11 h1:kOHCwgWFyfrz9GqnJct8knQw7Apyl3xHH+OGnome77s= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.11/go.mod h1:854cZkT2fdkdrzQklD49uzxkZnajXoelpVNgT/yk4J0= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.24 h1:jGzHjc2jDx08Rji1JTBtMxuXUDG7muTEw+ExYXqFzPg= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.24/go.mod h1:uJt75pK2cEyY80F5C+DK9YkCkJWsMkJViHxLYZRHl8Y= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.12 h1:hfO8FblXX93ZKHXSCqCjehwul1EtNJa9G8iFSOiJ+6w= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.12/go.mod h1:r8Eypc7ZRNdS7BsaLBUUdcXIEuKE6cVHZJ29u+IGaIc= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.11 h1:PO7Tx443ecMI00nIxDrT99DVcbeMBQ6BUtJlDMRnY24= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.11/go.mod h1:n/9RixikO12YvNomZLP8k4TgUSoEkT1W23QIrpFAPvw= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.11 h1:jNP7wZylDeSJhjc1nmjZBuG5deotp3cEYIfMazmzp+Y= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.11/go.mod h1:1US+8JBnF0xNm6GE6wM5u5ugFwSZpkd2686gMuql5tI= +github.com/aws/aws-sdk-go-v2/service/s3 v1.114.9 h1:MLHGyNy4melTyk9e63s+9QgLKCHPDJrP+1NfJhtU1g4= +github.com/aws/aws-sdk-go-v2/service/s3 v1.114.9/go.mod h1:zfUkiNK2JiYlR1/SnU8aJ8Gj1nMFLkPZ69rVLv6jWPo= +github.com/aws/smithy-go v1.28.8 h1:aEmdAq7gVCoGCnvf1NnELlu3I6D/nhqrC321FyFiQkQ= +github.com/aws/smithy-go v1.28.8/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc= +github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs= +github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c= +github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA= +github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= +github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= +github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k= +github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= +github.com/gabriel-vasile/mimetype v1.4.15 h1:05iP/CYtZ/w455R/KZM6rZ5ieAdh99UPtd+d3YzLmaI= +github.com/gabriel-vasile/mimetype v1.4.15/go.mod h1:azpTcoLcDZRNgFou5j+APrqQx9HqVPWa6ijYQIIVswQ= +github.com/go-chi/chi/v5 v5.5.0 h1:xdKQ1dJmVJsmgahrXSKhUx8NyVPzpwdlyrrQFpMPJgg= +github.com/go-chi/chi/v5 v5.5.0/go.mod h1:d1IG7GtKPSjPyBP365PH8sAGVHk3qfB/IhjFNRH1wWM= +github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s= +github.com/go-playground/assert/v2 v2.2.0/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= +github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA= +github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY= +github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY= +github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= +github.com/go-playground/validator/v10 v10.30.5 h1:YyCXvVShZbs2Sm3Mb53eNOlhRXctSOzW5QJAouCTZL4= +github.com/go-playground/validator/v10 v10.30.5/go.mod h1:wEqiaov48pXX1kjhc3Da8y0M0Dtg/BK7gurFBLgwFrQ= +github.com/go-viper/mapstructure/v2 v2.4.0 h1:EBsztssimR/CONLSZZ04E8qAkxNYq4Qp9LvH92wZUgs= +github.com/go-viper/mapstructure/v2 v2.4.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= +github.com/golang-jwt/jwt/v5 v5.3.1 h1:kYf81DTWFe7t+1VvL7eS+jKFVWaUnK9cB1qbwn63YCY= +github.com/golang-jwt/jwt/v5 v5.3.1/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/huandu/xstrings v1.5.0 h1:2ag3IFq9ZDANvthTwTiqSSZLjDc+BedvHPAp5tJy2TI= +github.com/huandu/xstrings v1.5.0/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE= +github.com/jackc/pgerrcode v0.0.0-20250907135507-afb5586c32a6 h1:D/V0gu4zQ3cL2WKeVNVM4r2gLxGGf6McLwgXzRTo2RQ= +github.com/jackc/pgerrcode v0.0.0-20250907135507-afb5586c32a6/go.mod h1:a/s9Lp5W7n/DD0VrVoyJ00FbP2ytTPDVOivvn2bMlds= +github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= +github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= +github.com/jackc/pgx/v5 v5.11.0 h1:IzBBtyK9AHqf98cctWFifYSci2hgQR/cd56wB4p+ogg= +github.com/jackc/pgx/v5 v5.11.0/go.mod h1:mal1tBGAFfLHvZzaYh77YS/eC6IX9OWbRV1QIIM0Jn4= +github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo= +github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= +github.com/jackc/tern/v2 v2.4.3 h1:g293d3OZgW7OFEhsYXgEv0C21jea2boNr0VR4K5I7OY= +github.com/jackc/tern/v2 v2.4.3/go.mod h1:rMpMuRYcff5wWLptoTSO1qcDxJ4OodysvK17i2SVBys= +github.com/klauspost/cpuid/v2 v2.4.0 h1:S6Hrbc7+ywsr0r+RLapfGBHfyefhCTwEh3A0tV913Dw= +github.com/klauspost/cpuid/v2 v2.4.0/go.mod h1:19jmZ9mjzoF//ddRSUsv0zfBTJWh3QJh9FNxZTMrGxU= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/leodido/go-urn v1.5.0 h1:pLqT2kq1zpHW/1D18QMjMpdtX7cekxqtJJjg5ANyWw0= +github.com/leodido/go-urn v1.5.0/go.mod h1:9BORnCDhdPBJNDEX+w1bJisa8yOKYi116VeO96s4ifE= +github.com/mcosta74/pgx-slog v0.4.1 h1:Rt25l/jE5tu1ioqPDrqY17Kv6REdsM0L9WCT+hFw1rw= +github.com/mcosta74/pgx-slog v0.4.1/go.mod h1:BCpubkiENkWQ8MvZ4a9LJgWuBzC5UpWoJmQ1/SOlv+M= +github.com/mitchellh/copystructure v1.2.0 h1:vpKXTN4ewci03Vljg/q9QvCGUDttBOGBIa15WveJJGw= +github.com/mitchellh/copystructure v1.2.0/go.mod h1:qLl+cE2AmVv+CoeAwDPye/v+N2HKCj9FbZEVFJRxO9s= +github.com/mitchellh/reflectwalk v1.0.2 h1:G2LzWKi524PWgd3mLHV8Y5k7s6XUvT0Gef6zxSIeXaQ= +github.com/mitchellh/reflectwalk v1.0.2/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= +github.com/oapi-codegen/runtime v1.4.2 h1:GMxFVYLzoYLua+/KvzgSphkyK1lLTReQI9Vf4hvATKE= +github.com/oapi-codegen/runtime v1.4.2/go.mod h1:GwV7hC2hviaMzj+ITfHVRESK5J2W/GefVwIND/bMGvU= +github.com/pelletier/go-toml/v2 v2.4.3-0.20260624044523-35f78d5908f1 h1:gIZQQ2+wbsdQGWLPDSdJFptR25tBCjSC0KbSIQZ1rgE= +github.com/pelletier/go-toml/v2 v2.4.3-0.20260624044523-35f78d5908f1/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/redis/go-redis/v9 v9.23.0 h1:/TTVdJa8BRNsybwwGGnmiYktHNlUXtbOQxlWpHBPL3s= +github.com/redis/go-redis/v9 v9.23.0/go.mod h1:EWP2UUk+XDaI1s5nQB5mablZN6vccXlmBQTyfnF87D0= +github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= +github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= +github.com/sagikazarmark/locafero v0.11.0 h1:1iurJgmM9G3PA/I+wWYIOw/5SyBtxapeHDcg+AAIFXc= +github.com/sagikazarmark/locafero v0.11.0/go.mod h1:nVIGvgyzw595SUSUE6tvCp3YYTeHs15MvlmU87WwIik= +github.com/shopspring/decimal v1.4.0 h1:bxl37RwXBklmTi0C79JfXCEBD1cqqHt0bbgBAGFp81k= +github.com/shopspring/decimal v1.4.0/go.mod h1:gawqmDU56v4yIKSwfBSFip1HdCCXN8/+DMd9qYNcwME= +github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8 h1:+jumHNA0Wrelhe64i8F6HNlS8pkoyMv5sreGx2Ry5Rw= +github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8/go.mod h1:3n1Cwaq1E1/1lhQhtRK2ts/ZwZEhjcQeJQ1RuC6Q/8U= +github.com/spf13/afero v1.15.0 h1:b/YBCLWAJdFWJTN9cLhiXXcD7mzKn9Dm86dNnfyQw1I= +github.com/spf13/afero v1.15.0/go.mod h1:NC2ByUVxtQs4b3sIUphxK0NioZnmxgyCrfzeuq8lxMg= +github.com/spf13/cast v1.10.0 h1:h2x0u2shc1QuLHfxi+cTJvs30+ZAHOGRic8uyGTDWxY= +github.com/spf13/cast v1.10.0/go.mod h1:jNfB8QC9IA6ZuY2ZjDp0KtFO2LZZlg4S/7bzP6qqeHo= +github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk= +github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/viper v1.21.0 h1:x5S+0EU27Lbphp4UKm1C+1oQO+rKx36vfCoaVebLFSU= +github.com/spf13/viper v1.21.0/go.mod h1:P0lhsswPGWD/1lZJ9ny3fYnVqxiegrlNrEmgLjbTCAY= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.12.1 h1:EuwCh5fleGS7H32xRwO3wRGT7DxrDhLAT6FF8MpWDWE= +github.com/stretchr/testify v1.12.1/go.mod h1:MDEgiDPPsNp5cuIrHPPCyornHKgEVbtFUmoNlxoYthg= +github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8= +github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= +github.com/vgarvardt/pgx-google-uuid/v5 v5.6.0 h1:EhPtK0mgrgaTMXpegE69hvoSOVC1Ahk8+QJ9B8b+OdU= +github.com/vgarvardt/pgx-google-uuid/v5 v5.6.0/go.mod h1:5LtFrNEkgzxHvXPO9eOvcXsSn9/KeKYgx9kjeI2oXQI= +github.com/zeebo/xxh3 v1.1.0 h1:s7DLGDK45Dyfg7++yxI0khrfwq9661w9EN78eP/UZVs= +github.com/zeebo/xxh3 v1.1.0/go.mod h1:IisAie1LELR4xhVinxWS5+zf1lA4p0MW4T+w+W07F5s= +go.uber.org/atomic v1.12.0 h1:BvcXdFKuviU4fTL/f+SxdQ5qJX/Jix8pAkgdUcb3XOE= +go.uber.org/atomic v1.12.0/go.mod h1:I6c4cg+6HCxRjfjSsYtApoFILnpc0CGUdGkXVqbYVNk= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw= +go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg= +golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M= +golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA= +golang.org/x/sync v0.23.0 h1:KameEIfc1IkluZyXWLn39Wd4tURc6GbCiISGiZm2bQk= +golang.org/x/sync v0.23.0/go.mod h1:sUUOizhqBxiL6pEWpqNLUiaJn1ShEbZ6BBqskPbjZm0= +golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo= +golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= +golang.org/x/text v0.42.0 h1:JbOZXgfeCPU9gacVtYliJqOhD+zhrEqK4LfdpmlUZqI= +golang.org/x/text v0.42.0/go.mod h1:ojzP1Z+2QtioaF8DTtO8K5q7JWVVYwZKenzujK0Zd0E= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260817212433-ac3dfec99bb1 h1:utmqiUzZAEINbLmRyAt1QjVkMyLQ+lvIwCpLaB1rZfw= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260817212433-ac3dfec99bb1/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= +google.golang.org/grpc v1.85.0 h1:BPBCfuVRQ1qvBhKDOq9uvQllMvO59rKsYhXpp7Dd8vw= +google.golang.org/grpc v1.85.0/go.mod h1:0rQrOD58yjal/0EaVbGBANindixsCZNQPDV3VKtaKuQ= +google.golang.org/protobuf v1.36.12 h1:pJOKDDOyeXErUroCihFAd5LQuwXBSpVnKGrj5o/fwxc= +google.golang.org/protobuf v1.36.12/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/service/internal/adapter/controller/http/attachment.go b/service/internal/adapter/controller/http/attachment.go new file mode 100644 index 00000000..d02cfda6 --- /dev/null +++ b/service/internal/adapter/controller/http/attachment.go @@ -0,0 +1 @@ +package http diff --git a/service/internal/adapter/controller/http/attachmentHandler.go b/service/internal/adapter/controller/http/attachmentHandler.go new file mode 100644 index 00000000..577723c4 --- /dev/null +++ b/service/internal/adapter/controller/http/attachmentHandler.go @@ -0,0 +1,91 @@ +package http + +import ( + "gitea.privcloud.tech/mtusi2026/internal/pkg/apperr" + "gitea.privcloud.tech/mtusi2026/internal/pkg/auth/jwtprovider" + "gitea.privcloud.tech/mtusi2026/internal/pkg/rest/framework" + "gitea.privcloud.tech/mtusi2026/internal/pkg/rest/router" + "gitea.privcloud.tech/mtusi2026/internal/service/attachment" + "io" + "net/http" + "strconv" + "strings" +) + +type AttachmentHandler struct { + router router.Router + provider jwtprovider.Provider + svc *attachment.Service +} + +func NewAttachmentHandler(r router.Router, p jwtprovider.Provider, s *attachment.Service) Handler { + return &AttachmentHandler{r, p, s} +} +func (h *AttachmentHandler) RegisterRoutes() { + h.router.Private("POST", "/api/v1/chats/{chat_id}/attachments", h.Upload) + h.router.Private("GET", "/api/v1/attachments/{attachment_id}", h.Get) + h.router.Private("GET", "/api/v1/attachments/{attachment_id}/content", h.Get) +} +func (h *AttachmentHandler) Upload(w http.ResponseWriter, r *http.Request) { + user, e := framework.ParseToken(r, h.provider) + if e != nil { + framework.JSONError(r.Context(), w, e) + return + } + id, e := framework.ParseIDFromPathParam(r.Context(), w, r, "chat_id") + if e != nil { + return + } + values := r.URL.Query()["filename"] + if len(values) != 1 { + framework.JSONError(r.Context(), w, apperr.New(apperr.CodeBadRequest, "expected one filename")) + return + } + a, e := h.svc.Upload(w, r, user.UserID, id, values[0]) + if e != nil { + if apperr.CodeFromError(e) == apperr.CodeTooManyRequests { + w.Header().Set("Retry-After", "1") + } + framework.JSONError(r.Context(), w, e) + return + } + framework.JSONResponse(w, http.StatusCreated, a) +} +func (h *AttachmentHandler) Get(w http.ResponseWriter, r *http.Request) { + user, e := framework.ParseToken(r, h.provider) + if e != nil { + framework.JSONError(r.Context(), w, e) + return + } + id, e := framework.ParseIDFromPathParam(r.Context(), w, r, "attachment_id") + if e != nil { + return + } + a, e := h.svc.Get(r.Context(), user.UserID, id) + if e != nil { + framework.JSONError(r.Context(), w, e) + return + } + if !strings.HasSuffix(r.URL.Path, "/content") { + framework.JSONResponse(w, 200, a.Public) + return + } + if v := r.URL.Query().Get("variant"); v != "" && v != "original" { + framework.JSONError(r.Context(), w, apperr.New(apperr.CodeBadRequest, "invalid variant")) + return + } + obj, e := h.svc.Store.Get(r.Context(), a.Key) + if e != nil { + framework.JSONError(r.Context(), w, attachment.Unavailable()) + return + } + defer obj.Body.Close() + w.Header().Set("Content-Type", string(a.Public.ContentType)) + w.Header().Set("Content-Length", strconv.Itoa(a.Public.SizeBytes)) + w.Header().Set("X-Content-Type-Options", "nosniff") + // Once streaming has started a transport failure terminates the response; no JSON + // error can be appended to an image. Content-Length exposes incomplete downloads. + if _, e = io.Copy(w, obj.Body); e != nil { + panic(http.ErrAbortHandler) + } +} diff --git a/service/internal/adapter/controller/http/auth.go b/service/internal/adapter/controller/http/auth.go new file mode 100644 index 00000000..73b263ca --- /dev/null +++ b/service/internal/adapter/controller/http/auth.go @@ -0,0 +1,88 @@ +package http + +import ( + "net/http" + + "gitea.privcloud.tech/mtusi2026/internal/dto" + "gitea.privcloud.tech/mtusi2026/internal/gen/contracts" + "gitea.privcloud.tech/mtusi2026/internal/pkg/apperr" + "gitea.privcloud.tech/mtusi2026/internal/pkg/rest/framework" + "gitea.privcloud.tech/mtusi2026/internal/pkg/rest/router" + "gitea.privcloud.tech/mtusi2026/internal/service/auth" +) + +type AuthHandler struct { + router router.Router + + auth auth.API +} + +func NewAuthHandler(r router.Router, api auth.API) Handler { + return &AuthHandler{ + router: r, + auth: api, + } +} + +func (h *AuthHandler) RegisterRoutes() { + h.router.Public(http.MethodPost, "/api/v1/auth/sessions", h.Login) + h.router.Private(http.MethodDelete, "/api/v1/auth/sessions/current", h.Logout) +} + +func (h *AuthHandler) Login(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + + username, password, ok := r.BasicAuth() + if !ok || username == "" || password == "" { + w.Header().Set( + "WWW-Authenticate", + `Basic realm="messenger", charset="UTF-8"`, + ) + framework.JSONError(ctx, w, apperr.New(apperr.CodeUnauthorized, "invalid credentials")) + return + } + + resp, err := h.auth.Login(ctx, dto.Login{ + Username: username, + Password: password, + }) + if err != nil { + if apperr.HTTPStatusFromError(err) == http.StatusUnauthorized { + w.Header().Set("WWW-Authenticate", `Basic realm="messenger", charset="UTF-8"`) + } + framework.JSONError(ctx, w, err) + return + } + + framework.JSONResponse(w, http.StatusCreated, contracts.Session{ + ExpiresAt: resp.ExpiresAt, + Token: resp.Tokens.RefreshToken.Payload, + TokenType: resp.TokenType, + User: contracts.User{ + CreatedAt: resp.User.CreatedAt, + DisplayName: resp.User.DisplayName, + Id: resp.User.ID, + Username: resp.User.Username, + }, + }) +} + +func (h *AuthHandler) Logout(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + + token, exists := framework.Token(r) + if !exists { + framework.JSONError(ctx, w, apperr.New(apperr.CodeUnauthorized, "missing token in header")) + return + } + + err := h.auth.Logout(ctx, dto.RefreshToken{ + Payload: token, + }) + if err != nil { + framework.JSONError(ctx, w, err) + return + } + + framework.JSONResponse(w, http.StatusNoContent, nil) +} diff --git a/service/internal/adapter/controller/http/chat.go b/service/internal/adapter/controller/http/chat.go new file mode 100644 index 00000000..dcf583de --- /dev/null +++ b/service/internal/adapter/controller/http/chat.go @@ -0,0 +1,287 @@ +package http + +import ( + "bytes" + "encoding/json" + "net/http" + "regexp" + "strings" + "unicode/utf8" + + "github.com/google/uuid" + + "gitea.privcloud.tech/mtusi2026/internal/dto" + "gitea.privcloud.tech/mtusi2026/internal/gen/contracts" + "gitea.privcloud.tech/mtusi2026/internal/pkg/apperr" + "gitea.privcloud.tech/mtusi2026/internal/pkg/auth/jwtprovider" + "gitea.privcloud.tech/mtusi2026/internal/pkg/rest/framework" + "gitea.privcloud.tech/mtusi2026/internal/pkg/rest/router" + "gitea.privcloud.tech/mtusi2026/internal/pkg/validator" + "gitea.privcloud.tech/mtusi2026/internal/service/chat" +) + +var messageKeyPattern = regexp.MustCompile(`^[A-Za-z0-9._:-]{1,128}$`) + +type ChatHandler struct { + router router.Router + provider jwtprovider.Provider + validator *validator.ValidatorImpl + + chat chat.API +} + +func NewChatHandler(r router.Router, provider jwtprovider.Provider, v *validator.ValidatorImpl, api chat.API) Handler { + return &ChatHandler{ + router: r, + provider: provider, + validator: v, + chat: api, + } +} + +func (h *ChatHandler) RegisterRoutes() { + h.router.Private(http.MethodPost, "/api/v1/chats", h.Create) + h.router.Private(http.MethodGet, "/api/v1/chats", h.Get) + h.router.Private(http.MethodGet, "/api/v1/chats/{chat_id}", h.GetByID) + h.router.Private(http.MethodPost, "/api/v1/chats/{chat_id}/messages", h.SendMessage) + h.router.Private(http.MethodGet, "/api/v1/chats/{chat_id}/messages", h.GetMessages) +} + +func (h *ChatHandler) Create(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + + userData, err := framework.ParseToken(r, h.provider) + if err != nil { + framework.JSONError(ctx, w, err) + return + } + + var req contracts.CreateChat + if !framework.Decode(w, r, &req) || !framework.Validate(w, r, h.validator, &req) { + return + } + + if userData.UserID == uuid.Nil || + !validText(req.Title, 100) || + len(req.MemberIds) < 1 || + len(req.MemberIds) > 100 { + framework.JSONError(ctx, w, apperr.New(apperr.CodeBadRequest, "invalid chat fields")) + return + } + + resp, err := h.chat.CreateChat(ctx, dto.CreateChat{ + CreatorID: userData.UserID, + Title: req.Title, + MemberIDs: req.MemberIds, + }) + if err != nil { + framework.JSONError(ctx, w, err) + return + } + + framework.JSONResponse(w, http.StatusCreated, contracts.Chat{ + Id: resp.ID, + Title: resp.Title, + MemberIds: resp.MemberIDs, + CreatedAt: resp.CreatedAt, + }) +} + +func (h *ChatHandler) Get(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + + userData, err := framework.ParseToken(r, h.provider) + if err != nil { + framework.JSONError(ctx, w, err) + return + } + + limit, afterID, ok := framework.PageParams(w, r) + if !ok { + return + } + + if userData.UserID == uuid.Nil { + framework.JSONError(ctx, w, apperr.New(apperr.CodeBadRequest, "invalid user id")) + return + } + if afterID != nil && *afterID == uuid.Nil { + framework.JSONError(ctx, w, apperr.New(apperr.CodeBadRequest, "invalid cursor")) + return + } + + page, err := h.chat.ListForMember(ctx, dto.ChatFilters{ + UserID: userData.UserID, + Limit: limit, + AfterID: afterID, + }) + if err != nil { + framework.JSONError(ctx, w, err) + return + } + + items := make([]contracts.Chat, 0, len(page.Items)) + for _, item := range page.Items { + items = append(items, contracts.Chat{ + CreatedAt: item.CreatedAt, + Id: item.ID, + MemberIds: item.MemberIDs, + Title: item.Title, + }) + } + + framework.JSONResponse(w, http.StatusOK, contracts.ChatPage{ + Items: items, + NextCursor: cursorString(page.NextCursorID), + }) +} + +func (h *ChatHandler) GetByID(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + + userData, err := framework.ParseToken(r, h.provider) + if err != nil { + framework.JSONError(ctx, w, err) + return + } + + chatID, err := framework.ParseIDFromPathParam(ctx, w, r, "chat_id") + if err != nil { + return + } + + resp, err := h.chat.GetForMember(ctx, dto.ChatMember{ + UserID: userData.UserID, + ChatID: chatID, + }) + if err != nil { + framework.JSONError(ctx, w, err) + return + } + + framework.JSONResponse(w, http.StatusOK, contracts.Chat{ + CreatedAt: resp.CreatedAt, + Id: resp.ID, + MemberIds: resp.MemberIDs, + Title: resp.Title, + }) +} + +func (h *ChatHandler) SendMessage(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + + userData, err := framework.ParseToken(r, h.provider) + if err != nil { + framework.JSONError(ctx, w, err) + return + } + + chatID, err := framework.ParseIDFromPathParam(ctx, w, r, "chat_id") + if err != nil { + return + } + + keyValues := r.Header.Values("Idempotency-Key") + if len(keyValues) != 1 || !messageKeyPattern.MatchString(keyValues[0]) { + framework.JSONError(ctx, w, apperr.New(apperr.CodeBadRequest, "invalid idempotency key")) + return + } + + var req struct { + Text *string `json:"text"` + AttachmentIDs json.RawMessage `json:"attachment_ids"` + } + if !framework.Decode(w, r, &req) || !framework.Validate(w, r, h.validator, &req) { + return + } + + if req.Text == nil { + framework.JSONError(ctx, w, apperr.New(apperr.CodeBadRequest, "text is required")) + return + } + + ids := []uuid.UUID{} + if len(req.AttachmentIDs) > 0 { + if bytes.Equal(bytes.TrimSpace(req.AttachmentIDs), []byte("null")) || json.Unmarshal(req.AttachmentIDs, &ids) != nil { + framework.JSONError(ctx, w, apperr.New(apperr.CodeBadRequest, "invalid attachment_ids")) + return + } + } + + message, err := h.chat.CreateMessage(ctx, dto.CreateMessage{ + IdempotencyKey: keyValues[0], + ChatID: chatID, + SenderID: userData.UserID, + Text: *req.Text, + AttachmentIDs: ids, + }) + if err != nil { + framework.JSONError(ctx, w, err) + return + } + + framework.JSONResponse(w, http.StatusCreated, contracts.Message{ + Id: message.ID, + ChatId: message.ChatID, + SenderId: message.SenderID, + Text: message.Text, + CreatedAt: message.CreatedAt, + AttachmentIds: message.AttachmentIDs, + }) +} + +func (h *ChatHandler) GetMessages(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + + userData, err := framework.ParseToken(r, h.provider) + if err != nil { + framework.JSONError(ctx, w, err) + return + } + + chatID, err := framework.ParseIDFromPathParam(ctx, w, r, "chat_id") + if err != nil { + return + } + + limit, afterID, ok := framework.PageParams(w, r) + if !ok { + return + } + + page, err := h.chat.ListForChat(ctx, dto.MessageFilters{ + ChatID: chatID, + UserID: userData.UserID, + Limit: limit, + AfterID: afterID, + }) + if err != nil { + framework.JSONError(ctx, w, err) + return + } + + items := make([]contracts.Message, 0, len(page.Items)) + for _, message := range page.Items { + items = append(items, contracts.Message{ + Id: message.ID, + ChatId: message.ChatID, + SenderId: message.SenderID, + Text: message.Text, + CreatedAt: message.CreatedAt, + AttachmentIds: message.AttachmentIDs, + }) + } + framework.JSONResponse(w, http.StatusOK, contracts.MessagePage{Items: items, NextCursor: cursorString(page.NextCursorID)}) +} + +func cursorString(id *uuid.UUID) *string { + if id == nil { + return nil + } + value := id.String() + return &value +} + +func validText(value string, maxLength int) bool { + return utf8.ValidString(value) && strings.TrimSpace(value) != "" && utf8.RuneCountInString(value) <= maxLength +} diff --git a/service/internal/adapter/controller/http/controller.go b/service/internal/adapter/controller/http/controller.go new file mode 100644 index 00000000..47178ced --- /dev/null +++ b/service/internal/adapter/controller/http/controller.go @@ -0,0 +1,102 @@ +package http + +import ( + "context" + "errors" + "log/slog" + "net" + "net/http" + "sync/atomic" + + "gitea.privcloud.tech/mtusi2026/internal/config" + "gitea.privcloud.tech/mtusi2026/internal/pkg/rest/router" +) + +type Server interface { + ShouldBeRunning() bool + Ready() bool + Start(ctx context.Context) error + Stop(ctx context.Context) error +} + +type server struct { + server http.Server + cfg *config.Server + ready atomic.Bool +} + +func New(cfg *config.Config, router router.Router) Server { + s := server{ + server: http.Server{ + Addr: cfg.Server.BindAddress(), + Handler: router, + ReadTimeout: cfg.Server.ReadTimeout, + WriteTimeout: cfg.Server.WriteTimeout, + IdleTimeout: cfg.Server.IdleTimeout, + }, + cfg: cfg.Server, + ready: atomic.Bool{}, + } + + if err := s.validate(); err != nil { + panic(err) + } + + return &s +} + +func (s *server) Start(_ context.Context) error { + if !s.ShouldBeRunning() { + slog.Info("server should not be running: skip starting") + return nil + } + listener, err := net.Listen("tcp", s.server.Addr) + if err != nil { + return err + } + + s.ready.Store(true) + go func() { + defer s.ready.Store(false) + if err := s.server.Serve(listener); err != nil && + !errors.Is(err, http.ErrServerClosed) { + slog.Error("rest server failed", "error", err) + } + }() + return nil +} + +func (s *server) Stop(ctx context.Context) error { + if !s.ShouldBeRunning() { + slog.Info("server should not be running: skip stopping") + return nil + } + + slog.Info("shutting down rest server...", "address", s.server.Addr) + s.ready.Store(false) + if err := s.server.Shutdown(ctx); err != nil { + slog.Error("rest server shutdown failed", "error", err) + _ = s.server.Close() + return err + } + slog.Info("rest server shutdown successfully") + return nil +} + +func (s *server) ShouldBeRunning() bool { + return s.cfg.Enabled +} + +func (s *server) Ready() bool { + return s.ready.Load() +} + +func (s *server) validate() error { + if s.cfg == nil { + slog.Error("failed to validate server: provided nil config") + return errors.New("failed to validate server: provided nil config") + } + + slog.Info("server successfully validated") + return nil +} diff --git a/service/internal/adapter/controller/http/handler.go b/service/internal/adapter/controller/http/handler.go new file mode 100644 index 00000000..275bee07 --- /dev/null +++ b/service/internal/adapter/controller/http/handler.go @@ -0,0 +1,5 @@ +package http + +type Handler interface { + RegisterRoutes() +} diff --git a/service/internal/adapter/controller/http/health.go b/service/internal/adapter/controller/http/health.go new file mode 100644 index 00000000..4d021a79 --- /dev/null +++ b/service/internal/adapter/controller/http/health.go @@ -0,0 +1,60 @@ +package http + +import ( + "context" + "net/http" + "time" + + "gitea.privcloud.tech/mtusi2026/internal/gen/contracts" + "gitea.privcloud.tech/mtusi2026/internal/pkg/apperr" + "gitea.privcloud.tech/mtusi2026/internal/pkg/rest/framework" + "gitea.privcloud.tech/mtusi2026/internal/pkg/rest/router" +) + +type databasePinger interface{ Ping(context.Context) error } + +type HealthHandler struct { + router router.Router + server Server + database databasePinger + storage []databasePinger +} + +func NewHealthHandler(r router.Router, server Server, database databasePinger, storage ...databasePinger) Handler { + return &HealthHandler{ + router: r, + server: server, + database: database, + storage: storage, + } +} + +func (h *HealthHandler) RegisterRoutes() { + h.router.Public(http.MethodGet, "/health/ready", h.Ready) + h.router.Public(http.MethodGet, "/health/live", h.Live) +} + +func (h *HealthHandler) Live(w http.ResponseWriter, _ *http.Request) { + framework.JSONResponse(w, http.StatusOK, contracts.Health{Status: contracts.Ok}) +} + +func (h *HealthHandler) Ready(w http.ResponseWriter, r *http.Request) { + if !h.server.Ready() { + framework.JSONError(r.Context(), w, apperr.New(apperr.CodeUnavailable, "server is not ready")) + return + } + ctx, cancel := context.WithTimeout(r.Context(), 200*time.Millisecond) + defer cancel() + if err := h.database.Ping(ctx); err != nil { + framework.JSONError(r.Context(), w, apperr.New(apperr.CodeUnavailable, "database unavailable")) + return + } + for _, dependency := range h.storage { + if err := dependency.Ping(ctx); err != nil { + framework.JSONError(r.Context(), w, apperr.New(apperr.CodeUnavailable, "storage unavailable")) + return + } + } + + framework.JSONResponse(w, http.StatusOK, contracts.Health{Status: contracts.Ok}) +} diff --git a/service/internal/adapter/controller/http/router.go b/service/internal/adapter/controller/http/router.go new file mode 100644 index 00000000..d02cfda6 --- /dev/null +++ b/service/internal/adapter/controller/http/router.go @@ -0,0 +1 @@ +package http diff --git a/service/internal/adapter/controller/http/user.go b/service/internal/adapter/controller/http/user.go new file mode 100644 index 00000000..7fdb640b --- /dev/null +++ b/service/internal/adapter/controller/http/user.go @@ -0,0 +1,113 @@ +package http + +import ( + "net/http" + "regexp" + "strings" + "unicode/utf8" + + "github.com/google/uuid" + + "gitea.privcloud.tech/mtusi2026/internal/dto" + "gitea.privcloud.tech/mtusi2026/internal/gen/contracts" + "gitea.privcloud.tech/mtusi2026/internal/pkg/apperr" + "gitea.privcloud.tech/mtusi2026/internal/pkg/auth/jwtprovider" + "gitea.privcloud.tech/mtusi2026/internal/pkg/rest/framework" + "gitea.privcloud.tech/mtusi2026/internal/pkg/rest/router" + "gitea.privcloud.tech/mtusi2026/internal/pkg/validator" + "gitea.privcloud.tech/mtusi2026/internal/service/user" +) + +type UserHandler struct { + router router.Router + provider jwtprovider.Provider + validator *validator.ValidatorImpl + + user user.API +} + +func NewUserHandler(r router.Router, provider jwtprovider.Provider, v *validator.ValidatorImpl, api user.API) Handler { + return &UserHandler{ + router: r, + provider: provider, + validator: v, + user: api, + } +} + +var usernamePattern = regexp.MustCompile(`^[a-z0-9_]{3,32}$`) + +func (h *UserHandler) RegisterRoutes() { + h.router.Public(http.MethodPost, "/api/v1/users", h.Create) + h.router.Private(http.MethodGet, "/api/v1/users/me", h.GetMe) +} + +func (h *UserHandler) Create(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + + var req contracts.CreateUser + if !framework.Decode(w, r, &req) || !framework.Validate(w, r, h.validator, &req) { + return + } + + toCreate := dto.CreateUser{ + Username: req.Username, + DisplayName: req.DisplayName, + } + + if req.Password != nil { + toCreate.Password = *req.Password + } + + if !usernamePattern.MatchString(req.Username) || !utf8.ValidString(req.DisplayName) || + strings.TrimSpace(req.DisplayName) == "" || utf8.RuneCountInString(req.DisplayName) > 100 { + framework.JSONError(ctx, w, apperr.New(apperr.CodeBadRequest, "invalid user fields")) + return + } + passwordLen := utf8.RuneCountInString(toCreate.Password) + if !utf8.ValidString(toCreate.Password) || strings.TrimSpace(toCreate.Password) == "" || passwordLen < 12 || passwordLen > 128 { + framework.JSONError(ctx, w, apperr.New(apperr.CodeBadRequest, "invalid password")) + return + } + + resp, err := h.user.Create(ctx, toCreate) + if err != nil { + framework.JSONError(ctx, w, err) + return + } + + framework.JSONResponse(w, http.StatusCreated, contracts.User{ + CreatedAt: resp.CreatedAt, + DisplayName: resp.DisplayName, + Id: resp.ID, + Username: resp.Username, + }) +} + +func (h *UserHandler) GetMe(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + + userData, err := framework.ParseToken(r, h.provider) + if err != nil { + framework.JSONError(ctx, w, err) + return + } + + if userData.UserID == uuid.Nil { + framework.JSONError(ctx, w, apperr.New(apperr.CodeBadRequest, "invalid user id")) + return + } + + resp, err := h.user.GetByID(ctx, userData.UserID) + if err != nil { + framework.JSONError(ctx, w, err) + return + } + + framework.JSONResponse(w, http.StatusOK, contracts.User{ + CreatedAt: resp.CreatedAt, + DisplayName: resp.DisplayName, + Id: resp.ID, + Username: resp.Username, + }) +} diff --git a/service/internal/adapter/repository/cache/outside/chat.go b/service/internal/adapter/repository/cache/outside/chat.go new file mode 100644 index 00000000..a5eb60c9 --- /dev/null +++ b/service/internal/adapter/repository/cache/outside/chat.go @@ -0,0 +1,78 @@ +package outside + +import ( + "context" + "encoding/json" + "fmt" + "time" + + "github.com/google/uuid" + + "gitea.privcloud.tech/mtusi2026/internal/dto" + "gitea.privcloud.tech/mtusi2026/internal/pkg/apperr" + "gitea.privcloud.tech/mtusi2026/internal/pkg/cache" +) + +const ( + // chatRedisKey equals "chat:{chat_id}" path + chatRedisKey = "chat:%s" +) + +type ChatCache struct { + client *cache.Client +} + +func NewChatCache(client *cache.Client) *ChatCache { + return &ChatCache{ + client: client, + } +} + +type chat struct { + ID uuid.UUID `json:"id"` + Title string `json:"title"` + MemberIDs []uuid.UUID `json:"member_ids"` + CreatedAt time.Time `json:"created_at"` +} + +func (r *ChatCache) Create(ctx context.Context, req dto.Chat) error { + data, err := json.Marshal(chat{ + ID: req.ID, + Title: req.Title, + MemberIDs: req.MemberIDs, + CreatedAt: req.CreatedAt, + }) + if err != nil { + return apperr.Wrap(err, apperr.CodeInternal, "failed to marshal chat cache") + } + + ctx, cancel := context.WithTimeout(ctx, cache.OperationTimeout) + defer cancel() + err = r.client.Set(ctx, fmt.Sprintf(chatRedisKey, req.ID), data, 0).Err() + if err != nil { + return apperr.WrapRedis(err, "failed to create chat cache") + } + + return nil +} + +func (r *ChatCache) Get(ctx context.Context, chatID uuid.UUID) (*dto.Chat, error) { + ctx, cancel := context.WithTimeout(ctx, cache.OperationTimeout) + defer cancel() + data, err := r.client.Get(ctx, fmt.Sprintf(chatRedisKey, chatID.String())).Bytes() + if err != nil { + return nil, apperr.WrapRedis(err, "failed to read chat cache") + } + + var item chat + if err = json.Unmarshal(data, &item); err != nil { + return nil, apperr.Wrap(err, apperr.CodeInternal, "failed to unmarshal chat cache") + } + + return &dto.Chat{ + ID: item.ID, + Title: item.Title, + MemberIDs: item.MemberIDs, + CreatedAt: item.CreatedAt, + }, nil +} diff --git a/service/internal/adapter/repository/storage/attachment_scan.go b/service/internal/adapter/repository/storage/attachment_scan.go new file mode 100644 index 00000000..15fa2134 --- /dev/null +++ b/service/internal/adapter/repository/storage/attachment_scan.go @@ -0,0 +1,34 @@ +package storage + +import ( + "context" + "gitea.privcloud.tech/mtusi2026/internal/pkg/apperr" + "github.com/google/uuid" + "github.com/jackc/pgx/v5/pgxpool" +) + +func (r *AttachmentsRepo) Page(ctx context.Context, after uuid.UUID) ([]*AttachmentRecord, error) { + rows, e := r.Pool.Query(ctx, "SELECT "+AttachmentColumns+" FROM attachments WHERE id>$1 ORDER BY id LIMIT 100", after) + if e != nil { + return nil, apperr.WrapPostgres(e, "scan failed") + } + defer rows.Close() + out := make([]*AttachmentRecord, 0, 100) + for rows.Next() { + a, e := ScanAttachment(rows) + if e != nil { + return nil, e + } + out = append(out, a) + } + return out, rows.Err() +} +func (r *AttachmentsRepo) Exists(ctx context.Context, c *pgxpool.Conn, key string) (bool, error) { + var found bool + e := c.QueryRow(ctx, "SELECT EXISTS(SELECT 1 FROM attachments WHERE object_key=$1)", key).Scan(&found) + return found, e +} +func (r *AttachmentsRepo) MarkMissing(ctx context.Context, c *pgxpool.Conn, id uuid.UUID, missing bool) (bool, error) { + tag, e := c.Exec(ctx, "UPDATE attachments SET integrity_missing=$2,updated_at=clock_timestamp() WHERE id=$1 AND integrity_missing<>$2", id, missing) + return tag.RowsAffected() == 1, e +} diff --git a/service/internal/adapter/repository/storage/attachments.go b/service/internal/adapter/repository/storage/attachments.go new file mode 100644 index 00000000..6d35c74c --- /dev/null +++ b/service/internal/adapter/repository/storage/attachments.go @@ -0,0 +1,71 @@ +package storage + +import ( + "context" + "time" + + "github.com/google/uuid" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" + + "gitea.privcloud.tech/mtusi2026/internal/gen/contracts" + "gitea.privcloud.tech/mtusi2026/internal/pkg/apperr" +) + +type AttachmentRecord struct { + Public contracts.Attachment + Key string + Missing bool +} +type AttachmentsRepo struct{ Pool *pgxpool.Pool } + +const AttachmentColumns = "id,chat_id,owner_id,filename,content_type,size_bytes,sha256,created_at,updated_at,object_key,integrity_missing" + +func ScanAttachment(row pgx.Row) (*AttachmentRecord, error) { + a := new(AttachmentRecord) + p := &a.Public + if err := row.Scan(&p.Id, &p.ChatId, &p.OwnerId, &p.Filename, &p.ContentType, &p.SizeBytes, &p.Sha256, &p.CreatedAt, &p.UpdatedAt, &a.Key, &a.Missing); err != nil { + return nil, apperr.WrapPostgres(err, "attachment unavailable") + } + p.Status = "ready" + p.Variants = []contracts.Variant{} + return a, nil +} +func (r *AttachmentsRepo) Get(ctx context.Context, id uuid.UUID) (*AttachmentRecord, error) { + return ScanAttachment(r.Pool.QueryRow(ctx, "SELECT "+AttachmentColumns+" FROM attachments WHERE id=$1", id)) +} +func (r *AttachmentsRepo) Save(ctx context.Context, conn *pgxpool.Conn, a *AttachmentRecord) error { + p := a.Public + _, e := conn.Exec(ctx, `INSERT INTO attachments(id,chat_id,owner_id,filename,content_type,size_bytes,sha256,created_at,updated_at,object_key) VALUES($1,$2,$3,$4,$5,$6,$7,$8,$9,$10)`, p.Id, p.ChatId, p.OwnerId, p.Filename, p.ContentType, p.SizeBytes, p.Sha256, p.CreatedAt, p.UpdatedAt, a.Key) + if e != nil { + return apperr.WrapPostgres(e, "failed to save attachment") + } + return nil +} + +// Session locks span S3 work and the database commit. A lost connection releases +// the lock; the mandatory grace period also exceeds the maximum upload lifetime. +func (r *AttachmentsRepo) Lock(ctx context.Context, key string) (*pgxpool.Conn, func(), error) { + c, e := r.Pool.Acquire(ctx) + if e != nil { + return nil, nil, apperr.WrapPostgres(e, "database unavailable") + } + if _, e = c.Exec(ctx, "SELECT pg_advisory_lock(hashtextextended($1, 42))", key); e != nil { + // The server may have acquired the session lock before cancellation was + // observed by the client. Never return that uncertain session to the pool. + clean, cancel := context.WithTimeout(context.Background(), 2*time.Second) + _ = c.Conn().Close(clean) + cancel() + c.Release() + return nil, nil, apperr.WrapPostgres(e, "lock unavailable") + } + return c, func() { + // Close rather than return a possibly locked session when unlock fails. + clean, cancel := context.WithTimeout(context.Background(), 2*time.Second) + defer cancel() + if _, e := c.Exec(clean, "SELECT pg_advisory_unlock(hashtextextended($1, 42))", key); e != nil { + _ = c.Conn().Close(clean) + } + c.Release() + }, nil +} diff --git a/service/internal/adapter/repository/storage/chats.go b/service/internal/adapter/repository/storage/chats.go new file mode 100644 index 00000000..15a72982 --- /dev/null +++ b/service/internal/adapter/repository/storage/chats.go @@ -0,0 +1,91 @@ +package storage + +import ( + "context" + "errors" + + "github.com/google/uuid" + "github.com/jackc/pgerrcode" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgconn" + "github.com/jackc/pgx/v5/pgxpool" + + "gitea.privcloud.tech/mtusi2026/internal/dto" + "gitea.privcloud.tech/mtusi2026/internal/pkg/apperr" +) + +type ChatsRepo struct { + pool *pgxpool.Pool +} + +func NewChatsRepo(pool *pgxpool.Pool) *ChatsRepo { + return &ChatsRepo{pool: pool} +} + +func (r *ChatsRepo) CreateTx(ctx context.Context, tx pgx.Tx, req dto.Chat) error { + const insertChat = `INSERT INTO chats (id, title, created_at) VALUES ($1, $2, $3)` + if _, err := tx.Exec(ctx, insertChat, req.ID, req.Title, req.CreatedAt); err != nil { + return apperr.WrapPostgres(err, "failed to create chat") + } + const insertMembers = `INSERT INTO chat_members (chat_id, user_id) +SELECT $1, user_id FROM unnest($2::uuid[]) AS members(user_id)` + if _, err := tx.Exec(ctx, insertMembers, req.ID, req.MemberIDs); err != nil { + if pgErr, ok := errors.AsType[*pgconn.PgError](err); ok && + pgErr.Code == pgerrcode.ForeignKeyViolation && pgErr.ConstraintName == "chat_members_user_fk" { + return apperr.Wrap(err, apperr.CodeBadRequest, "unknown chat member") + } + return apperr.WrapPostgres(err, "failed to create chat members") + } + return nil +} + +func (r *ChatsRepo) GetForMember(ctx context.Context, req dto.ChatMember) (*dto.Chat, error) { + const query = `SELECT c.id, c.title, c.created_at, + ARRAY(SELECT m.user_id FROM chat_members m WHERE m.chat_id = c.id ORDER BY m.user_id) +FROM chats c +WHERE c.id = $1 + AND EXISTS (SELECT 1 FROM chat_members access WHERE access.chat_id = c.id AND access.user_id = $2)` + return scanChat(r.pool.QueryRow(ctx, query, req.ChatID, req.UserID)) +} + +func (r *ChatsRepo) ListForMember(ctx context.Context, userID uuid.UUID, page dto.PageQuery) ([]dto.Chat, error) { + query := `WITH page AS MATERIALIZED ( + SELECT c.id, c.title, c.created_at + FROM chats c JOIN chat_members access ON access.chat_id = c.id + WHERE access.user_id = $1` + args := []any{userID, page.Limit} + if page.After != nil { + query += ` AND (c.created_at, c.id) > ($3, $4)` + args = append(args, page.After.CreatedAt, page.After.ID) + } + query += ` ORDER BY c.created_at, c.id LIMIT $2 +) +SELECT p.id, p.title, p.created_at, + ARRAY(SELECT m.user_id FROM chat_members m WHERE m.chat_id = p.id ORDER BY m.user_id) +FROM page p ORDER BY p.created_at, p.id` + rows, err := r.pool.Query(ctx, query, args...) + if err != nil { + return nil, apperr.WrapPostgres(err, "failed to list chats") + } + defer rows.Close() + chats := make([]dto.Chat, 0) + for rows.Next() { + chat, err := scanChat(rows) + if err != nil { + return nil, err + } + chats = append(chats, *chat) + } + if err = rows.Err(); err != nil { + return nil, apperr.WrapPostgres(err, "failed to list chats") + } + return chats, nil +} + +func scanChat(row pgx.Row) (*dto.Chat, error) { + var chat dto.Chat + if err := row.Scan(&chat.ID, &chat.Title, &chat.CreatedAt, &chat.MemberIDs); err != nil { + return nil, apperr.WrapPostgres(err, "failed to get chat") + } + return &chat, nil +} diff --git a/service/internal/adapter/repository/storage/idempotency_test.go b/service/internal/adapter/repository/storage/idempotency_test.go new file mode 100644 index 00000000..2d151e25 --- /dev/null +++ b/service/internal/adapter/repository/storage/idempotency_test.go @@ -0,0 +1,303 @@ +package storage + +import ( + "context" + "crypto/sha256" + "errors" + "fmt" + "os" + "strings" + "testing" + "time" + + "gitea.privcloud.tech/mtusi2026/internal/config" + "gitea.privcloud.tech/mtusi2026/internal/dto" + "gitea.privcloud.tech/mtusi2026/internal/pkg/apperr" + "gitea.privcloud.tech/mtusi2026/internal/pkg/migrator" + "gitea.privcloud.tech/mtusi2026/internal/pkg/postgres" + "gitea.privcloud.tech/mtusi2026/internal/pkg/postgres/txmanager" + chatsvc "gitea.privcloud.tech/mtusi2026/internal/service/chat" + "github.com/google/uuid" +) + +// The explicit runner supplies a newly created database; never alter the API database. +func TestIdempotencyTransactionAndExpiry(t *testing.T) { + database := os.Getenv("TEST_DATABASE_NAME") + if database == "" { + t.Skip("run make test-storage for an isolated PostgreSQL database") + } + if database == os.Getenv("POSTGRES_DATABASE") { + t.Fatal("test database must differ from application database") + } + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + cfg, err := config.New() + if err != nil { + t.Fatal(err) + } + cfg.Postgres.Database = database + cfg.Postgres.RunMigrations = true + if err = migrator.New(cfg).Start(ctx); err != nil { + t.Fatal(err) + } + pool := postgres.New(cfg.Postgres) + defer pool.Close() + repo := NewMessagesRepo(pool) + user, chat := uuid.New(), uuid.New() + if _, err = pool.Exec(ctx, `INSERT INTO users(id,name,username,password_hash) VALUES($1,'Synthetic','atomic_test','unused')`, user); err != nil { + t.Fatal(err) + } + if _, err = pool.Exec(ctx, `INSERT INTO chats(id,title) VALUES($1,'Atomic test')`, chat); err != nil { + t.Fatal(err) + } + if _, err = pool.Exec(ctx, `INSERT INTO chat_members(chat_id,user_id) VALUES($1,$2)`, chat, user); err != nil { + t.Fatal(err) + } + req := dto.CreateMessage{ChatID: chat, SenderID: user, Text: "atomic", IdempotencyKey: "atomic-key"} + tx, err := pool.Begin(ctx) + if err != nil { + t.Fatal(err) + } + if _, err = repo.CreateIdempotentTx(ctx, tx, req); err != nil { + t.Fatal(err) + } + if err = tx.Rollback(ctx); err != nil { + t.Fatal(err) + } + var messages, keys int + if err = pool.QueryRow(ctx, `SELECT (SELECT count(*) FROM messages),(SELECT count(*) FROM message_idempotency)`).Scan(&messages, &keys); err != nil { + t.Fatal(err) + } + if messages != 0 || keys != 0 { + t.Fatalf("rollback left %d messages, %d keys", messages, keys) + } + tx, err = pool.Begin(ctx) + if err != nil { + t.Fatal(err) + } + first, err := repo.CreateIdempotentTx(ctx, tx, req) + if err != nil { + t.Fatal(err) + } + if err = tx.Commit(ctx); err != nil { + t.Fatal(err) + } + // Expire the test record only: no shortened production TTL and no wall-clock wait. + if _, err = pool.Exec(ctx, `UPDATE message_idempotency SET expires_at=clock_timestamp()-interval '1 second' WHERE user_id=$1`, user); err != nil { + t.Fatal(err) + } + req.Text = "new after expiry" + tx, err = pool.Begin(ctx) + if err != nil { + t.Fatal(err) + } + second, err := repo.CreateIdempotentTx(ctx, tx, req) + if err != nil { + t.Fatal(err) + } + if err = tx.Commit(ctx); err != nil { + t.Fatal(err) + } + if first.ID == second.ID { + t.Fatal("expired key replayed old message") + } + if err = pool.QueryRow(ctx, `SELECT (SELECT count(*) FROM messages),(SELECT count(*) FROM message_idempotency)`).Scan(&messages, &keys); err != nil { + t.Fatal(err) + } + if messages != 2 || keys != 1 { + t.Fatalf("expiry left %d messages, %d keys; want 2,1", messages, keys) + } + if _, err = pool.Exec(ctx, `CREATE FUNCTION reject_test_key() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN IF NEW.key='failure-key' THEN RAISE EXCEPTION 'synthetic failure'; END IF; RETURN NEW; END $$; CREATE TRIGGER reject_test_key BEFORE INSERT ON message_idempotency FOR EACH ROW EXECUTE FUNCTION reject_test_key()`); err != nil { + t.Fatal(err) + } + req.IdempotencyKey = "failure-key" + tx, err = pool.Begin(ctx) + if err != nil { + t.Fatal(err) + } + if _, err = repo.CreateIdempotentTx(ctx, tx, req); err == nil { + t.Fatal("expected injected failure") + } + if err = tx.Rollback(ctx); err != nil { + t.Fatal(err) + } + if err = pool.QueryRow(ctx, `SELECT (SELECT count(*) FROM messages),(SELECT count(*) FROM message_idempotency)`).Scan(&messages, &keys); err != nil { + t.Fatal(err) + } + if messages != 2 || keys != 1 { + t.Fatal("failed result persisted a side effect") + } + if _, err = pool.Exec(ctx, `DROP TRIGGER reject_test_key ON message_idempotency; DROP FUNCTION reject_test_key()`); err != nil { + t.Fatal(err) + } + tx, err = pool.Begin(ctx) + if err != nil { + t.Fatal(err) + } + if _, err = repo.CreateIdempotentTx(ctx, tx, req); err != nil { + t.Fatal("transient failure was cached", err) + } + if err = tx.Commit(ctx); err != nil { + t.Fatal(err) + } + + t.Run("service owns message content rules", func(t *testing.T) { + svc := chatsvc.New(txmanager.New(pool), NewChatsRepo(pool), repo, uncachedChats{}) + own, foreign, missing := uuid.New(), uuid.New(), uuid.New() + otherChat := uuid.New() + if _, err := pool.Exec(ctx, `INSERT INTO chats(id,title) VALUES($1,'Other chat');`, otherChat); err != nil { + t.Fatal(err) + } + if _, err := pool.Exec(ctx, `INSERT INTO chat_members(chat_id,user_id) VALUES($1,$2)`, otherChat, user); err != nil { + t.Fatal(err) + } + for _, fixture := range []struct { + id, chat uuid.UUID + missing bool + }{{own, chat, false}, {foreign, otherChat, false}, {missing, chat, true}} { + if _, err := pool.Exec(ctx, `INSERT INTO attachments(id,chat_id,owner_id,filename,object_key,content_type,size_bytes,sha256,integrity_missing,created_at,updated_at) VALUES($1,$2,$3,'test.png',$4,'image/png',1,$5,$6,now(),now())`, fixture.id, fixture.chat, user, "originals/"+fixture.id.String(), strings.Repeat("a", 64), fixture.missing); err != nil { + t.Fatal(err) + } + } + counts := func() string { + t.Helper() + var messages, links, keys int + if err := pool.QueryRow(ctx, `SELECT (SELECT count(*) FROM messages),(SELECT count(*) FROM message_attachments),(SELECT count(*) FROM message_idempotency)`).Scan(&messages, &links, &keys); err != nil { + t.Fatal(err) + } + return fmt.Sprintf("%d/%d/%d", messages, links, keys) + } + for _, tc := range []struct { + name, text string + ids []uuid.UUID + code apperr.Code + }{ + {"empty", "", nil, apperr.CodeBadRequest}, + {"whitespace", " \t\n\u2003", nil, apperr.CodeBadRequest}, + {"invalid UTF8", string([]byte{0xff}), nil, apperr.CodeBadRequest}, + {"nul", "\x00", nil, apperr.CodeBadRequest}, + {"too long", strings.Repeat("я", 4001), nil, apperr.CodeBadRequest}, + {"duplicate", "", []uuid.UUID{own, own}, apperr.CodeBadRequest}, + {"too many", "text", make([]uuid.UUID, 11), apperr.CodeBadRequest}, + {"nil id", "", []uuid.UUID{uuid.Nil}, apperr.CodeBadRequest}, + {"unknown", "", []uuid.UUID{uuid.New()}, apperr.CodeNotFound}, + {"foreign chat", "", []uuid.UUID{foreign}, apperr.CodeNotFound}, + {"missing object", "", []uuid.UUID{missing}, apperr.CodeUnavailable}, + } { + t.Run(tc.name, func(t *testing.T) { + before := counts() + _, err := svc.CreateMessage(ctx, dto.CreateMessage{ChatID: chat, SenderID: user, IdempotencyKey: uuid.NewString(), Text: tc.text, AttachmentIDs: tc.ids}) + if err == nil || apperr.CodeFromError(err) != tc.code { + t.Fatalf("got %v, want code %v", err, tc.code) + } + if after := counts(); after != before { + t.Fatalf("rejected message changed counts: %s -> %s", before, after) + } + }) + } + for _, text := range []string{"", " \t\u2003"} { + request := dto.CreateMessage{ChatID: chat, SenderID: user, IdempotencyKey: uuid.NewString(), Text: text, AttachmentIDs: []uuid.UUID{own}} + first, err := svc.CreateMessage(ctx, request) + if err != nil { + t.Fatal(err) + } + replay, err := svc.CreateMessage(ctx, request) + if err != nil || replay.ID != first.ID || len(replay.AttachmentIDs) != 1 || replay.AttachmentIDs[0] != own { + t.Fatalf("attachment replay failed: %v, %v", replay, err) + } + page, err := svc.ListForChat(ctx, dto.MessageFilters{UserID: user, ChatID: chat}) + if err != nil { + t.Fatal(err) + } + found := false + for _, msg := range page.Items { + if msg.ID == first.ID { + found = len(msg.AttachmentIDs) == 1 && msg.AttachmentIDs[0] == own + } + } + if !found { + t.Fatal("history lost attachment") + } + request.AttachmentIDs = []uuid.UUID{foreign} + if _, err = svc.CreateMessage(ctx, request); apperr.CodeFromError(err) != apperr.CodeNotFound { + t.Fatalf("foreign replay: %v", err) + } + } + // Preserve legacy hashes written before the attachment migration. + legacy := dto.CreateMessage{ChatID: chat, SenderID: user, IdempotencyKey: uuid.NewString(), Text: "legacy"} + first, err := svc.CreateMessage(ctx, legacy) + if err != nil { + t.Fatal(err) + } + hash := sha256.Sum256([]byte(legacy.Text)) + if _, err = pool.Exec(ctx, `UPDATE message_idempotency SET request_hash=$1,hash_version=1 WHERE key=$2`, hash[:], legacy.IdempotencyKey); err != nil { + t.Fatal(err) + } + replay, err := svc.CreateMessage(ctx, legacy) + if err != nil || replay.ID != first.ID { + t.Fatalf("legacy replay failed: %v", err) + } + legacy.AttachmentIDs = []uuid.UUID{own} + if _, err = svc.CreateMessage(ctx, legacy); apperr.CodeFromError(err) != apperr.CodeConflict { + t.Fatalf("legacy hash ignored new attachment: %v", err) + } + textOnly := dto.CreateMessage{ChatID: chat, SenderID: user, IdempotencyKey: uuid.NewString(), Text: "text only"} + first, err = svc.CreateMessage(ctx, textOnly) + if err != nil { + t.Fatal(err) + } + textOnly.AttachmentIDs = []uuid.UUID{} + replay, err = svc.CreateMessage(ctx, textOnly) + if err != nil || replay.ID != first.ID { + t.Fatalf("nil and empty attachment lists differ: %v", err) + } + // Force link failure using the relational FK and prove all preceding writes roll back. + before := counts() + tx, err := pool.Begin(ctx) + if err != nil { + t.Fatal(err) + } + _, err = repo.CreateIdempotentTx(ctx, tx, dto.CreateMessage{ChatID: chat, SenderID: user, IdempotencyKey: uuid.NewString(), Text: "link rollback", AttachmentIDs: []uuid.UUID{own, uuid.New()}}) + if err == nil { + t.Fatal("expected missing attachment FK failure") + } + _ = tx.Rollback(ctx) + if after := counts(); after != before { + t.Fatalf("failed link changed counts: %s -> %s", before, after) + } + var triggers int + if err = pool.QueryRow(ctx, `SELECT count(*) FROM pg_trigger WHERE tgname='message_content'`).Scan(&triggers); err != nil || triggers != 0 { + t.Fatalf("message trigger remains: %d, %v", triggers, err) + } + }) + + t.Run("object lock cancellation does not leak a pooled lock", func(t *testing.T) { + attachments := &AttachmentsRepo{Pool: pool} + key := "originals/" + uuid.NewString() + _, unlock, err := attachments.Lock(ctx, key) + if err != nil { + t.Fatal(err) + } + waiting, cancel := context.WithTimeout(ctx, 50*time.Millisecond) + _, _, err = attachments.Lock(waiting, key) + cancel() + unlock() + if err == nil { + t.Fatal("contending lock ignored deadline") + } + _, release, err := attachments.Lock(ctx, key) + if err != nil { + t.Fatalf("lock leaked after cancellation: %v", err) + } + release() + }) + + t.Log("rollback removed both side effect and key; expired key created one new message") +} + +type uncachedChats struct{} + +func (uncachedChats) Create(context.Context, dto.Chat) error { return nil } +func (uncachedChats) Get(context.Context, uuid.UUID) (*dto.Chat, error) { + return nil, errors.New("cache disabled for storage test") +} diff --git a/service/internal/adapter/repository/storage/messages.go b/service/internal/adapter/repository/storage/messages.go new file mode 100644 index 00000000..61c29ba8 --- /dev/null +++ b/service/internal/adapter/repository/storage/messages.go @@ -0,0 +1,153 @@ +package storage + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/json" + "errors" + "time" + + "github.com/google/uuid" + "github.com/jackc/pgerrcode" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgconn" + "github.com/jackc/pgx/v5/pgxpool" + + "gitea.privcloud.tech/mtusi2026/internal/dto" + "gitea.privcloud.tech/mtusi2026/internal/pkg/apperr" +) + +type MessagesRepo struct { + pool *pgxpool.Pool +} + +func NewMessagesRepo(pool *pgxpool.Pool) *MessagesRepo { + return &MessagesRepo{pool: pool} +} + +// CreateIdempotentTx serializes only requests sharing an idempotency scope. +// The transaction commits both the message and its replay record, or neither. +func (r *MessagesRepo) CreateIdempotentTx(ctx context.Context, tx pgx.Tx, req dto.CreateMessage) (*dto.Message, error) { + scope := req.SenderID.String() + ":POST:" + req.ChatID.String() + ":" + req.IdempotencyKey + if _, err := tx.Exec(ctx, `SELECT pg_advisory_xact_lock(hashtextextended($1, 0))`, scope); err != nil { + return nil, apperr.WrapPostgres(err, "failed to lock message key") + } + hash := messageHash(req) + var hashVersion int + var storedHash []byte + var messageID uuid.UUID + err := tx.QueryRow(ctx, `SELECT request_hash, message_id, hash_version FROM message_idempotency + WHERE user_id=$1 AND method='POST' AND chat_id=$2 AND key=$3 AND expires_at>clock_timestamp()`, + req.SenderID, req.ChatID, req.IdempotencyKey).Scan(&storedHash, &messageID, &hashVersion) + if err == nil { + if hashVersion == 1 && len(req.AttachmentIDs) == 0 { + hash = sha256.Sum256([]byte(req.Text)) + } else if hashVersion != 2 { + return nil, apperr.New(apperr.CodeConflict, "idempotency key used with different content") + } + if !bytes.Equal(hash[:], storedHash) { + return nil, apperr.New(apperr.CodeConflict, "idempotency key used with different content") + } + return scanMessage(tx.QueryRow(ctx, `SELECT `+messageColumns+` FROM messages WHERE id=$1`, messageID)) + } + if !errors.Is(err, pgx.ErrNoRows) { + return nil, apperr.WrapPostgres(err, "failed to read message key") + } + message := dto.Message{ID: uuid.New(), ChatID: req.ChatID, SenderID: req.SenderID, Text: req.Text, AttachmentIDs: append([]uuid.UUID{}, req.AttachmentIDs...), CreatedAt: time.Now().UTC().Truncate(time.Microsecond)} + if _, err = tx.Exec(ctx, `INSERT INTO messages(id,chat_id,sender_id,text,created_at) VALUES($1,$2,$3,$4,$5)`, + message.ID, message.ChatID, message.SenderID, message.Text, message.CreatedAt); err != nil { + if pgErr, ok := errors.AsType[*pgconn.PgError](err); ok && pgErr.Code == pgerrcode.ForeignKeyViolation { + return nil, apperr.Wrap(err, apperr.CodeNotFound, "chat not found") + } + return nil, apperr.WrapPostgres(err, "failed to create message") + } + for position, attachmentID := range req.AttachmentIDs { + if _, err = tx.Exec(ctx, `INSERT INTO message_attachments(message_id,chat_id,attachment_id,position) VALUES($1,$2,$3,$4)`, message.ID, message.ChatID, attachmentID, position); err != nil { + return nil, apperr.WrapPostgres(err, "failed to link attachment") + } + } + + if _, err = tx.Exec(ctx, `INSERT INTO message_idempotency(user_id,method,chat_id,key,request_hash,message_id,expires_at,hash_version) + VALUES($1,'POST',$2,$3,$4,$5,clock_timestamp()+interval '24 hours',2) + ON CONFLICT(user_id,method,chat_id,key) DO UPDATE SET request_hash=EXCLUDED.request_hash, + message_id=EXCLUDED.message_id,expires_at=EXCLUDED.expires_at,hash_version=EXCLUDED.hash_version`, req.SenderID, req.ChatID, req.IdempotencyKey, hash[:], message.ID); err != nil { + return nil, apperr.WrapPostgres(err, "failed to save message key") + } + return &message, nil +} + +func (r *MessagesRepo) GetByIDForChat(ctx context.Context, chatID, messageID uuid.UUID) (*dto.Message, error) { + const query = `SELECT ` + messageColumns + ` FROM messages WHERE chat_id = $1 AND id = $2` + return scanMessage(r.pool.QueryRow(ctx, query, chatID, messageID)) +} + +func (r *MessagesRepo) ListForChat(ctx context.Context, chatID uuid.UUID, page dto.PageQuery) ([]dto.Message, error) { + query := `SELECT ` + messageColumns + ` FROM messages WHERE chat_id = $1` + args := []any{chatID, page.Limit} + if page.After != nil { + query += ` AND (created_at, id) > ($3, $4)` + args = append(args, page.After.CreatedAt, page.After.ID) + } + query += ` ORDER BY created_at, id LIMIT $2` + rows, err := r.pool.Query(ctx, query, args...) + if err != nil { + return nil, apperr.WrapPostgres(err, "failed to list messages") + } + defer rows.Close() + messages := make([]dto.Message, 0) + for rows.Next() { + message, err := scanMessage(rows) + if err != nil { + return nil, err + } + messages = append(messages, *message) + } + if err = rows.Err(); err != nil { + return nil, apperr.WrapPostgres(err, "failed to list messages") + } + return messages, nil +} + +func scanMessage(row pgx.Row) (*dto.Message, error) { + var message dto.Message + if err := row.Scan(&message.ID, &message.ChatID, &message.SenderID, &message.Text, &message.CreatedAt, &message.AttachmentIDs); err != nil { + return nil, apperr.WrapPostgres(err, "failed to get message") + } + return &message, nil +} + +const messageColumns = `id, chat_id, sender_id, text, created_at, + ARRAY(SELECT attachment_id FROM message_attachments WHERE message_id=messages.id ORDER BY position)` + +func messageHash(req dto.CreateMessage) [32]byte { + // Field order and a non-nil list define the canonical v2 representation. + canonical := struct { + Text string `json:"text"` + AttachmentIDs []uuid.UUID `json:"attachment_ids"` + }{req.Text, append([]uuid.UUID{}, req.AttachmentIDs...)} + encoded, _ := json.Marshal(canonical) // strings and UUIDs have no encoding failures. + return sha256.Sum256(encoded) +} + +func (r *MessagesRepo) AttachmentsForMessageTx(ctx context.Context, tx pgx.Tx, ids []uuid.UUID) ([]dto.MessageAttachment, error) { + // Prevent a concurrent integrity update from invalidating the checked rows + // until the message and its links have committed together. + rows, err := tx.Query(ctx, `SELECT id, chat_id, integrity_missing FROM attachments WHERE id=ANY($1) ORDER BY id FOR SHARE`, ids) + if err != nil { + return nil, apperr.WrapPostgres(err, "failed to read attachments") + } + defer rows.Close() + result := make([]dto.MessageAttachment, 0, len(ids)) + for rows.Next() { + var a dto.MessageAttachment + if err := rows.Scan(&a.ID, &a.ChatID, &a.Missing); err != nil { + return nil, apperr.WrapPostgres(err, "failed to read attachment") + } + result = append(result, a) + } + if err := rows.Err(); err != nil { + return nil, apperr.WrapPostgres(err, "failed to read attachments") + } + return result, nil +} diff --git a/service/internal/adapter/repository/storage/refresh_tokens.go b/service/internal/adapter/repository/storage/refresh_tokens.go new file mode 100644 index 00000000..9b3ca8c4 --- /dev/null +++ b/service/internal/adapter/repository/storage/refresh_tokens.go @@ -0,0 +1,193 @@ +package storage + +import ( + "context" + "time" + + "github.com/google/uuid" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" + + "gitea.privcloud.tech/mtusi2026/internal/dto" + "gitea.privcloud.tech/mtusi2026/internal/pkg/apperr" + "gitea.privcloud.tech/mtusi2026/internal/pkg/postgres" +) + +type RefreshTokensRepo struct { + pool *pgxpool.Pool +} + +func NewRefreshTokensRepo(pool *pgxpool.Pool) *RefreshTokensRepo { + return &RefreshTokensRepo{ + pool: pool, + } +} + +func (r *RefreshTokensRepo) Create(ctx context.Context, req dto.StoredRefreshToken) error { + return r.create(ctx, r.pool, req) +} + +func (r *RefreshTokensRepo) CreateTx(ctx context.Context, tx pgx.Tx, req dto.StoredRefreshToken) error { + return r.create(ctx, tx, req) +} + +func (r *RefreshTokensRepo) GetByID(ctx context.Context, id uuid.UUID) (*dto.StoredRefreshToken, error) { + return r.getByID(ctx, r.pool, id, false) +} + +func (r *RefreshTokensRepo) GetByIDTx(ctx context.Context, tx pgx.Tx, id uuid.UUID) (*dto.StoredRefreshToken, error) { + return r.getByID(ctx, tx, id, false) +} + +func (r *RefreshTokensRepo) GetByIDForUpdate(ctx context.Context, id uuid.UUID) (*dto.StoredRefreshToken, error) { + return r.getByID(ctx, r.pool, id, true) +} + +func (r *RefreshTokensRepo) GetByIDForUpdateTx(ctx context.Context, tx pgx.Tx, id uuid.UUID) (*dto.StoredRefreshToken, error) { + return r.getByID(ctx, tx, id, true) +} + +func (r *RefreshTokensRepo) Revoke(ctx context.Context, req dto.RevokeRefreshToken) error { + return r.revoke(ctx, r.pool, req) +} + +func (r *RefreshTokensRepo) RevokeTx(ctx context.Context, tx pgx.Tx, req dto.RevokeRefreshToken) error { + return r.revoke(ctx, tx, req) +} + +func (r *RefreshTokensRepo) RevokeAllByUserID(ctx context.Context, req dto.RevokeRefreshTokensByUserID) error { + return r.revokeAllByUserID(ctx, r.pool, req) +} + +func (r *RefreshTokensRepo) RevokeAllByUserIDTx(ctx context.Context, tx pgx.Tx, req dto.RevokeRefreshTokensByUserID) error { + return r.revokeAllByUserID(ctx, tx, req) +} + +func (r *RefreshTokensRepo) DeleteExpired(ctx context.Context) error { + const query = ` +DELETE FROM refresh_tokens +WHERE expires_at < $1` + + _, err := r.pool.Exec(ctx, query, time.Now()) + if err != nil { + return apperr.WrapPostgres(err, "failed to delete expired refresh tokens") + } + + return nil +} + +func (r *RefreshTokensRepo) create(ctx context.Context, conn postgres.Conn, req dto.StoredRefreshToken) error { + const query = `INSERT INTO refresh_tokens (id, + user_id, + token_hash, + user_agent, + ip_address, + role, + expires_at, + created_at) +VALUES ($1, $2, $3, $4, $5, $6, $7, $8)` + + ct, err := conn.Exec( + ctx, + query, + req.ID, + req.UserID, + req.TokenHash, + req.UserAgent, + req.IPAddress, + req.Role, + req.ExpiresAt, + req.CreatedAt, + ) + if err != nil { + return apperr.WrapPostgres(err, "failed to create refresh token") + } + + if ct.RowsAffected() == 0 { + return apperr.New(apperr.CodeAlreadyExists, "refresh token already exists") + } + + return nil +} + +func (r *RefreshTokensRepo) getByID(ctx context.Context, conn postgres.Conn, id uuid.UUID, lock bool) (*dto.StoredRefreshToken, error) { + const query = `SELECT id, + user_id, + token_hash, + user_agent, + ip_address, + role, + expires_at, + revoked_at, + created_at +FROM refresh_tokens +WHERE id = $1` + const queryLock = `SELECT id, + user_id, + token_hash, + user_agent, + ip_address, + role, + expires_at, + revoked_at, + created_at +FROM refresh_tokens +WHERE id = $1 FOR UPDATE` + + use := query + if lock { + use = queryLock + } + + var token dto.StoredRefreshToken + err := conn.QueryRow(ctx, use, id).Scan( + &token.ID, + &token.UserID, + &token.TokenHash, + &token.UserAgent, + &token.IPAddress, + &token.Role, + &token.ExpiresAt, + &token.RevokedAt, + &token.CreatedAt, + ) + if err != nil { + return nil, apperr.WrapPostgres(err, "failed to get refresh by id") + } + + return &token, nil +} + +func (r *RefreshTokensRepo) revoke(ctx context.Context, conn postgres.Conn, req dto.RevokeRefreshToken) error { + const query = ` +UPDATE refresh_tokens +SET revoked_at = $2 +WHERE id = $1 + AND revoked_at IS NULL` + + ct, err := conn.Exec(ctx, query, req.ID, req.RevokedAt) + if err != nil { + return apperr.WrapPostgres(err, "failed to revoke refresh token") + } + + if ct.RowsAffected() == 0 { + return apperr.New(apperr.CodeNotFound, "refresh token not found") + } + + return nil +} + +func (r *RefreshTokensRepo) revokeAllByUserID(ctx context.Context, conn postgres.Conn, req dto.RevokeRefreshTokensByUserID) error { + const query = ` +UPDATE refresh_tokens +SET revoked_at = $2 +WHERE user_id = $1 + AND revoked_at IS NULL` + + _, err := conn.Exec(ctx, query, req.UserID, req.RevokedAt) + if err != nil { + return apperr.WrapPostgres(err, "failed to revoke refresh tokens") + } + + return nil +} diff --git a/service/internal/adapter/repository/storage/users.go b/service/internal/adapter/repository/storage/users.go new file mode 100644 index 00000000..11cdedb5 --- /dev/null +++ b/service/internal/adapter/repository/storage/users.go @@ -0,0 +1,103 @@ +package storage + +import ( + "context" + "errors" + + "github.com/google/uuid" + "github.com/jackc/pgerrcode" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgconn" + "github.com/jackc/pgx/v5/pgxpool" + + "gitea.privcloud.tech/mtusi2026/internal/dto" + "gitea.privcloud.tech/mtusi2026/internal/pkg/apperr" + "gitea.privcloud.tech/mtusi2026/internal/pkg/postgres" +) + +type UsersRepo struct { + pool *pgxpool.Pool +} + +func NewUsersRepo(pool *pgxpool.Pool) *UsersRepo { + return &UsersRepo{pool: pool} +} + +func (r *UsersRepo) Create(ctx context.Context, req dto.CreateStoredUser) error { + return r.create(ctx, r.pool, req) +} + +func (r *UsersRepo) CreateTx(ctx context.Context, tx pgx.Tx, req dto.CreateStoredUser) error { + return r.create(ctx, tx, req) +} + +func (r *UsersRepo) GetByID(ctx context.Context, id uuid.UUID) (*dto.User, error) { + return r.getByID(ctx, r.pool, id) +} + +func (r *UsersRepo) GetByIDTx(ctx context.Context, tx pgx.Tx, id uuid.UUID) (*dto.User, error) { + return r.getByID(ctx, tx, id) +} + +func (r *UsersRepo) GetByUsername(ctx context.Context, username string) (*dto.User, error) { + const query = `SELECT id, username, name, created_at FROM users WHERE username = $1` + return scanUser(r.pool.QueryRow(ctx, query, username)) +} + +func (r *UsersRepo) GetCredentialsByUsername(ctx context.Context, username string) (*dto.UserCredentials, error) { + const query = `SELECT id, username, name, created_at, password_hash FROM users WHERE username = $1` + + var credentials dto.UserCredentials + err := r.pool.QueryRow(ctx, query, username).Scan( + &credentials.User.ID, + &credentials.User.Username, + &credentials.User.DisplayName, + &credentials.User.CreatedAt, + &credentials.PasswordHash, + ) + if err != nil { + return nil, apperr.WrapPostgres(err, "failed to get user credentials") + } + return &credentials, nil +} + +func (r *UsersRepo) Delete(ctx context.Context, id uuid.UUID) error { + const query = `DELETE FROM users WHERE id = $1` + + ct, err := r.pool.Exec(ctx, query, id) + if err != nil { + return apperr.WrapPostgres(err, "failed to delete user") + } + if ct.RowsAffected() == 0 { + return apperr.New(apperr.CodeNotFound, "user not found") + } + return nil +} + +func (r *UsersRepo) create(ctx context.Context, conn postgres.Conn, req dto.CreateStoredUser) error { + const query = `INSERT INTO users (id, username, name, password_hash, created_at, updated_at) +VALUES ($1, $2, $3, $4, $5, $6)` + + _, err := conn.Exec(ctx, query, req.ID, req.Username, req.DisplayName, req.PasswordHash, req.CreatedAt, req.UpdatedAt) + if err != nil { + if pgErr, ok := errors.AsType[*pgconn.PgError](err); ok && + pgErr.Code == pgerrcode.UniqueViolation && pgErr.ConstraintName == "users_username_unique" { + return apperr.Wrap(err, apperr.CodeConflict, "username already exists") + } + return apperr.WrapPostgres(err, "failed to create user") + } + return nil +} + +func (r *UsersRepo) getByID(ctx context.Context, conn postgres.Conn, id uuid.UUID) (*dto.User, error) { + const query = `SELECT id, username, name, created_at FROM users WHERE id = $1` + return scanUser(conn.QueryRow(ctx, query, id)) +} + +func scanUser(row pgx.Row) (*dto.User, error) { + var user dto.User + if err := row.Scan(&user.ID, &user.Username, &user.DisplayName, &user.CreatedAt); err != nil { + return nil, apperr.WrapPostgres(err, "failed to get user") + } + return &user, nil +} diff --git a/service/internal/config/abstractions.go b/service/internal/config/abstractions.go new file mode 100644 index 00000000..2490832c --- /dev/null +++ b/service/internal/config/abstractions.go @@ -0,0 +1,7 @@ +package config + +import "github.com/spf13/viper" + +type Loader interface { + loadConfig(v *viper.Viper) error +} diff --git a/service/internal/config/application.go b/service/internal/config/application.go new file mode 100644 index 00000000..782dcc2d --- /dev/null +++ b/service/internal/config/application.go @@ -0,0 +1,63 @@ +package config + +import ( + "errors" + "time" + + "github.com/spf13/viper" + + "gitea.privcloud.tech/mtusi2026/internal/pkg/types/environment" +) + +type Application struct { + LogLevel string + ShutdownTimeout time.Duration + Environment environment.ApplicationEnvironment + RequestRateLimit float64 + BurstRateLimit int + RateLimitTTL time.Duration +} + +func (c *Application) Validate() error { + environmentErr := c.Environment.Validate() + + switch { + case c.LogLevel == "": + return wrapValidationError("application", errors.New("log_level cannot be empty in application config")) + case c.ShutdownTimeout == 0: + return wrapValidationError("application", errors.New("shutdown_timeout cannot be zero in application config")) + case environmentErr != nil: + return wrapValidationError("application", environmentErr) + //case c.RequestRateLimit == 0: + // return wrapValidationError("application", errors.New("request_rate_limit cannot be zero in application config")) + //case c.BurstRateLimit == 0: + // return wrapValidationError("application", errors.New("burst_rate_limit cannot be zero in application config")) + //case c.RateLimitTTL == 0: + // return wrapValidationError("application", errors.New("rate_limit_ttl cannot be zero in application config")) + default: + return nil + } +} + +func (c *Application) loadConfig(v *viper.Viper) error { + v.SetEnvPrefix("APPLICATION") + c.LogLevel = v.GetString("LOG_LEVEL") + c.ShutdownTimeout = v.GetDuration("SHUTDOWN_TIMEOUT") + c.Environment = environment.ParseEnvironment(v.GetString("ENVIRONMENT")) + // todo add cool value + c.RequestRateLimit = v.GetFloat64("REQUEST_RATE_LIMIT") + c.BurstRateLimit = v.GetInt("BURST_RATE_LIMIT") + c.RateLimitTTL = v.GetDuration("RATE_LIMIT_TTL") + return nil +} + +func (c *Application) copy() *Application { + return &Application{ + LogLevel: c.LogLevel, + ShutdownTimeout: c.ShutdownTimeout, + Environment: c.Environment, + RequestRateLimit: c.RequestRateLimit, + BurstRateLimit: c.BurstRateLimit, + RateLimitTTL: c.RateLimitTTL, + } +} diff --git a/service/internal/config/attachment.go b/service/internal/config/attachment.go new file mode 100644 index 00000000..38b1a2ee --- /dev/null +++ b/service/internal/config/attachment.go @@ -0,0 +1,58 @@ +package config + +import ( + "fmt" + "os" + "strconv" + "strings" + "time" +) + +type Attachment struct { + Endpoint, Region, Bucket, AccessKey, SecretKey string + MaxBytes, MaxPixels int64 + Timeout, Grace time.Duration + Uploads, Decoders int +} + +func LoadAttachment() (*Attachment, error) { + c := &Attachment{Endpoint: env("S3_ENDPOINT", "http://s3:9000"), Region: env("S3_REGION", "us-east-1"), Bucket: env("S3_BUCKET", "messenger-attachments"), MaxBytes: 10485760, MaxPixels: 40000000, Timeout: 120 * time.Second, Grace: time.Hour, Uploads: 4, Decoders: 1} + var err error + for key, dst := range map[string]*string{"S3_ACCESS_KEY_ID_FILE": &c.AccessKey, "S3_SECRET_ACCESS_KEY_FILE": &c.SecretKey} { + b, e := os.ReadFile(os.Getenv(key)) + if e != nil { + return nil, fmt.Errorf("read %s: %w", key, e) + } + *dst = strings.TrimSpace(string(b)) + if *dst == "" { + return nil, fmt.Errorf("empty %s", key) + } + } + for key, dst := range map[string]*int64{"MAX_UPLOAD_BYTES": &c.MaxBytes, "MAX_IMAGE_PIXELS": &c.MaxPixels} { + if v := os.Getenv(key); v != "" { + *dst, err = strconv.ParseInt(v, 10, 64) + if err != nil { + return nil, fmt.Errorf("invalid %s", key) + } + } + } + for key, dst := range map[string]*time.Duration{"UPLOAD_TIMEOUT": &c.Timeout, "RECONCILE_GRACE_PERIOD": &c.Grace} { + if v := os.Getenv(key); v != "" { + *dst, err = time.ParseDuration(v) + if err != nil { + return nil, fmt.Errorf("invalid %s", key) + } + } + } + if c.MaxBytes < 1 || c.MaxBytes > 10485760 || c.MaxPixels < 1 || c.MaxPixels > 40000000 || c.Timeout < time.Second || c.Timeout > 120*time.Second || c.Grace <= 2*c.Timeout { + return nil, fmt.Errorf("invalid attachment limits or grace period") + } + return c, nil +} + +func env(k, d string) string { + if v := os.Getenv(k); v != "" { + return v + } + return d +} diff --git a/service/internal/config/config.go b/service/internal/config/config.go new file mode 100644 index 00000000..e67e907a --- /dev/null +++ b/service/internal/config/config.go @@ -0,0 +1,110 @@ +package config + +import ( + "strings" + "time" + + "github.com/spf13/viper" + "go.uber.org/multierr" + + "gitea.privcloud.tech/mtusi2026/internal/pkg/types/environment" +) + +var defaultConfig = &Config{ + Server: &Server{ + Host: "localhost", + Port: 8001, + ReadTimeout: 10 * time.Second, + WriteTimeout: 10 * time.Second, + IdleTimeout: 10 * time.Second, + ShutdownTimeout: 10 * time.Second, + Enabled: false, + }, + Postgres: &Postgres{ + Host: "localhost", + Port: 5432, + User: "postgres", + Password: "postgres", + Database: "postgres", + VersionTable: "versions", + RunMigrations: false, + MaxConnections: 10, + ConnectTimeout: 500 * time.Millisecond, + QueryTimeout: 2 * time.Second, + }, + Redis: &Redis{ + Host: "localhost", + Port: 6379, + Username: "", + Password: "", + DB: 0, + }, + JWT: &JWT{ + AccessTokenLifetime: 4 * time.Hour, + RefreshTokenLifetime: time.Hour, + PublicKeyPath: "certs/private.pem", + PrivateKeyPath: "certs/public.pem", + }, + Application: &Application{ + LogLevel: "info", + ShutdownTimeout: 10 * time.Second, + Environment: environment.ApplicationEnvironmentDev, + RequestRateLimit: 100, + }, +} + +type Config struct { + Server *Server + Postgres *Postgres + Redis *Redis + JWT *JWT + Application *Application +} + +func New() (*Config, error) { + v := viper.New() + v.SetEnvKeyReplacer(strings.NewReplacer(".", "_")) + v.AutomaticEnv() + + cfg := defaultConfig.copy() + + if err := cfg.loadConfig(v); err != nil { + return nil, err + } + + if err := cfg.Validate(); err != nil { + return nil, err + } + + return cfg, nil +} + +func (c *Config) Validate() error { + return multierr.Combine( + c.Server.Validate(), + c.Postgres.Validate(), + c.JWT.Validate(), + c.Redis.Validate(), + c.Application.Validate(), + ) +} + +func (c *Config) loadConfig(v *viper.Viper) error { + return multierr.Combine( + c.Postgres.loadConfig(v), + c.Server.loadConfig(v), + c.JWT.loadConfig(v), + c.Redis.loadConfig(v), + c.Application.loadConfig(v), + ) +} + +func (c *Config) copy() *Config { + return &Config{ + Server: c.Server.copy(), + Postgres: c.Postgres.copy(), + Redis: c.Redis.copy(), + JWT: c.JWT.copy(), + Application: c.Application.copy(), + } +} diff --git a/service/internal/config/cookies.go b/service/internal/config/cookies.go new file mode 100644 index 00000000..b6f7d881 --- /dev/null +++ b/service/internal/config/cookies.go @@ -0,0 +1,65 @@ +package config + +import ( + "errors" + "net/http" + "strings" + + "github.com/spf13/viper" +) + +type Cookies struct { + Enabled bool + Domain string + Path string + Secure bool + SameSite http.SameSite + AllowedOrigins []string +} + +func (c *Cookies) Validate() error { + switch { + case c.Domain == "": + return wrapValidationError("cookies", errors.New("domain cannot be empty")) + case c.Path == "": + return wrapValidationError("cookies", errors.New("path cannot be empty")) + case c.SameSite == http.SameSiteDefaultMode: + return wrapValidationError("cookies", errors.New("same_site_set_to_none_mode cannot be empty")) + //case c.Enabled && !c.Secure: + // return wrapValidationError("cookies", errors.New("if cookies_enabled equals true then cookies_secure must be true")) + case c.AllowedOrigins == nil || len(c.AllowedOrigins) == 0: + return wrapValidationError("cookies", errors.New("allowed_origins cannot be empty")) + default: + return nil + } +} + +func (c *Cookies) loadConfig(v *viper.Viper) error { + v.SetEnvPrefix("COOKIES") + c.Enabled = v.GetBool("ENABLED") + c.Domain = v.GetString("DOMAIN") + c.Path = v.GetString("PATH") + c.Secure = v.GetBool("SECURE") + c.AllowedOrigins = v.GetStringSlice("ALLOWED_ORIGINS") + sameSite := v.GetString("SAME_SITE") + switch strings.ToLower(strings.TrimSpace(sameSite)) { + case "strict": + c.SameSite = http.SameSiteStrictMode + case "none": + c.SameSite = http.SameSiteNoneMode + default: + c.SameSite = http.SameSiteLaxMode + } + return nil +} + +func (c *Cookies) copy() *Cookies { + return &Cookies{ + Enabled: c.Enabled, + Domain: c.Domain, + Path: c.Path, + Secure: c.Secure, + SameSite: c.SameSite, + AllowedOrigins: c.AllowedOrigins, + } +} diff --git a/service/internal/config/jwt.go b/service/internal/config/jwt.go new file mode 100644 index 00000000..a48bdd6b --- /dev/null +++ b/service/internal/config/jwt.go @@ -0,0 +1,63 @@ +package config + +import ( + "errors" + "os" + "strconv" + "time" + + "github.com/spf13/viper" +) + +type JWT struct { + AccessTokenLifetime time.Duration + RefreshTokenLifetime time.Duration + PublicKeyPath string + PrivateKeyPath string +} + +func (c *JWT) Validate() error { + switch { + case c.AccessTokenLifetime <= 0: + return wrapValidationError("jwt", errors.New("access_token_lifetime cannot be empty")) + case c.RefreshTokenLifetime <= 0: + return wrapValidationError("jwt", errors.New("refresh_token_lifetime cannot be empty")) + case c.PrivateKeyPath == "": + return wrapValidationError("jwt", errors.New("private_key_path cannot be empty")) + case c.PublicKeyPath == "": + return wrapValidationError("jwt", errors.New("public_key_path cannot be empty")) + default: + return nil + } +} + +func (c *JWT) loadConfig(v *viper.Viper) error { + v.SetEnvPrefix("JWT") + c.AccessTokenLifetime = v.GetDuration("ACCESS_TOKEN_LIFETIME") + c.RefreshTokenLifetime = time.Hour + if value := os.Getenv("SESSION_TTL_SECONDS"); value != "" { + seconds, err := strconv.ParseInt(value, 10, 64) + if err != nil || seconds <= 0 || seconds > int64((1<<63-1)/time.Second) { + return wrapValidationError("session", errors.New("SESSION_TTL_SECONDS must be a positive integer duration in seconds")) + } + c.RefreshTokenLifetime = time.Duration(seconds) * time.Second + } else if legacy := v.GetString("REFRESH_TOKEN_LIFETIME"); legacy != "" { + duration, err := time.ParseDuration(legacy) + if err != nil { + return wrapValidationError("jwt", errors.New("invalid refresh_token_lifetime")) + } + c.RefreshTokenLifetime = duration + } + c.PublicKeyPath = v.GetString("PUBLIC_KEY_PATH") + c.PrivateKeyPath = v.GetString("PRIVATE_KEY_PATH") + return nil +} + +func (c *JWT) copy() *JWT { + return &JWT{ + AccessTokenLifetime: c.AccessTokenLifetime, + RefreshTokenLifetime: c.RefreshTokenLifetime, + PublicKeyPath: c.PublicKeyPath, + PrivateKeyPath: c.PrivateKeyPath, + } +} diff --git a/service/internal/config/postgres.go b/service/internal/config/postgres.go new file mode 100644 index 00000000..478642a7 --- /dev/null +++ b/service/internal/config/postgres.go @@ -0,0 +1,110 @@ +package config + +import ( + "errors" + "fmt" + "math" + "time" + + "github.com/spf13/viper" +) + +const ( + uri = "postgres://%s:%s@%s:%d/%s?sslmode=disable" +) + +type Postgres struct { + Host string + Port int + User string + Password string + Database string + VersionTable string + RunMigrations bool + MaxConnections int + ConnectTimeout time.Duration + QueryTimeout time.Duration +} + +func (c *Postgres) Validate() error { + switch { + case c.Host == "": + return wrapValidationError("postgres", errors.New("host cannot be empty")) + case c.Port == 0: + return wrapValidationError("postgres", errors.New("port cannot be empty")) + case c.User == "": + return wrapValidationError("postgres", errors.New("user cannot be empty")) + case c.Password == "": + return wrapValidationError("postgres", errors.New("password cannot be empty")) + case c.Database == "": + return wrapValidationError("postgres", errors.New("database cannot be empty")) + case c.VersionTable == "": + return wrapValidationError("postgres", errors.New("version_table name cannot be empty")) + case c.MaxConnections <= 0 || int64(c.MaxConnections) > math.MaxInt32: + return wrapValidationError("postgres", errors.New("max_connections must be between 1 and 2147483647")) + case c.ConnectTimeout <= 0: + return wrapValidationError("postgres", errors.New("connect_timeout must be positive")) + case c.QueryTimeout < time.Millisecond: + return wrapValidationError("postgres", errors.New("query_timeout must be at least 1ms")) + default: + return nil + } +} + +func (c *Postgres) GetMaxConnections() int32 { return int32(c.MaxConnections) } +func (c *Postgres) GetConnectTimeout() time.Duration { return c.ConnectTimeout } +func (c *Postgres) GetQueryTimeout() time.Duration { return c.QueryTimeout } + +func (c *Postgres) GetVersionTableName() string { + if c == nil { + return "" + } + return c.VersionTable +} + +func (c *Postgres) GetURI() string { + if c == nil { + return "" + } + return fmt.Sprintf(uri, c.User, c.Password, c.Host, c.Port, c.Database) +} + +func (c *Postgres) GetRunMigrations() bool { + if c == nil { + return false + } + return c.RunMigrations +} + +func (c *Postgres) loadConfig(v *viper.Viper) error { + v.SetEnvPrefix("POSTGRES") + c.Host = v.GetString("HOST") + c.Port = v.GetInt("PORT") + c.User = v.GetString("USER") + c.Password = v.GetString("PASSWORD") + c.Database = v.GetString("DATABASE") + c.VersionTable = v.GetString("VERSION_TABLE") + c.RunMigrations = v.GetBool("RUN_MIGRATIONS") + c.MaxConnections = v.GetInt("MAX_CONNECTIONS") + c.ConnectTimeout = v.GetDuration("CONNECT_TIMEOUT") + c.QueryTimeout = v.GetDuration("QUERY_TIMEOUT") + return nil +} + +func (c *Postgres) copy() *Postgres { + if c == nil { + return nil + } + return &Postgres{ + Host: c.Host, + Port: c.Port, + User: c.User, + Password: c.Password, + Database: c.Database, + MaxConnections: c.MaxConnections, + ConnectTimeout: c.ConnectTimeout, + QueryTimeout: c.QueryTimeout, + RunMigrations: c.RunMigrations, + VersionTable: c.VersionTable, + } +} diff --git a/service/internal/config/redis.go b/service/internal/config/redis.go new file mode 100644 index 00000000..cdeafa59 --- /dev/null +++ b/service/internal/config/redis.go @@ -0,0 +1,72 @@ +package config + +import ( + "errors" + "fmt" + "net" + "time" + + "github.com/redis/go-redis/v9" + "github.com/spf13/viper" +) + +type Redis struct { + Host string + Port int + Username string + Password string + DB int +} + +func (c *Redis) Validate() error { + switch { + case c.Host == "": + return wrapValidationError("redis", errors.New("host cannot be empty")) + case c.Port == 0: + return wrapValidationError("redis", errors.New("port cannot be empty")) + default: + return nil + } +} + +func (c *Redis) BuildRedisOptions() *redis.Options { + if c == nil { + return nil + } + return &redis.Options{ + Addr: net.JoinHostPort(c.Host, fmt.Sprintf("%d", c.Port)), + Username: c.Username, + Password: c.Password, + DB: c.DB, + DialTimeout: 200 * time.Millisecond, + ReadTimeout: 200 * time.Millisecond, + WriteTimeout: 200 * time.Millisecond, + PoolTimeout: 200 * time.Millisecond, + MaxRetries: -1, + DialerRetries: 1, + ContextTimeoutEnabled: true, + } +} + +func (c *Redis) loadConfig(v *viper.Viper) error { + v.SetEnvPrefix("REDIS") + c.Host = v.GetString("HOST") + c.Port = v.GetInt("PORT") + c.DB = v.GetInt("DB") + c.Password = v.GetString("PASSWORD") + c.Username = v.GetString("USERNAME") + return nil +} + +func (c *Redis) copy() *Redis { + if c == nil { + return nil + } + return &Redis{ + Host: c.Host, + Port: c.Port, + Username: c.Username, + Password: c.Password, + DB: c.DB, + } +} diff --git a/service/internal/config/server.go b/service/internal/config/server.go new file mode 100644 index 00000000..d3f9fbe7 --- /dev/null +++ b/service/internal/config/server.go @@ -0,0 +1,69 @@ +package config + +import ( + "errors" + "fmt" + "time" + + "github.com/spf13/viper" +) + +type Server struct { + Host string + Port int + ReadTimeout time.Duration + WriteTimeout time.Duration + IdleTimeout time.Duration + ShutdownTimeout time.Duration + Enabled bool +} + +func (c *Server) Validate() error { + if !c.Enabled { + return nil + } + switch { + case c.Host == "": + return wrapValidationError("server", errors.New("host cannot be empty")) + case c.Port == 0: + return wrapValidationError("server", errors.New("port cannot be empty")) + case c.ReadTimeout == 0: + return wrapValidationError("server", errors.New("read_timeout cannot be empty")) + case c.WriteTimeout == 0: + return wrapValidationError("server", errors.New("write_timeout cannot be empty")) + case c.IdleTimeout == 0: + return wrapValidationError("server", errors.New("idle_timeout cannot be empty")) + case c.ShutdownTimeout == 0: + return wrapValidationError("server", errors.New("shutdown_timeout name cannot be empty")) + default: + return nil + } +} + +func (c *Server) loadConfig(v *viper.Viper) error { + v.SetEnvPrefix("APP") + c.Host = v.GetString("HOST") + c.Port = v.GetInt("PORT") + c.ReadTimeout = v.GetDuration("READ_TIMEOUT") + c.WriteTimeout = v.GetDuration("WRITE_TIMEOUT") + c.IdleTimeout = v.GetDuration("IDLE_TIMEOUT") + c.ShutdownTimeout = v.GetDuration("SHUTDOWN_TIMEOUT") + c.Enabled = v.GetBool("ENABLED") + return c.Validate() +} + +func (c *Server) BindAddress() string { + return fmt.Sprintf("%s:%d", c.Host, c.Port) +} + +func (c *Server) copy() *Server { + return &Server{ + Host: c.Host, + Port: c.Port, + ReadTimeout: c.ReadTimeout, + WriteTimeout: c.WriteTimeout, + IdleTimeout: c.IdleTimeout, + ShutdownTimeout: c.ShutdownTimeout, + Enabled: c.Enabled, + } +} diff --git a/service/internal/config/validation.go b/service/internal/config/validation.go new file mode 100644 index 00000000..aeb45020 --- /dev/null +++ b/service/internal/config/validation.go @@ -0,0 +1,11 @@ +package config + +import "fmt" + +func wrapValidationError(name string, err error) error { + if err == nil { + return nil + } + + return fmt.Errorf("%s validation error: %w", name, err) +} diff --git a/service/internal/dto/attachment.go b/service/internal/dto/attachment.go new file mode 100644 index 00000000..76d3a172 --- /dev/null +++ b/service/internal/dto/attachment.go @@ -0,0 +1 @@ +package dto diff --git a/service/internal/dto/auth.go b/service/internal/dto/auth.go new file mode 100644 index 00000000..5b566863 --- /dev/null +++ b/service/internal/dto/auth.go @@ -0,0 +1,126 @@ +package dto + +import ( + "errors" + "net/netip" + "time" + + "github.com/google/uuid" + + "gitea.privcloud.tech/mtusi2026/internal/pkg/types/role" +) + +// ---- Tokens ---- + +type UserDataInToken struct { + TokenID uuid.UUID + UserID uuid.UUID + Role role.Role + IsAccess bool + + PhoneNumber *string + Email *string + + ExpiresAt time.Time + IssuedAt time.Time +} + +func (u UserDataInToken) Validate() error { + if u.TokenID == uuid.Nil { + return errors.New("invalid token id") + } + if u.UserID == uuid.Nil { + return errors.New("invalid user id") + } + if !u.Role.IsValid() { + return errors.New("invalid role") + } + if u.ExpiresAt.IsZero() { + return errors.New("invalid expiry") + } + if u.IssuedAt.IsZero() { + return errors.New("invalid issued") + } + return nil +} + +type Token struct { + Payload string +} + +type ( + AccessToken Token + RefreshToken Token +) + +type Tokens struct { + AccessToken AccessToken + RefreshToken RefreshToken +} + +func (t AccessToken) Token() *Token { + return new(Token(t)) +} + +func (t RefreshToken) Token() *Token { + return new(Token(t)) +} + +type StoredRefreshToken struct { + ID uuid.UUID + UserID uuid.UUID + TokenHash string + UserAgent string + IPAddress netip.Addr + Role role.Role + ExpiresAt time.Time + RevokedAt *time.Time + CreatedAt time.Time +} + +type RevokeRefreshToken struct { + ID uuid.UUID + RevokedAt time.Time +} + +type RevokeRefreshTokensByUserID struct { + UserID uuid.UUID + RevokedAt time.Time +} + +// ---- Auth ---- + +type Session struct { + ExpiresAt time.Time + Token string + TokenType string + User User +} + +type Login struct { + Username string + Password string +} + +type LoginResult struct { + ExpiresAt time.Time + TokenType string + User User + Tokens Tokens +} + +// UserCredentials is internal authentication data, never a public user response. +type UserCredentials struct { + User User + PasswordHash string `json:"-"` +} + +type Logout struct { + Token string +} + +type Register struct { + Email string + Password string + RepeatPassword string +} diff --git a/service/internal/dto/chat.go b/service/internal/dto/chat.go new file mode 100644 index 00000000..eaf6bc81 --- /dev/null +++ b/service/internal/dto/chat.go @@ -0,0 +1,31 @@ +package dto + +import ( + "time" + + "github.com/google/uuid" +) + +type Chat struct { + ID uuid.UUID + Title string + MemberIDs []uuid.UUID + CreatedAt time.Time +} + +type CreateChat struct { + CreatorID uuid.UUID + Title string + MemberIDs []uuid.UUID +} + +type ChatMember struct { + UserID uuid.UUID + ChatID uuid.UUID +} + +type ChatFilters struct { + UserID uuid.UUID + Limit *int + AfterID *uuid.UUID +} diff --git a/service/internal/dto/message.go b/service/internal/dto/message.go new file mode 100644 index 00000000..b85296da --- /dev/null +++ b/service/internal/dto/message.go @@ -0,0 +1,38 @@ +package dto + +import ( + "time" + + "github.com/google/uuid" +) + +type Message struct { + AttachmentIDs []uuid.UUID + ID uuid.UUID + ChatID uuid.UUID + SenderID uuid.UUID + Text string + CreatedAt time.Time +} + +type CreateMessage struct { + AttachmentIDs []uuid.UUID + IdempotencyKey string + ChatID uuid.UUID + SenderID uuid.UUID + Text string +} + +type MessageFilters struct { + ChatID uuid.UUID + UserID uuid.UUID + Limit *int + AfterID *uuid.UUID +} + +// MessageAttachment contains only the data needed to authorize linking an attachment. +type MessageAttachment struct { + ID uuid.UUID + ChatID uuid.UUID + Missing bool +} diff --git a/service/internal/dto/page.go b/service/internal/dto/page.go new file mode 100644 index 00000000..2af3f605 --- /dev/null +++ b/service/internal/dto/page.go @@ -0,0 +1,25 @@ +package dto + +import ( + "time" + + "github.com/google/uuid" +) + +// Page is a page of records with the ID to use for the next page. +type Page[T any] struct { + Items []T + NextCursorID *uuid.UUID +} + +// PagePosition is a resolved database cursor, ordered by creation time and ID. +type PagePosition struct { + CreatedAt time.Time + ID uuid.UUID +} + +// PageQuery is prepared by a service; Limit includes the lookahead row. +type PageQuery struct { + Limit int + After *PagePosition +} diff --git a/service/internal/dto/user.go b/service/internal/dto/user.go new file mode 100644 index 00000000..b617e0a6 --- /dev/null +++ b/service/internal/dto/user.go @@ -0,0 +1,30 @@ +package dto + +import ( + "time" + + "github.com/google/uuid" +) + +type User struct { + ID uuid.UUID + Username string + DisplayName string + CreatedAt time.Time +} + +type CreateUser struct { + Username string + DisplayName string + Password string +} + +// CreateStoredUser contains the prepared database record, never a plaintext password. +type CreateStoredUser struct { + ID uuid.UUID + Username string + DisplayName string + PasswordHash string `json:"-"` + CreatedAt time.Time + UpdatedAt time.Time +} diff --git a/service/internal/gen/contracts/types.gen.go b/service/internal/gen/contracts/types.gen.go new file mode 100644 index 00000000..28d12dbf --- /dev/null +++ b/service/internal/gen/contracts/types.gen.go @@ -0,0 +1,226 @@ +// Package contracts provides primitives to interact with the openapi HTTP API. +// +// Code generated by github.com/oapi-codegen/oapi-codegen/v2 version v2.5.1 DO NOT EDIT. +package contracts + +import ( + "time" + + openapi_types "github.com/oapi-codegen/runtime/types" +) + +const ( + BasicAuthScopes = "basicAuth.Scopes" + BearerAuthScopes = "bearerAuth.Scopes" +) + +// Defines values for AttachmentContentType. +const ( + AttachmentContentTypeImagejpeg AttachmentContentType = "image/jpeg" + AttachmentContentTypeImagepng AttachmentContentType = "image/png" +) + +// Defines values for AttachmentStatus. +const ( + Ready AttachmentStatus = "ready" +) + +// Defines values for ErrorErrorCode. +const ( + Conflict ErrorErrorCode = "conflict" + InternalError ErrorErrorCode = "internal_error" + InvalidRequest ErrorErrorCode = "invalid_request" + NotFound ErrorErrorCode = "not_found" + PayloadTooLarge ErrorErrorCode = "payload_too_large" + RateLimited ErrorErrorCode = "rate_limited" + Unauthorized ErrorErrorCode = "unauthorized" + Unavailable ErrorErrorCode = "unavailable" + UnsupportedMediaType ErrorErrorCode = "unsupported_media_type" +) + +// Defines values for HealthStatus. +const ( + Ok HealthStatus = "ok" +) + +// Defines values for VariantContentType. +const ( + VariantContentTypeImagejpeg VariantContentType = "image/jpeg" + VariantContentTypeImagepng VariantContentType = "image/png" + VariantContentTypeImagewebp VariantContentType = "image/webp" +) + +// Defines values for VariantName. +const ( + Crop VariantName = "crop" + Optimized VariantName = "optimized" + Thumbnail VariantName = "thumbnail" +) + +// Defines values for DownloadAttachmentParamsVariant. +const ( + Original DownloadAttachmentParamsVariant = "original" +) + +// Attachment defines model for Attachment. +type Attachment struct { + ChatId openapi_types.UUID `json:"chat_id"` + ContentType AttachmentContentType `json:"content_type"` + CreatedAt time.Time `json:"created_at"` + ErrorCode *string `json:"error_code"` + Filename string `json:"filename"` + Id openapi_types.UUID `json:"id"` + OwnerId openapi_types.UUID `json:"owner_id"` + Sha256 string `json:"sha256"` + SizeBytes int `json:"size_bytes"` + Status AttachmentStatus `json:"status"` + UpdatedAt time.Time `json:"updated_at"` + Variants []Variant `json:"variants"` +} + +// AttachmentContentType defines model for Attachment.ContentType. +type AttachmentContentType string + +// AttachmentStatus defines model for Attachment.Status. +type AttachmentStatus string + +// Chat defines model for Chat. +type Chat struct { + CreatedAt time.Time `json:"created_at"` + Id openapi_types.UUID `json:"id"` + MemberIds []openapi_types.UUID `json:"member_ids"` + Title string `json:"title"` +} + +// ChatPage defines model for ChatPage. +type ChatPage struct { + Items []Chat `json:"items"` + NextCursor *string `json:"next_cursor"` +} + +// CreateChat defines model for CreateChat. +type CreateChat struct { + MemberIds []openapi_types.UUID `json:"member_ids"` + Title string `json:"title"` +} + +// CreateMessage defines model for CreateMessage. +type CreateMessage struct { + AttachmentIds *[]openapi_types.UUID `json:"attachment_ids,omitempty"` + Text *string `json:"text,omitempty"` +} + +// CreateUser defines model for CreateUser. +type CreateUser struct { + DisplayName string `json:"display_name"` + Password *string `json:"password,omitempty"` + Username string `json:"username"` +} + +// Error defines model for Error. +type Error struct { + Error struct { + Code ErrorErrorCode `json:"code"` + Message string `json:"message"` + RequestId string `json:"request_id"` + } `json:"error"` +} + +// ErrorErrorCode defines model for Error.Error.Code. +type ErrorErrorCode string + +// Health defines model for Health. +type Health struct { + Status HealthStatus `json:"status"` +} + +// HealthStatus defines model for Health.Status. +type HealthStatus string + +// Message defines model for Message. +type Message struct { + AttachmentIds []openapi_types.UUID `json:"attachment_ids"` + ChatId openapi_types.UUID `json:"chat_id"` + CreatedAt time.Time `json:"created_at"` + Id openapi_types.UUID `json:"id"` + SenderId openapi_types.UUID `json:"sender_id"` + Text string `json:"text"` +} + +// MessagePage defines model for MessagePage. +type MessagePage struct { + Items []Message `json:"items"` + NextCursor *string `json:"next_cursor"` +} + +// Session defines model for Session. +type Session struct { + ExpiresAt time.Time `json:"expires_at"` + Token string `json:"token"` + TokenType string `json:"token_type"` + User User `json:"user"` +} + +// User defines model for User. +type User struct { + CreatedAt time.Time `json:"created_at"` + DisplayName string `json:"display_name"` + Id openapi_types.UUID `json:"id"` + Username string `json:"username"` +} + +// Variant defines model for Variant. +type Variant struct { + ContentType VariantContentType `json:"content_type"` + Height int `json:"height"` + Name VariantName `json:"name"` + Sha256 string `json:"sha256"` + SizeBytes int `json:"size_bytes"` + Width int `json:"width"` +} + +// VariantContentType defines model for Variant.ContentType. +type VariantContentType string + +// VariantName defines model for Variant.Name. +type VariantName string + +// DownloadAttachmentParams defines parameters for DownloadAttachment. +type DownloadAttachmentParams struct { + Variant *DownloadAttachmentParamsVariant `form:"variant,omitempty" json:"variant,omitempty"` +} + +// DownloadAttachmentParamsVariant defines parameters for DownloadAttachment. +type DownloadAttachmentParamsVariant string + +// ListChatsParams defines parameters for ListChats. +type ListChatsParams struct { + Limit *int `form:"limit,omitempty" json:"limit,omitempty"` + Cursor *string `form:"cursor,omitempty" json:"cursor,omitempty"` +} + +// UploadAttachmentParams defines parameters for UploadAttachment. +type UploadAttachmentParams struct { + Filename string `form:"filename" json:"filename"` +} + +// ListMessagesParams defines parameters for ListMessages. +type ListMessagesParams struct { + Limit *int `form:"limit,omitempty" json:"limit,omitempty"` + Cursor *string `form:"cursor,omitempty" json:"cursor,omitempty"` +} + +// CreateMessageParams defines parameters for CreateMessage. +type CreateMessageParams struct { + // IdempotencyKey Обязателен с lab3. Область: пользователь + метод + путь + ключ; хранить результат не менее 24 ч. Повтор того же JSON возвращает исходный 201 и то же сообщение, другой JSON — 409. + IdempotencyKey string `json:"Idempotency-Key"` +} + +// CreateChatJSONRequestBody defines body for CreateChat for application/json ContentType. +type CreateChatJSONRequestBody = CreateChat + +// CreateMessageJSONRequestBody defines body for CreateMessage for application/json ContentType. +type CreateMessageJSONRequestBody = CreateMessage + +// RegisterUserJSONRequestBody defines body for RegisterUser for application/json ContentType. +type RegisterUserJSONRequestBody = CreateUser diff --git a/service/internal/pkg/apperr/code.go b/service/internal/pkg/apperr/code.go new file mode 100644 index 00000000..429d6402 --- /dev/null +++ b/service/internal/pkg/apperr/code.go @@ -0,0 +1,274 @@ +package apperr + +import ( + "errors" + "net/http" + + "github.com/jackc/pgerrcode" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgconn" + grpcCodes "google.golang.org/grpc/codes" +) + +//go:generate stringer -type=Code -linecomment +type Code uint16 + +const ( + // CodeBadRequest = {AppError: BAD_REQUEST, HTTP:400} + CodeBadRequest Code = iota + // CodePayloadTooLarge = {AppError: REQUEST_ENTITY_TOO_LARGE, HTTP:413} + CodePayloadTooLarge + // CodeValidation = {AppError: VALIDATION, HTTP:400} + CodeValidation + // CodeUnauthorized = {AppError: UNAUTHORIZED, HTTP:401} + CodeUnauthorized + // CodeForbidden = {AppError: FORBIDDEN, HTTP:403} + CodeForbidden + // CodeNotFound = {AppError: NOT_FOUND, HTTP:404} + CodeNotFound + // CodeAlreadyExists = {AppError: ALREADY_EXISTS, HTTP:409} + CodeAlreadyExists + // CodeConflict = {AppError: CONFLICT, HTTP:409} + CodeConflict + // CodeTooManyRequests = {AppError: TOO_MANY_REQUESTS, HTTP:429} + CodeTooManyRequests + // CodeTimeout = {AppError: TIMEOUT, HTTP:504} + CodeTimeout + // CodeInternal = {AppError: INTERNAL, HTTP:500} + CodeInternal + // CodeServiceUnavailable = {AppError: SERVICE_UNAVAILABLE, HTTP:503} + CodeServiceUnavailable + // CodeOK = {AppError: OK, HTTP:200} + CodeOK + // CodeLocked = {AppError: LOCKED, HTTP:423} + CodeLocked + // CodeUnsupportedMediaType = {AppError: UNSUPPORTED_MEDIA_TYPE, HTTP:415} + CodeUnsupportedMediaType + // CodeUnavailable = {AppError: SERVICE_UNAVAILABLE, HTTP: 503} + CodeUnavailable +) + +func (c Code) HTTPStatus() int { + if status, ok := httpStatusByCode[c]; ok { + return status + } + return http.StatusInternalServerError +} + +func (c Code) GRPCCode() grpcCodes.Code { + if code, ok := grpcCodeByCode[c]; ok { + return code + } + return grpcCodes.Internal +} + +func (c Code) PostgresCodes() []string { + codes := postgresCodesByCode[c] + if len(codes) == 0 { + return nil + } + + cp := make([]string, len(codes)) + copy(cp, codes) + return cp +} + +func (c Code) PostgresCode() string { + codes := c.PostgresCodes() + if len(codes) == 0 { + return "" + } + return codes[0] +} + +func CodeFromError(err error) Code { + if err == nil { + return CodeInternal + } + + var coded interface { + Code() Code + } + if errors.As(err, &coded) { + return coded.Code() + } + + if code, ok := codeFromRedisError(err, false, false); ok { + return code + } + + return CodeFromPostgresError(err) +} + +func HTTPStatusFromError(err error) int { + return CodeFromError(err).HTTPStatus() +} + +func GRPCCodeFromError(err error) grpcCodes.Code { + return CodeFromError(err).GRPCCode() +} + +func CodeFromHTTPStatus(status int) Code { + if code, ok := codeByHTTPStatus[status]; ok { + return code + } + return CodeInternal +} + +func CodeFromGRPC(code grpcCodes.Code) Code { + if appCode, ok := codeByGRPCCode[code]; ok { + return appCode + } + return CodeInternal +} + +func CodeFromPostgres(pgCode string) Code { + if code, ok := codeByPostgresCode[pgCode]; ok { + return code + } + return CodeInternal +} + +func CodeFromPostgresError(err error) Code { + if err == nil { + return CodeInternal + } + + if code, ok := codeByPgxError[err]; ok { + return code + } + + if pgErr, ok := errors.AsType[*pgconn.PgError](err); ok { + return CodeFromPostgres(pgErr.Code) + } + + return CodeInternal +} + +var ( + httpStatusByCode = map[Code]int{ + CodeBadRequest: http.StatusBadRequest, + CodeUnauthorized: http.StatusUnauthorized, + CodeForbidden: http.StatusForbidden, + CodeNotFound: http.StatusNotFound, + CodeAlreadyExists: http.StatusConflict, + CodeConflict: http.StatusConflict, + CodePayloadTooLarge: http.StatusRequestEntityTooLarge, + CodeTooManyRequests: http.StatusTooManyRequests, + CodeValidation: http.StatusBadRequest, + CodeTimeout: http.StatusGatewayTimeout, + CodeInternal: http.StatusInternalServerError, + CodeServiceUnavailable: http.StatusServiceUnavailable, + CodeUnsupportedMediaType: http.StatusUnsupportedMediaType, + CodeUnavailable: http.StatusServiceUnavailable, + CodeOK: http.StatusOK, + CodeLocked: http.StatusLocked, + } + codeByHTTPStatus = map[int]Code{ + http.StatusBadRequest: CodeBadRequest, + http.StatusUnauthorized: CodeUnauthorized, + http.StatusForbidden: CodeForbidden, + http.StatusNotFound: CodeNotFound, + http.StatusConflict: CodeConflict, + http.StatusRequestEntityTooLarge: CodePayloadTooLarge, + http.StatusTooManyRequests: CodeTooManyRequests, + http.StatusUnprocessableEntity: CodeValidation, + http.StatusGatewayTimeout: CodeTimeout, + http.StatusInternalServerError: CodeInternal, + http.StatusServiceUnavailable: CodeServiceUnavailable, + http.StatusOK: CodeOK, + http.StatusLocked: CodeLocked, + http.StatusUnsupportedMediaType: CodeUnavailable, + } + grpcCodeByCode = map[Code]grpcCodes.Code{ + CodeBadRequest: grpcCodes.InvalidArgument, + CodePayloadTooLarge: grpcCodes.ResourceExhausted, + CodeValidation: grpcCodes.InvalidArgument, + CodeUnauthorized: grpcCodes.Unauthenticated, + CodeForbidden: grpcCodes.PermissionDenied, + CodeNotFound: grpcCodes.NotFound, + CodeAlreadyExists: grpcCodes.AlreadyExists, + CodeConflict: grpcCodes.Aborted, + CodeTooManyRequests: grpcCodes.ResourceExhausted, + CodeTimeout: grpcCodes.DeadlineExceeded, + CodeInternal: grpcCodes.Internal, + CodeServiceUnavailable: grpcCodes.Unavailable, + CodeOK: grpcCodes.OK, + CodeLocked: grpcCodes.FailedPrecondition, + } + codeByGRPCCode = map[grpcCodes.Code]Code{ + grpcCodes.InvalidArgument: CodeBadRequest, + grpcCodes.ResourceExhausted: CodeTooManyRequests, + grpcCodes.Unauthenticated: CodeUnauthorized, + grpcCodes.PermissionDenied: CodeForbidden, + grpcCodes.NotFound: CodeNotFound, + grpcCodes.AlreadyExists: CodeAlreadyExists, + grpcCodes.Aborted: CodeConflict, + grpcCodes.FailedPrecondition: CodeLocked, + grpcCodes.DeadlineExceeded: CodeTimeout, + grpcCodes.Internal: CodeInternal, + grpcCodes.Unimplemented: CodeInternal, + grpcCodes.Unavailable: CodeServiceUnavailable, + grpcCodes.OK: CodeOK, + } + postgresCodesByCode = map[Code][]string{ + CodeBadRequest: { + pgerrcode.InvalidTextRepresentation, + pgerrcode.InvalidParameterValue, + }, + CodeValidation: { + pgerrcode.NotNullViolation, + pgerrcode.CheckViolation, + pgerrcode.StringDataRightTruncationDataException, + pgerrcode.NumericValueOutOfRange, + }, + CodeNotFound: { + pgerrcode.ForeignKeyViolation, + }, + CodeAlreadyExists: { + pgerrcode.UniqueViolation, + }, + CodeConflict: { + pgerrcode.ExclusionViolation, + pgerrcode.SerializationFailure, + pgerrcode.DeadlockDetected, + }, + CodeLocked: { + pgerrcode.LockNotAvailable, + pgerrcode.ObjectInUse, + }, + CodeTooManyRequests: { + pgerrcode.TooManyConnections, + }, + CodeTimeout: { + pgerrcode.QueryCanceled, + }, + CodeInternal: { + pgerrcode.ConnectionException, + pgerrcode.ConnectionFailure, + pgerrcode.SQLClientUnableToEstablishSQLConnection, + pgerrcode.SQLServerRejectedEstablishmentOfSQLConnection, + pgerrcode.TransactionResolutionUnknown, + }, + CodeServiceUnavailable: { + pgerrcode.CannotConnectNow, + }, + } + codeByPostgresCode = buildCodeByPostgresCode(postgresCodesByCode) + codeByPgxError = map[error]Code{ + pgx.ErrNoRows: CodeNotFound, + pgx.ErrTxClosed: CodeInternal, + pgx.ErrTxCommitRollback: CodeConflict, + pgx.ErrTooManyRows: CodeInternal, + } +) + +func buildCodeByPostgresCode(src map[Code][]string) map[string]Code { + dst := make(map[string]Code, len(src)) + for code, pgCodes := range src { + for _, pgCode := range pgCodes { + dst[pgCode] = code + } + } + return dst +} diff --git a/service/internal/pkg/apperr/code_string.go b/service/internal/pkg/apperr/code_string.go new file mode 100644 index 00000000..be690bf1 --- /dev/null +++ b/service/internal/pkg/apperr/code_string.go @@ -0,0 +1,37 @@ +// Code generated by "stringer -type=Code -linecomment"; DO NOT EDIT. + +package apperr + +import "strconv" + +func _() { + // An "invalid array index" compiler error signifies that the constant values have changed. + // Re-run the stringer command to generate them again. + var x [1]struct{} + _ = x[CodeBadRequest-0] + _ = x[CodePayloadTooLarge-1] + _ = x[CodeValidation-2] + _ = x[CodeUnauthorized-3] + _ = x[CodeForbidden-4] + _ = x[CodeNotFound-5] + _ = x[CodeAlreadyExists-6] + _ = x[CodeConflict-7] + _ = x[CodeTooManyRequests-8] + _ = x[CodeTimeout-9] + _ = x[CodeInternal-10] + _ = x[CodeServiceUnavailable-11] + _ = x[CodeOK-12] + _ = x[CodeLocked-13] +} + +const _Code_name = "CodeBadRequestCodePayloadTooLargeCodeValidationCodeUnauthorizedCodeForbiddenCodeNotFoundCodeAlreadyExistsCodeConflictCodeTooManyRequestsCodeTimeoutCodeInternalCodeServiceUnavailableCodeOKCodeLocked" + +var _Code_index = [...]uint8{0, 14, 33, 47, 63, 76, 88, 105, 117, 136, 147, 159, 181, 187, 197} + +func (i Code) String() string { + idx := int(i) - 0 + if i < 0 || idx >= len(_Code_index)-1 { + return "Code(" + strconv.FormatInt(int64(i), 10) + ")" + } + return _Code_name[_Code_index[idx]:_Code_index[idx+1]] +} diff --git a/service/internal/pkg/apperr/error.go b/service/internal/pkg/apperr/error.go new file mode 100644 index 00000000..7b45fe0e --- /dev/null +++ b/service/internal/pkg/apperr/error.go @@ -0,0 +1,123 @@ +package apperr + +import ( + "errors" + "fmt" + + grpcCodes "google.golang.org/grpc/codes" +) + +type AppError struct { + err error + msg string + code Code + meta metadata +} + +func New(code Code, msg string) error { + return &AppError{ + msg: msg, + code: code, + meta: metadata{}, + } +} + +func Wrap(err error, code Code, msg string) error { + m := AppError{ + err: err, + msg: msg, + code: code, + meta: metadata{}, + } + return &m +} + +func WrapAppError(err error, msg string) error { + m := AppError{ + err: err, + msg: msg, + code: CodeInternal, + meta: metadata{}, + } + + if mistake := As(err); mistake != nil { + m.err = mistake + m.code = mistake.code + m.meta = mistake.meta + } + + return &m +} + +func As(err error) *AppError { + if mistakeErr, ok := errors.AsType[*AppError](err); ok { + return mistakeErr + } + return nil +} + +func CodeIs(err error, code Code) bool { + if appErr := As(err); appErr != nil && appErr.Code() == code { + return true + } + return false +} + +func (e *AppError) Error() string { + if e == nil { + return "" + } + + if e.err == nil { + if e.msg != "" { + return e.msg + } + return e.code.String() + } + + if e.msg == "" { + return e.err.Error() + } + + return fmt.Sprintf("%s: %s", e.msg, e.err.Error()) +} + +func (e *AppError) Unwrap() error { + if e == nil { + return nil + } + return e.err +} + +func (e *AppError) Code() Code { + if e == nil { + return CodeInternal + } + return e.code +} + +func (e *AppError) Message() string { + if e == nil { + return "" + } + return e.msg +} + +func (e *AppError) Cause() error { + if e == nil { + return nil + } + return e.err +} + +func (e *AppError) Metadata() Metadata { + return e.meta +} + +func (e *AppError) HTTPStatus() int { + return e.Code().HTTPStatus() +} + +func (e *AppError) GRPCCode() grpcCodes.Code { + return e.Code().GRPCCode() +} diff --git a/service/internal/pkg/apperr/meta.go b/service/internal/pkg/apperr/meta.go new file mode 100644 index 00000000..195666ef --- /dev/null +++ b/service/internal/pkg/apperr/meta.go @@ -0,0 +1,42 @@ +package apperr + +import ( + "fmt" + "maps" + "strings" +) + +var _ Metadata = (*metadata)(nil) + +type Metadata interface { + Get(key metakey) string + Clone() map[metakey]string + CloneToAnyMap() map[string]any + String() string +} + +type metadata map[metakey]string + +func (m metadata) Get(key metakey) string { + return m[key] +} + +func (m metadata) Clone() map[metakey]string { + return maps.Clone(m) +} + +func (m metadata) CloneToAnyMap() map[string]any { + newM := make(map[string]any, len(m)) + for k, v := range m { + newM[k.String()] = v + } + return newM +} + +func (m metadata) String() string { + var b strings.Builder + for k, v := range m { + b.WriteString(fmt.Sprintf("%s: %s, ", k, v)) + } + return b.String() +} diff --git a/service/internal/pkg/apperr/metakey.go b/service/internal/pkg/apperr/metakey.go new file mode 100644 index 00000000..7dc091fe --- /dev/null +++ b/service/internal/pkg/apperr/metakey.go @@ -0,0 +1,19 @@ +package apperr + +type metakey uint16 + +const ( + PgErrorCode metakey = iota + RedisErrorCode +) + +func (m metakey) String() string { + switch m { + case PgErrorCode: + return "pg_err_code" + case RedisErrorCode: + return "redis_err_code" + default: + return "unknown" + } +} diff --git a/service/internal/pkg/apperr/postgres.go b/service/internal/pkg/apperr/postgres.go new file mode 100644 index 00000000..9b82e2c0 --- /dev/null +++ b/service/internal/pkg/apperr/postgres.go @@ -0,0 +1,81 @@ +package apperr + +import ( + "context" + "errors" + "fmt" + "io" + "net" + "strings" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgconn" +) + +func WrapPostgres(err error, msg string) error { + m := AppError{ + err: err, + msg: msg, + code: CodeInternal, + meta: metadata{}, + } + + m.handlePostgres() + + return &m +} + +func (e *AppError) handlePostgres() { + if e.err == nil { + e.code = CodeOK + return + } + + var networkError net.Error + var connectError *pgconn.ConnectError + if errors.Is(e.err, context.DeadlineExceeded) || errors.Is(e.err, io.EOF) || + errors.Is(e.err, io.ErrUnexpectedEOF) || errors.As(e.err, &networkError) || errors.As(e.err, &connectError) { + e.code = CodeServiceUnavailable + e.msg = "database unavailable" + return + } + if pgErr, ok := errors.AsType[*pgconn.PgError](e.err); ok { + if strings.HasPrefix(pgErr.Code, "08") || pgErr.Code == "57014" || + pgErr.Code == "57P01" || pgErr.Code == "57P02" || pgErr.Code == "57P03" || pgErr.Code == "53300" { + e.code = CodeServiceUnavailable + e.msg = "database unavailable" + return + } + e.meta[PgErrorCode] = pgErr.Code + e.code = CodeFromPostgres(pgErr.Code) + return + } + switch { + case errors.Is(e.err, pgx.ErrNoRows): + e.msg = fmt.Sprintf("not found: %s", e.msg) + e.code = CodeNotFound + return + case errors.Is(e.err, pgx.ErrTooManyRows): + e.msg = fmt.Sprintf("got too many rows: %s", e.msg) + e.code = CodeInternal + return + case errors.Is(e.err, pgx.ErrTxClosed): + e.msg = fmt.Sprintf("transaction is closed: %s", e.msg) + e.code = CodeInternal + return + case errors.Is(e.err, pgx.ErrTxCommitRollback): + e.msg = fmt.Sprintf("transaction commit resulted in rollback: %s", e.msg) + e.code = CodeInternal + return + } +} + +func (e *AppError) PgCode() string { + if e == nil { + return "" + } + if e.meta == nil { + return "" + } + return e.meta.Get(PgErrorCode) +} diff --git a/service/internal/pkg/apperr/redis.go b/service/internal/pkg/apperr/redis.go new file mode 100644 index 00000000..f2afe561 --- /dev/null +++ b/service/internal/pkg/apperr/redis.go @@ -0,0 +1,170 @@ +package apperr + +import ( + "context" + "errors" + "fmt" + "net" + "strings" + + "github.com/redis/go-redis/v9" +) + +func WrapRedis(err error, msg string) error { + m := AppError{ + err: err, + msg: msg, + meta: metadata{}, + } + + m.handleRedis() + + return &m +} + +func CodeFromRedisError(err error) Code { + code, ok := codeFromRedisError(err, true, true) + if !ok { + return CodeInternal + } + return code +} + +func (e *AppError) handleRedis() { + if e.err == nil { + e.code = CodeOK + return + } + + if code := redisErrorCode(e.err); code != "" { + e.meta[RedisErrorCode] = code + } + + if errors.Is(e.err, redis.Nil) { + e.msg = fmt.Sprintf("not found: %s", e.msg) + } + + e.code = CodeFromRedisError(e.err) +} + +func (e *AppError) RedisCode() string { + if e == nil { + return "" + } + if e.meta == nil { + return "" + } + return e.meta.Get(RedisErrorCode) +} + +func codeFromRedisError(err error, includeTransport bool, includeProtocolString bool) (Code, bool) { + if err == nil { + return CodeInternal, false + } + + switch { + case errors.Is(err, redis.Nil): + return CodeNotFound, true + case errors.Is(err, redis.TxFailedErr): + return CodeConflict, true + case errors.Is(err, redis.ErrCrossSlot): + return CodeBadRequest, true + case errors.Is(err, redis.ErrPoolTimeout): + return CodeTimeout, true + case errors.Is(err, redis.ErrPoolExhausted): + return CodeTooManyRequests, true + case errors.Is(err, redis.ErrClosed): + return CodeServiceUnavailable, true + } + + if includeTransport { + if errors.Is(err, context.Canceled) || + errors.Is(err, context.DeadlineExceeded) { + return CodeTimeout, true + } + + if netErr, ok := errors.AsType[net.Error](err); ok { + if netErr.Timeout() { + return CodeTimeout, true + } + return CodeServiceUnavailable, true + } + } + + hasRedisError := isRedisError(err) + if !hasRedisError && !includeProtocolString { + return CodeInternal, false + } + + switch { + case redis.IsExecAbortError(err): + return CodeConflict, true + case redis.IsMaxClientsError(err): + return CodeTooManyRequests, true + case redis.IsLoadingError(err), + redis.IsReadOnlyError(err), + redis.IsClusterDownError(err), + redis.IsTryAgainError(err), + redis.IsMasterDownError(err), + redis.IsNoReplicasError(err), + redis.IsOOMError(err): + return CodeServiceUnavailable, true + case redis.IsAuthError(err), + redis.IsPermissionError(err): + return CodeInternal, true + } + + if _, ok := redis.IsMovedError(err); ok { + return CodeServiceUnavailable, true + } + if _, ok := redis.IsAskError(err); ok { + return CodeServiceUnavailable, true + } + + if hasRedisError { + return CodeInternal, true + } + + return CodeInternal, false +} + +func isRedisError(err error) bool { + if err == nil { + return false + } + + var redisErr redis.Error + return errors.As(err, &redisErr) +} + +func redisErrorCode(err error) string { + switch { + case err == nil: + return "" + case errors.Is(err, redis.Nil): + return "nil" + case errors.Is(err, redis.TxFailedErr): + return "tx_failed" + case errors.Is(err, redis.ErrCrossSlot): + return "CROSSSLOT" + case errors.Is(err, redis.ErrPoolTimeout): + return "pool_timeout" + case errors.Is(err, redis.ErrPoolExhausted): + return "pool_exhausted" + case errors.Is(err, redis.ErrClosed): + return "closed" + } + + var redisErr redis.Error + if !errors.As(err, &redisErr) { + return "" + } + + msg := strings.TrimSpace(redisErr.Error()) + msg = strings.TrimPrefix(msg, "ERR ") + parts := strings.Fields(msg) + if len(parts) == 0 { + return "" + } + return parts[0] +} diff --git a/service/internal/pkg/auth/jwtprovider/claims.go b/service/internal/pkg/auth/jwtprovider/claims.go new file mode 100644 index 00000000..bc8dfac9 --- /dev/null +++ b/service/internal/pkg/auth/jwtprovider/claims.go @@ -0,0 +1,145 @@ +package jwtprovider + +import ( + "fmt" + "log/slog" + "time" + + "github.com/golang-jwt/jwt/v5" + "github.com/google/uuid" + + "gitea.privcloud.tech/mtusi2026/internal/dto" + "gitea.privcloud.tech/mtusi2026/internal/pkg/types/role" +) + +var _ Claims = (*jwtClaims)(nil) + +type Claims interface { + GetTopTokenID() (uuid.UUID, bool) + GetTopUserID() (uuid.UUID, bool) + GetTopRole() (role.Role, bool) + GetTopIsAccess() (bool, bool) + GetTopExpiresAt() (time.Time, bool) + GetTopIssuedAt() (time.Time, bool) +} + +type claimsParseFunc func(token *dto.Token) (*jwtClaims, error) + +type jwtClaims struct { + jwt.RegisteredClaims + + Nonce string `json:"nonce,omitempty"` + IsAccess bool `json:"is_access"` + Role role.Role `json:"role"` +} + +func (claims *jwtClaims) GetTopTokenID() (uuid.UUID, bool) { + parsed, err := uuid.Parse(claims.ID) + if err != nil { + return uuid.Nil, false + } + return parsed, true +} + +func (claims *jwtClaims) GetTopUserID() (uuid.UUID, bool) { + parsed, err := uuid.Parse(claims.Subject) + if err != nil { + return uuid.Nil, false + } + return parsed, true +} + +func (claims *jwtClaims) GetTopRole() (role.Role, bool) { + return claims.Role, true +} + +func (claims *jwtClaims) GetTopIsAccess() (bool, bool) { + return claims.IsAccess, true +} + +func (claims *jwtClaims) GetTopExpiresAt() (time.Time, bool) { + if claims.ExpiresAt == nil { + return time.Time{}, false + } + return claims.ExpiresAt.Time, true +} + +func (claims *jwtClaims) GetTopIssuedAt() (time.Time, bool) { + if claims.IssuedAt == nil { + return time.Time{}, false + } + return claims.IssuedAt.Time, true +} + +func (claims *jwtClaims) userData() (*dto.UserDataInToken, error) { + var u dto.UserDataInToken + + tokenID, err := uuid.Parse(claims.ID) + if err != nil { + return nil, fmt.Errorf("failed to parse token id: %w", err) + } + u.TokenID = tokenID + + userID, err := uuid.Parse(claims.Subject) + if err != nil { + return nil, fmt.Errorf("failed to parse user id from token %q: %w", claims.Subject, err) + } + u.UserID = userID + + u.Role = claims.Role + u.IsAccess = claims.IsAccess + + if claims.ExpiresAt != nil { + u.ExpiresAt = claims.ExpiresAt.Time + } + + if claims.IssuedAt != nil { + u.IssuedAt = claims.IssuedAt.Time + } + + return &u, nil +} + +func (p *provider) parseClaimsFromProtectedJWT(token *dto.Token) (*jwtClaims, error) { + if token == nil { + return nil, ErrTokenIsMissing + } + + claims := &jwtClaims{} + + parsed, err := jwt.ParseWithClaims( + token.Payload, + claims, + p.readKeyFunc, + jwt.WithValidMethods([]string{jwt.SigningMethodRS256.Alg()}), + jwt.WithExpirationRequired(), + jwt.WithIssuedAt(), + ) + if err != nil { + slog.Error("failed to parse token with claims", "error", err) + return nil, ErrInvalidToken + } + + if !parsed.Valid { + slog.Error("failed to parse token: provided invalid token", "valid", parsed.Valid) + return nil, ErrInvalidToken + } + + return claims, nil +} + +func (p *provider) parseClaimsFromExposedJWT(token *dto.Token) (*jwtClaims, error) { + if token == nil { + return nil, ErrTokenIsMissing + } + + claims := &jwtClaims{} + + _, _, err := new(jwt.Parser).ParseUnverified(token.Payload, claims) + if err != nil { + slog.Error("failed to parse token: provided invalid token") + return nil, ErrInvalidToken + } + + return claims, nil +} diff --git a/service/internal/pkg/auth/jwtprovider/cookie.go b/service/internal/pkg/auth/jwtprovider/cookie.go new file mode 100644 index 00000000..8609585f --- /dev/null +++ b/service/internal/pkg/auth/jwtprovider/cookie.go @@ -0,0 +1,84 @@ +package jwtprovider + +import ( + "net/http" +) + +const ( + accessTokenCookieKey = "auth-access-token" + refreshTokenCookieKey = "auth-refresh-token" + expiresAtCookieKey = "auth-exp" +) + +type CookieBuilderFabric struct { + fabric func() http.Cookie +} + +func newCookieBuilder( + domain string, + path string, + secure bool, + sameSite http.SameSite, +) *CookieBuilderFabric { + return &CookieBuilderFabric{ + fabric: newCookieFabric(domain, path, secure, sameSite), + } +} + +func (b *CookieBuilderFabric) Builder() *CookieBuilder { + return &CookieBuilder{ + cookie: b.fabric(), + } +} + +func newCookieFabric( + _ string, + path string, + secure bool, + sameSite http.SameSite) func() http.Cookie { + return func() http.Cookie { + return http.Cookie{ + Path: path, + Secure: secure, + SameSite: sameSite, + } + } +} + +type CookieBuilder struct { + cookie http.Cookie +} + +func (b *CookieBuilder) AccessName() *CookieBuilder { + b.cookie.Name = accessTokenCookieKey + return b +} + +func (b *CookieBuilder) ExpiresName() *CookieBuilder { + b.cookie.Name = expiresAtCookieKey + return b +} + +func (b *CookieBuilder) RefreshName() *CookieBuilder { + b.cookie.Name = refreshTokenCookieKey + return b +} + +func (b *CookieBuilder) Value(value string) *CookieBuilder { + b.cookie.Value = value + return b +} + +func (b *CookieBuilder) HTTPOnly() *CookieBuilder { + b.cookie.HttpOnly = true + return b +} + +func (b *CookieBuilder) MaxAge(maxAge int) *CookieBuilder { + b.cookie.MaxAge = maxAge + return b +} + +func (b *CookieBuilder) Build() *http.Cookie { + return new(b.cookie) +} diff --git a/service/internal/pkg/auth/jwtprovider/error.go b/service/internal/pkg/auth/jwtprovider/error.go new file mode 100644 index 00000000..6fde8128 --- /dev/null +++ b/service/internal/pkg/auth/jwtprovider/error.go @@ -0,0 +1,18 @@ +package jwtprovider + +import ( + "errors" +) + +var ( + ErrDecode = errors.New("failed to decode argument") + ErrNotFound = errors.New("not found") + ErrUnexpectedAlgorithm = errors.New("unexpected signing algorithm") + ErrTokenIsMissing = errors.New("token is missing") + ErrInvalidToken = errors.New("provided invalid token") + ErrInvalidUserDataInToken = errors.New("invalid user data in token") + ErrInvalidTokenClaims = errors.New("provided invalid token claims") + ErrInvalidBirthDateFormat = errors.New("provided invalid birth date format") + ErrTokenExpired = errors.New("token is expired") + ErrFailedToSign = errors.New("failed to sign jwt") +) diff --git a/service/internal/pkg/auth/jwtprovider/interface.go b/service/internal/pkg/auth/jwtprovider/interface.go new file mode 100644 index 00000000..ced83eeb --- /dev/null +++ b/service/internal/pkg/auth/jwtprovider/interface.go @@ -0,0 +1,26 @@ +package jwtprovider + +import ( + "net/http" + + "github.com/google/uuid" + + "gitea.privcloud.tech/mtusi2026/internal/dto" + "gitea.privcloud.tech/mtusi2026/internal/pkg/types/role" +) + +type Provider interface { + Parse(token *dto.Token) (*dto.UserDataInToken, error) + CreateRefreshAndAccessTokensForUser(userID uuid.UUID, role role.Role) (*dto.Tokens, error) + ParseVerified(token *dto.Token) (*dto.UserDataInToken, error) + ParseUnverified(token *dto.Token) (*dto.UserDataInToken, error) + ValidateUserDataInToken(u *dto.UserDataInToken) error +} + +type Cookie interface { + Enabled() bool + Set(w http.ResponseWriter, tokens *dto.Tokens) + Clear(w http.ResponseWriter) + AccessToken(r *http.Request) *dto.AccessToken + RefreshToken(r *http.Request) *dto.RefreshToken +} diff --git a/service/internal/pkg/auth/jwtprovider/manager.go b/service/internal/pkg/auth/jwtprovider/manager.go new file mode 100644 index 00000000..baeda313 --- /dev/null +++ b/service/internal/pkg/auth/jwtprovider/manager.go @@ -0,0 +1,199 @@ +package jwtprovider + +import ( + "fmt" + "net/http" + "strconv" + "time" + + "gitea.privcloud.tech/mtusi2026/internal/config" + "gitea.privcloud.tech/mtusi2026/internal/dto" +) + +type manager struct { + enabled bool + cookieFabric *CookieBuilderFabric + prv Provider +} + +func NewCookieManager(cfg *config.Config, provider Provider) Cookie { + m := &manager{ + //enabled: cfg.Cookies.Enabled, + //cookieFabric: newCookieBuilder( + // cfg.Cookies.Domain, + // cfg.Cookies.Path, + // cfg.Cookies.Secure, + // cfg.Cookies.SameSite, + //), + prv: provider, + } + + if !m.enabled { + return m + } + + if err := m.cookieFabric. + Builder(). + AccessName(). + Value("test"). + HTTPOnly(). + MaxAge(0). + Build(). + Valid(); err != nil { + panic(fmt.Errorf("cookie validation failed: %w", err)) + } + + return m +} + +func (m *manager) Enabled() bool { + return m.enabled +} + +func (m *manager) Set(w http.ResponseWriter, tokens *dto.Tokens) { + if !m.enabled || tokens == nil { + return + } + + if tokens.AccessToken.Payload != "" { + m.setAccess(tokens.AccessToken, w) + } + + if tokens.RefreshToken.Payload != "" { + m.setRefresh(tokens.RefreshToken, w) + } +} + +func (m *manager) Clear(w http.ResponseWriter) { + if !m.enabled { + return + } + + for _, name := range []string{accessTokenCookieKey, refreshTokenCookieKey, expiresAtCookieKey} { + b := m.cookieFabric. + Builder(). + MaxAge(-1) + + switch name { + case expiresAtCookieKey: + b.ExpiresName() + case refreshTokenCookieKey: + b.RefreshName(). + HTTPOnly() + case accessTokenCookieKey: + b.AccessName(). + HTTPOnly() + } + http.SetCookie(w, b.Build()) + } +} + +func (m *manager) AccessToken(r *http.Request) *dto.AccessToken { + if !m.enabled { + return nil + } + + c, err := r.Cookie(accessTokenCookieKey) + if err != nil { + return nil + } + + return &dto.AccessToken{ + Payload: c.Value, + } +} + +func (m *manager) RefreshToken(r *http.Request) *dto.RefreshToken { + if !m.enabled { + return nil + } + + c, err := r.Cookie(refreshTokenCookieKey) + if err != nil { + return nil + } + + return &dto.RefreshToken{ + Payload: c.Value, + } +} + +func (m *manager) setAccess(token dto.AccessToken, w http.ResponseWriter) { + accessTTL := m.tokenTTL(token.Token()) + if accessTTL < 0 { + http.SetCookie(w, m.cookieFabric. + Builder(). + AccessName(). + HTTPOnly(). + MaxAge(-1). + Build(), + ) + http.SetCookie(w, m.cookieFabric. + Builder(). + ExpiresName(). + MaxAge(-1). + Build(), + ) + } else if tokenExpiresAt := m.tokenExpiry(token.Token()); !tokenExpiresAt.IsZero() { + http.SetCookie(w, m.cookieFabric. + Builder(). + AccessName(). + Value(token.Payload). + HTTPOnly(). + MaxAge(accessTTL). + Build(), + ) + http.SetCookie(w, m.cookieFabric. + Builder(). + ExpiresName(). + Value(strconv.FormatInt(tokenExpiresAt.Unix(), 10)). + MaxAge(accessTTL). + Build(), + ) + } +} + +func (m *manager) setRefresh(token dto.RefreshToken, w http.ResponseWriter) { + refreshTTL := m.tokenTTL(token.Token()) + if refreshTTL < 0 { + http.SetCookie(w, m.cookieFabric. + Builder(). + RefreshName(). + HTTPOnly(). + MaxAge(-1). + Build(), + ) + } else { + http.SetCookie(w, m.cookieFabric. + Builder(). + RefreshName(). + Value(token.Payload). + HTTPOnly(). + MaxAge(refreshTTL). + Build(), + ) + } +} + +func (m *manager) tokenTTL(token *dto.Token) int { + expiresAt := m.tokenExpiry(token) + + if expiresAt.IsZero() { + return -1 + } + + ttl := int(time.Until(expiresAt).Seconds()) + if ttl <= 0 { + return -1 + } + + return ttl +} + +func (m *manager) tokenExpiry(token *dto.Token) time.Time { + data, err := m.prv.ParseUnverified(token) + if err != nil { + return time.Time{} + } + return data.ExpiresAt +} diff --git a/service/internal/pkg/auth/jwtprovider/provider.go b/service/internal/pkg/auth/jwtprovider/provider.go new file mode 100644 index 00000000..a387854c --- /dev/null +++ b/service/internal/pkg/auth/jwtprovider/provider.go @@ -0,0 +1,187 @@ +package jwtprovider + +import ( + "crypto/rand" + "crypto/rsa" + "encoding/base64" + "fmt" + "log/slog" + "time" + + "github.com/golang-jwt/jwt/v5" + "github.com/google/uuid" + + "gitea.privcloud.tech/mtusi2026/internal/config" + "gitea.privcloud.tech/mtusi2026/internal/dto" + "gitea.privcloud.tech/mtusi2026/internal/pkg/types/environment" + "gitea.privcloud.tech/mtusi2026/internal/pkg/types/role" +) + +var ( + _ Provider = (*provider)(nil) +) + +type provider struct { + accessLifetime time.Duration + refreshLifetime time.Duration + publicKey *rsa.PublicKey + privateKey *rsa.PrivateKey + parseFunc claimsParseFunc +} + +func New(cfg *config.Config) Provider { + p := provider{ + accessLifetime: cfg.JWT.AccessTokenLifetime, + refreshLifetime: cfg.JWT.RefreshTokenLifetime, + } + p.parseFunc = p.parseClaimsFromProtectedJWT + + if cfg.Application.Environment == environment.ApplicationEnvironmentDev { + p.parseFunc = p.parseClaimsFromExposedJWT + } + + var err error + if p.publicKey, err = readPublicKeyFromFile(cfg.JWT.PublicKeyPath); err != nil { + slog.Error("failed to read public key", "error", err) + panic(fmt.Errorf("failed to read public key: %w", err)) + } + if p.privateKey, err = readPrivateKeyFromFile(cfg.JWT.PrivateKeyPath); err != nil { + slog.Error("failed to read private key", "error", err) + panic(fmt.Errorf("failed to read private key: %w", err)) + } + + return &p +} + +func (p *provider) Parse(token *dto.Token) (*dto.UserDataInToken, error) { + claims, err := p.parseFunc(token) + if err != nil { + return nil, err + } + return claims.userData() +} + +func (p *provider) ParseUnverified(t *dto.Token) (*dto.UserDataInToken, error) { + if t == nil { + return nil, ErrTokenIsMissing + } + + claims, err := p.parseClaimsFromExposedJWT(t) + if err != nil { + return nil, err + } + + return claims.userData() +} + +func (p *provider) ParseVerified(t *dto.Token) (*dto.UserDataInToken, error) { + if t == nil { + return nil, ErrTokenIsMissing + } + claims, err := p.parseClaimsFromProtectedJWT(t) + if err != nil { + return nil, fmt.Errorf("failed to parse token: %w", err) + } + + return claims.userData() +} + +func (p *provider) CreateAccessTokenForUser(userID uuid.UUID, role role.Role) (*dto.AccessToken, error) { + t, err := p.createTokenForUser(userID, true, role) + if err != nil { + return nil, err + } + return &dto.AccessToken{ + Payload: t.Payload, + }, nil +} + +func (p *provider) CreateRefreshTokenForUser(userID uuid.UUID, role role.Role) (*dto.RefreshToken, error) { + t, err := p.createTokenForUser(userID, false, role) + if err != nil { + return nil, err + } + return &dto.RefreshToken{ + Payload: t.Payload, + }, nil +} + +func (p *provider) CreateRefreshAndAccessTokensForUser(userID uuid.UUID, role role.Role) (*dto.Tokens, error) { + access, err := p.CreateAccessTokenForUser(userID, role) + if err != nil { + return nil, err + } + refresh, err := p.CreateRefreshTokenForUser(userID, role) + if err != nil { + return nil, err + } + return &dto.Tokens{ + AccessToken: *access, + RefreshToken: *refresh, + }, nil +} + +func (p *provider) createTokenForUser(userID uuid.UUID, isAccess bool, role role.Role) (*dto.Token, error) { + now := time.Now() + var tokenExpiresAt time.Time + if isAccess { + tokenExpiresAt = now.Add(p.accessLifetime) + } else { + tokenExpiresAt = now.Add(p.refreshLifetime) + } + var nonce [32]byte + if _, err := rand.Read(nonce[:]); err != nil { + return nil, fmt.Errorf("generate token nonce: %w", err) + } + claims := &jwtClaims{ + Nonce: base64.RawURLEncoding.EncodeToString(nonce[:]), + RegisteredClaims: jwt.RegisteredClaims{ + Subject: userID.String(), + ExpiresAt: jwt.NewNumericDate(tokenExpiresAt), + NotBefore: jwt.NewNumericDate(now), + IssuedAt: jwt.NewNumericDate(now), + ID: uuid.NewString(), + }, + IsAccess: isAccess, + Role: role, + } + t := jwt.NewWithClaims(jwt.SigningMethodRS256, claims) + + payload, err := t.SignedString(p.privateKey) + if err != nil { + slog.Error("failed to sign jwt", "error", err) + return nil, ErrFailedToSign + } + return &dto.Token{Payload: payload}, nil +} + +// TODO refactor maybe delete +func (p *provider) ValidateUserDataInToken(u *dto.UserDataInToken) error { + if u == nil { + return ErrInvalidUserDataInToken + } + + switch { + case u.TokenID == uuid.Nil: + return fmt.Errorf("%w: unexpected nil token id", ErrInvalidUserDataInToken) + case u.UserID == uuid.Nil: + return fmt.Errorf("%w: unexpected nil user id", ErrInvalidUserDataInToken) + case u.IssuedAt.IsZero(): + return fmt.Errorf("%w: unexpected zero issued at", ErrInvalidUserDataInToken) + case u.Email != nil && *u.Email == "": + return fmt.Errorf("%w: unexpected empty email", ErrInvalidUserDataInToken) + case u.PhoneNumber != nil && *u.PhoneNumber == "": + return fmt.Errorf("%w: unexpected empty phone number", ErrInvalidUserDataInToken) + case u.ExpiresAt.IsZero(): + return fmt.Errorf("%w: unexpected zero expires at", ErrInvalidUserDataInToken) + } + + return nil +} + +func (p *provider) readKeyFunc(token *jwt.Token) (interface{}, error) { + if token.Method != jwt.SigningMethodRS256 { + return nil, fmt.Errorf("unexpected signing method: %v", token.Header["alg"]) + } + return p.publicKey, nil +} diff --git a/service/internal/pkg/auth/jwtprovider/readers.go b/service/internal/pkg/auth/jwtprovider/readers.go new file mode 100644 index 00000000..f2c3ae52 --- /dev/null +++ b/service/internal/pkg/auth/jwtprovider/readers.go @@ -0,0 +1,35 @@ +package jwtprovider + +import ( + "crypto/rsa" + "fmt" + "os" + + "github.com/golang-jwt/jwt/v5" +) + +func readPrivateKeyFromFile(path string) (*rsa.PrivateKey, error) { + key, err := os.ReadFile(path) + if err != nil { + return nil, fmt.Errorf("failed to read public key: %w", err) + } + + privateKey, err := jwt.ParseRSAPrivateKeyFromPEM(key) + if err != nil { + return nil, fmt.Errorf("failed to parse private key: %w", err) + } + return privateKey, nil +} + +func readPublicKeyFromFile(path string) (*rsa.PublicKey, error) { + key, err := os.ReadFile(path) + if err != nil { + return nil, fmt.Errorf("failed to read public key: %w", err) + } + + privateKey, err := jwt.ParseRSAPublicKeyFromPEM(key) + if err != nil { + return nil, fmt.Errorf("failed to parse private key: %w", err) + } + return privateKey, nil +} diff --git a/service/internal/pkg/cache/redis.go b/service/internal/pkg/cache/redis.go new file mode 100644 index 00000000..91f90eb4 --- /dev/null +++ b/service/internal/pkg/cache/redis.go @@ -0,0 +1,43 @@ +package cache + +import ( + "context" + "time" + + "github.com/redis/go-redis/v9" + + "gitea.privcloud.tech/mtusi2026/internal/config" +) + +const defaultRedisPort = 6379 + +// OperationTimeout bounds each optional cache operation independently of the request. +const OperationTimeout = 300 * time.Millisecond + +type Client struct { + *redis.Client +} + +func New(cfg *config.Config) *Client { + if cfg.Redis.Port == 0 { + cfg.Redis.Port = defaultRedisPort + } + + return &Client{ + Client: redis.NewClient(cfg.Redis.BuildRedisOptions()), + } +} + +func (c *Client) Ping(ctx context.Context) error { + ctx, cancel := context.WithTimeout(ctx, OperationTimeout) + defer cancel() + return c.Client.Ping(ctx).Err() +} + +func (c *Client) Start(ctx context.Context) error { + return c.Ping(ctx) +} + +func (c *Client) Stop(_ context.Context) error { + return c.Client.Close() +} diff --git a/service/internal/pkg/encrypting/argon2id.go b/service/internal/pkg/encrypting/argon2id.go new file mode 100644 index 00000000..3a52f9b8 --- /dev/null +++ b/service/internal/pkg/encrypting/argon2id.go @@ -0,0 +1,71 @@ +package encrypting + +import ( + "context" + "crypto/rand" + "crypto/subtle" + "encoding/base64" + "fmt" + "strings" + + "golang.org/x/crypto/argon2" +) + +const ( + argon2idMemory = 64 * 1024 // KiB + argon2idTime = 3 + argon2idThreads = 4 + argon2idSaltLen = 16 + argon2idKeyLen = 32 + // Only this bounded profile is accepted; stored parameters cannot request arbitrary resources. + argon2idPrefix = "$argon2id$v=19$m=65536,t=3,p=4$" +) + +func (e *encryptor) EncryptSecretUsingArgon2id(ctx context.Context, secret string) (string, error) { + if err := ctx.Err(); err != nil { + return "", err + } + var salt [argon2idSaltLen]byte + if _, err := rand.Read(salt[:]); err != nil { + return "", fmt.Errorf("%w: %w", ErrFailedToEncryptPassword, err) + } + key := argon2.IDKey([]byte(secret), salt[:], argon2idTime, argon2idMemory, argon2idThreads, argon2idKeyLen) + if err := ctx.Err(); err != nil { + return "", err + } + return argon2idPrefix + base64.RawStdEncoding.EncodeToString(salt[:]) + "$" + base64.RawStdEncoding.EncodeToString(key), nil +} + +func (e *encryptor) CompareHashAndSecretUsingArgon2id(ctx context.Context, hash, secret string) error { + if err := ctx.Err(); err != nil { + return err + } + encoding := base64.RawStdEncoding.Strict() + if len(hash) != len(argon2idPrefix)+encoding.EncodedLen(argon2idSaltLen)+1+encoding.EncodedLen(argon2idKeyLen) { + return ErrFailedToComparePasswords + } + payload, ok := strings.CutPrefix(hash, argon2idPrefix) + if !ok { + return ErrFailedToComparePasswords + } + saltText, keyText, ok := strings.Cut(payload, "$") + if !ok { + return ErrFailedToComparePasswords + } + salt, err := encoding.DecodeString(saltText) + if err != nil || len(salt) != argon2idSaltLen { + return ErrFailedToComparePasswords + } + storedKey, err := encoding.DecodeString(keyText) + if err != nil || len(storedKey) != argon2idKeyLen { + return ErrFailedToComparePasswords + } + key := argon2.IDKey([]byte(secret), salt, argon2idTime, argon2idMemory, argon2idThreads, argon2idKeyLen) + if err = ctx.Err(); err != nil { + return err + } + if subtle.ConstantTimeCompare(storedKey, key) != 1 { + return ErrFailedToComparePasswords + } + return nil +} diff --git a/service/internal/pkg/encrypting/encryptor.go b/service/internal/pkg/encrypting/encryptor.go new file mode 100644 index 00000000..1495a5be --- /dev/null +++ b/service/internal/pkg/encrypting/encryptor.go @@ -0,0 +1,72 @@ +package encrypting + +import ( + "context" + "crypto/sha256" + "crypto/subtle" + "encoding/hex" + "strings" + + "golang.org/x/crypto/bcrypt" +) + +type encryptor struct { + cost int + generateFromPasswordFunc func([]byte, int) ([]byte, error) + compareHashAndPasswordFunc func([]byte, []byte) error +} + +func New() Encryptor { + return &encryptor{ + cost: bcrypt.DefaultCost, + generateFromPasswordFunc: bcrypt.GenerateFromPassword, + compareHashAndPasswordFunc: bcrypt.CompareHashAndPassword, + } +} + +func (e *encryptor) EncryptSecret(_ context.Context, secret string) (string, error) { + res, err := e.generateFromPasswordFunc([]byte(secret), e.cost) + if err != nil { + return "", ErrFailedToEncryptPassword + } + return string(res), nil +} + +// EncryptSecretUsingSum256 hashes a high-entropy session token, not a password. +func (e *encryptor) EncryptSecretUsingSum256(ctx context.Context, secret string) (string, error) { + if err := ctx.Err(); err != nil { + return "", err + } + sum := sha256.Sum256([]byte(secret)) + return "sha256:" + hex.EncodeToString(sum[:]), nil +} + +func (e *encryptor) CompareHashAndSecret(_ context.Context, hash, password string) error { + err := e.compareHashAndPasswordFunc([]byte(hash), []byte(password)) + if err != nil { + return ErrFailedToComparePasswords + } + return nil +} + +func (e *encryptor) CompareHashAndSecretUsingSum256(ctx context.Context, hash, secret string) error { + if err := ctx.Err(); err != nil { + return err + } + sum := sha256.Sum256([]byte(secret)) + if payload, ok := strings.CutPrefix(hash, "sha256:"); ok { + stored, err := hex.DecodeString(payload) + if err != nil || len(stored) != sha256.Size || subtle.ConstantTimeCompare(stored, sum[:]) != 1 { + return ErrFailedToComparePasswords + } + return nil + } + // Preserve existing lab2/lab3 sessions across a rolling deployment. + // Passwords continue to use Argon2id; only old random-token fingerprints use this fallback. + if strings.HasPrefix(hash, "$2") { + if err := e.compareHashAndPasswordFunc([]byte(hash), []byte(hex.EncodeToString(sum[:]))); err == nil { + return nil + } + } + return ErrFailedToComparePasswords +} diff --git a/service/internal/pkg/encrypting/error.go b/service/internal/pkg/encrypting/error.go new file mode 100644 index 00000000..ffbc22f6 --- /dev/null +++ b/service/internal/pkg/encrypting/error.go @@ -0,0 +1,8 @@ +package encrypting + +import "errors" + +var ( + ErrFailedToEncryptPassword = errors.New("failed to encrypt password") + ErrFailedToComparePasswords = errors.New("failed to compare passwords") +) diff --git a/service/internal/pkg/encrypting/interface.go b/service/internal/pkg/encrypting/interface.go new file mode 100644 index 00000000..4f60f24d --- /dev/null +++ b/service/internal/pkg/encrypting/interface.go @@ -0,0 +1,12 @@ +package encrypting + +import "context" + +type Encryptor interface { + EncryptSecretUsingArgon2id(ctx context.Context, secret string) (string, error) + CompareHashAndSecretUsingArgon2id(ctx context.Context, hash, secret string) error + EncryptSecret(ctx context.Context, secret string) (string, error) + EncryptSecretUsingSum256(_ context.Context, secret string) (string, error) + CompareHashAndSecret(ctx context.Context, hash, secret string) error + CompareHashAndSecretUsingSum256(ctx context.Context, hash, secret string) error +} diff --git a/service/internal/pkg/encrypting/token_test.go b/service/internal/pkg/encrypting/token_test.go new file mode 100644 index 00000000..20934b11 --- /dev/null +++ b/service/internal/pkg/encrypting/token_test.go @@ -0,0 +1,43 @@ +package encrypting + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "golang.org/x/crypto/bcrypt" + "testing" +) + +func TestSessionFingerprints(t *testing.T) { + e := New() + ctx := context.Background() + token := "synthetic-high-entropy-session-value" + hash, err := e.EncryptSecretUsingSum256(ctx, token) + if err != nil { + t.Fatal(err) + } + if err = e.CompareHashAndSecretUsingSum256(ctx, hash, token); err != nil { + t.Fatal(err) + } + for _, bad := range []string{"sha256:00", "sha256:invalid", "unknown"} { + if e.CompareHashAndSecretUsingSum256(ctx, bad, token) == nil { + t.Fatal("accepted malformed fingerprint") + } + } + if e.CompareHashAndSecretUsingSum256(ctx, hash, token+"x") == nil { + t.Fatal("accepted wrong token") + } + sum := sha256.Sum256([]byte(token)) + legacy, err := bcrypt.GenerateFromPassword([]byte(hex.EncodeToString(sum[:])), bcrypt.MinCost) + if err != nil { + t.Fatal(err) + } + if err = e.CompareHashAndSecretUsingSum256(ctx, string(legacy), token); err != nil { + t.Fatal("legacy session rejected", err) + } + canceled, cancel := context.WithCancel(ctx) + cancel() + if e.CompareHashAndSecretUsingSum256(canceled, hash, token) == nil { + t.Fatal("ignored cancellation") + } +} diff --git a/service/internal/pkg/migrator/tern.go b/service/internal/pkg/migrator/tern.go new file mode 100644 index 00000000..493fce6e --- /dev/null +++ b/service/internal/pkg/migrator/tern.go @@ -0,0 +1,164 @@ +package migrator + +import ( + "context" + "io/fs" + "log/slog" + + "github.com/jackc/pgx/v5" + "github.com/jackc/tern/v2/migrate" + + "gitea.privcloud.tech/mtusi2026/internal/config" + "gitea.privcloud.tech/mtusi2026/internal/pkg/apperr" + "gitea.privcloud.tech/mtusi2026/migrations" +) + +type Migrator interface { + MigrateUp(ctx context.Context) error + MigrateDown(ctx context.Context) error + MigrateTo(ctx context.Context, version int32) error + GetVersion(ctx context.Context) (int32, error) + Configure(ctx context.Context) error + Close(ctx context.Context) error + Start(ctx context.Context) error + Stop(ctx context.Context) error +} + +type migrator struct { + cfg *config.Config + conn *pgx.Conn + migrator *migrate.Migrator + migrations fs.FS + configured bool + closed bool +} + +func New(cfg *config.Config) Migrator { + return &migrator{ + cfg: cfg, + conn: nil, + migrator: nil, + migrations: migrations.Migrations, + configured: false, + closed: false, + } +} + +func (m *migrator) MigrateUp(ctx context.Context) error { + if !m.configured { + return apperr.New(apperr.CodeInternal, "migrator was not configured") + } + return m.migrator.Migrate(ctx) +} + +func (m *migrator) MigrateDown(ctx context.Context) error { + if !m.configured { + return apperr.New(apperr.CodeInternal, "migrator was not configured") + } + return m.migrator.MigrateTo(ctx, 0) +} + +func (m *migrator) MigrateTo(ctx context.Context, version int32) error { + if !m.configured { + return apperr.New(apperr.CodeInternal, "migrator was not configured") + } + return m.migrator.MigrateTo(ctx, version) +} + +func (m *migrator) GetVersion(ctx context.Context) (int32, error) { + if !m.configured { + return -1, apperr.New(apperr.CodeInternal, "migrator was not configured") + } + return m.migrator.GetCurrentVersion(ctx) +} + +func (m *migrator) Configure(ctx context.Context) error { + if err := m.initConn(ctx); err != nil { + return err + } + if err := m.initMigrator(ctx); err != nil { + return err + } + if err := m.loadMigrations(); err != nil { + return err + } + m.configured = true + return nil +} + +func (m *migrator) Close(ctx context.Context) error { + if m.conn == nil || m.closed { + return nil + } + if err := m.closeConn(ctx); err != nil { + return err + } + m.closed = true + return nil +} + +func (m *migrator) Start(ctx context.Context) error { + if !m.cfg.Postgres.GetRunMigrations() { + slog.Info("flag run migrations is disabled: skipping migrating") + return nil + } + + err := m.Configure(ctx) + if err != nil { + slog.Error("failed to configure migrator", "error", err) + return err + } + err = m.MigrateUp(ctx) + if err != nil { + slog.Error("failed to migrate database up", "error", err) + return err + } + + if err = m.Close(ctx); err != nil { + slog.Error("failed close connection to database", "error", err) + return err + } + + m.closed = true + + return nil +} + +func (m *migrator) Stop(ctx context.Context) error { + return m.Close(ctx) +} + +func (m *migrator) loadMigrations() (err error) { + err = m.migrator.LoadMigrations(m.migrations) + if err != nil { + slog.Error("failed to load migrations", "error", err) + return apperr.New(apperr.CodeInternal, "failed to load migrations") + } + slog.Info("successfully loaded migrations", "error", err) + return nil +} + +func (m *migrator) initConn(ctx context.Context) (err error) { + m.closed = false + m.conn, err = pgx.Connect(ctx, m.cfg.Postgres.GetURI()) + if err != nil { + slog.Error("failed to create connection to database", "error", err) + return apperr.New(apperr.CodeInternal, "failed to create connection to database") + } + slog.Info("connected to database", "db", m.cfg.Postgres.Database) + return nil +} + +func (m *migrator) initMigrator(ctx context.Context) (err error) { + m.migrator, err = migrate.NewMigrator(ctx, m.conn, m.cfg.Postgres.GetVersionTableName()) + if err != nil { + slog.Error("failed to initialize migrator", "error", err) + return apperr.New(apperr.CodeInternal, "failed to initialize migrator") + } + slog.Info("successfully initialized migrator") + return nil +} + +func (m *migrator) closeConn(ctx context.Context) error { + return m.conn.Close(ctx) +} diff --git a/service/internal/pkg/objectstore/s3.go b/service/internal/pkg/objectstore/s3.go new file mode 100644 index 00000000..5f90f281 --- /dev/null +++ b/service/internal/pkg/objectstore/s3.go @@ -0,0 +1,69 @@ +package objectstore + +import ( + "context" + "errors" + "io" + "net/http" + "time" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/credentials" + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/aws/smithy-go" + + "gitea.privcloud.tech/mtusi2026/internal/config" +) + +type Store struct { + Client *s3.Client + Bucket string +} + +func New(c *config.Attachment) *Store { + return &Store{ + Bucket: c.Bucket, + Client: s3.New( + s3.Options{ + Region: c.Region, + BaseEndpoint: aws.String(c.Endpoint), + UsePathStyle: true, + Credentials: credentials.NewStaticCredentialsProvider( + c.AccessKey, + c.SecretKey, + "", + ), + RetryMaxAttempts: 1, + HTTPClient: &http.Client{Timeout: 30 * time.Second}, + }, + ), + } +} + +func (s *Store) Put(ctx context.Context, key, mime string, r io.Reader, size int64) error { + _, err := s.Client.PutObject(ctx, &s3.PutObjectInput{Bucket: &s.Bucket, Key: &key, Body: r, ContentLength: &size, ContentType: &mime}) + return err +} + +func (s *Store) Get(ctx context.Context, key string) (*s3.GetObjectOutput, error) { + return s.Client.GetObject(ctx, &s3.GetObjectInput{Bucket: &s.Bucket, Key: &key}) +} + +func (s *Store) Head(ctx context.Context, key string) (*s3.HeadObjectOutput, error) { + return s.Client.HeadObject(ctx, &s3.HeadObjectInput{Bucket: &s.Bucket, Key: &key}) +} + +func (s *Store) Delete(ctx context.Context, key string) error { + _, e := s.Client.DeleteObject(ctx, &s3.DeleteObjectInput{Bucket: &s.Bucket, Key: &key}) + return e +} + +func (s *Store) Ping(ctx context.Context) error { + _, e := s.Client.HeadBucket(ctx, &s3.HeadBucketInput{Bucket: &s.Bucket}) + return e +} + +func Missing(err error) bool { + var api smithy.APIError + return errors.As(err, &api) && (api.ErrorCode() == "NoSuchKey" || api.ErrorCode() == "NotFound" || api.ErrorCode() == "404") +} diff --git a/service/internal/pkg/postgres/after-connect.go b/service/internal/pkg/postgres/after-connect.go new file mode 100644 index 00000000..eafcc631 --- /dev/null +++ b/service/internal/pkg/postgres/after-connect.go @@ -0,0 +1,25 @@ +package postgres + +import ( + "context" + + "github.com/jackc/pgx/v5" + pgxUUID "github.com/vgarvardt/pgx-google-uuid/v5" +) + +type AfterConnect func(ctx context.Context, conn *pgx.Conn) error + +func AddUUIDSupport(_ context.Context, conn *pgx.Conn) error { + pgxUUID.Register(conn.TypeMap()) + return nil +} +func WithEnumTypeSupport(enumName string) AfterConnect { + return func(ctx context.Context, conn *pgx.Conn) error { + dt, err := conn.LoadType(ctx, enumName) + if err != nil { + return err + } + conn.TypeMap().RegisterType(dt) + return nil + } +} diff --git a/service/internal/pkg/postgres/config.go b/service/internal/pkg/postgres/config.go new file mode 100644 index 00000000..b0df40a5 --- /dev/null +++ b/service/internal/pkg/postgres/config.go @@ -0,0 +1,7 @@ +package postgres + +type MigrationConfig interface { + GetURI() string + GetVersionTableName() string + GetRunMigrations() bool +} diff --git a/service/internal/pkg/postgres/err.go b/service/internal/pkg/postgres/err.go new file mode 100644 index 00000000..4380b247 --- /dev/null +++ b/service/internal/pkg/postgres/err.go @@ -0,0 +1,10 @@ +package postgres + +import ( + "errors" +) + +var ( + errCreatingPool = errors.New("failed to create postgres pool") + errParseConfig = errors.New("failed to parse postgres config") +) diff --git a/service/internal/pkg/postgres/interfaces.go b/service/internal/pkg/postgres/interfaces.go new file mode 100644 index 00000000..6afe65f4 --- /dev/null +++ b/service/internal/pkg/postgres/interfaces.go @@ -0,0 +1,15 @@ +package postgres + +import ( + "context" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgconn" +) + +type Conn interface { + SendBatch(ctx context.Context, b *pgx.Batch) pgx.BatchResults + Exec(ctx context.Context, sql string, arguments ...any) (commandTag pgconn.CommandTag, err error) + Query(ctx context.Context, sql string, args ...any) (pgx.Rows, error) + QueryRow(ctx context.Context, sql string, args ...any) pgx.Row +} diff --git a/service/internal/pkg/postgres/postgres.go b/service/internal/pkg/postgres/postgres.go new file mode 100644 index 00000000..cdcb6e58 --- /dev/null +++ b/service/internal/pkg/postgres/postgres.go @@ -0,0 +1,95 @@ +package postgres + +import ( + "context" + "log/slog" + "strconv" + "time" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" + "github.com/jackc/pgx/v5/tracelog" + + pgxslog "github.com/mcosta74/pgx-slog" +) + +type ConfigWithLog interface { + GetLogLevel() string +} + +type ConfigWithTimeouts interface { + GetConnectTimeout() time.Duration + GetQueryTimeout() time.Duration +} + +type ConfigWithMaxConn interface { + GetMaxConnections() int32 +} + +func getLevelFromConfig(cfg MigrationConfig) tracelog.LogLevel { + if logCfg, ok := cfg.(ConfigWithLog); ok { + lvl, _ := tracelog.LogLevelFromString(logCfg.GetLogLevel()) + return lvl + } + return tracelog.LogLevelError +} + +func addMaxConnectionsToCfg(cfg any, poolCfg *pgxpool.Config) { + if connCfg, ok := cfg.(ConfigWithMaxConn); ok { + poolCfg.MaxConns = connCfg.GetMaxConnections() + } +} + +func New(cfg MigrationConfig, afterConnect ...AfterConnect) *pgxpool.Pool { + lvl := getLevelFromConfig(cfg) + + if len(afterConnect) == 0 { + afterConnect = []AfterConnect{AddUUIDSupport} + } + + poolConfig, err := pgxpool.ParseConfig(cfg.GetURI()) + if err != nil { + slog.Error("failed to parse postgres config", "error", err) + panic(err) + } + + addMaxConnectionsToCfg(cfg, poolConfig) + if timeoutCfg, ok := cfg.(ConfigWithTimeouts); ok { + poolConfig.ConnConfig.ConnectTimeout = timeoutCfg.GetConnectTimeout() + poolConfig.ConnConfig.RuntimeParams["statement_timeout"] = strconv.FormatInt(timeoutCfg.GetQueryTimeout().Milliseconds(), 10) + } + poolConfig.ConnConfig.RuntimeParams["application_name"] = "mtusi-api" + slog.Info("postgres_pool_config", "max_conns", poolConfig.MaxConns, + "connect_timeout", poolConfig.ConnConfig.ConnectTimeout, + "statement_timeout_ms", poolConfig.ConnConfig.RuntimeParams["statement_timeout"]) + + poolConfig.ConnConfig.Tracer = &tracelog.TraceLog{ + Logger: pgxslog.NewLogger(slog.Default()), + LogLevel: lvl, + Config: &tracelog.TraceLogConfig{ + TimeKey: "ts", + }, + } + + poolConfig.AfterConnect = load(afterConnect...) + + pool, err := pgxpool.NewWithConfig(context.Background(), poolConfig) + if err != nil { + slog.Error("failed to create postgres pool", "error", err) + panic(err) + } + + slog.Info("successfully connected to postgres pool") + return pool +} + +func load(afterConnects ...AfterConnect) func(ctx context.Context, conn *pgx.Conn) error { + return func(ctx context.Context, conn *pgx.Conn) error { + for _, afterConnect := range afterConnects { + if err := afterConnect(ctx, conn); err != nil { + return err + } + } + return nil + } +} diff --git a/service/internal/pkg/postgres/txmanager/tx.go b/service/internal/pkg/postgres/txmanager/tx.go new file mode 100644 index 00000000..c4aa2fe1 --- /dev/null +++ b/service/internal/pkg/postgres/txmanager/tx.go @@ -0,0 +1,59 @@ +package txmanager + +import ( + "context" + "errors" + "fmt" + "log/slog" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" +) + +type TransactionManager struct { + pool *pgxpool.Pool +} + +func New(pool *pgxpool.Pool) *TransactionManager { + return &TransactionManager{ + pool: pool, + } +} + +func (tm *TransactionManager) RunInTransaction(ctx context.Context, opts *pgx.TxOptions, fn func(tx pgx.Tx) error) (err error) { + var tx pgx.Tx + + if opts != nil { + tx, err = tm.pool.BeginTx(ctx, *opts) + } else { + tx, err = tm.pool.Begin(ctx) + } + if err != nil { + return fmt.Errorf("failed to begin transaction: %w", err) + } + + var committed bool + + defer func() { + if p := recover(); p != nil { + err = fmt.Errorf("panic in transaction: %v", p) + } + if !committed { + rollbackErr := tx.Rollback(ctx) + if rollbackErr != nil && !errors.Is(rollbackErr, pgx.ErrTxClosed) { + slog.Error("failed to rollback transaction", "error", rollbackErr) + } + } + }() + + if err = fn(tx); err != nil { + return fmt.Errorf("failed to execute transaction: %w", err) + } + + if err = tx.Commit(ctx); err != nil { + return fmt.Errorf("failed to commit transaction: %w", err) + } + + committed = true + return nil +} diff --git a/service/internal/pkg/rest/framework/framework.go b/service/internal/pkg/rest/framework/framework.go new file mode 100644 index 00000000..0b976c85 --- /dev/null +++ b/service/internal/pkg/rest/framework/framework.go @@ -0,0 +1,257 @@ +package framework + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "log/slog" + "mime" + "net/http" + "strconv" + "strings" + + "github.com/go-chi/chi/v5" + v10 "github.com/go-playground/validator/v10" + "github.com/google/uuid" + + "gitea.privcloud.tech/mtusi2026/internal/dto" + "gitea.privcloud.tech/mtusi2026/internal/gen/contracts" + "gitea.privcloud.tech/mtusi2026/internal/pkg/apperr" + "gitea.privcloud.tech/mtusi2026/internal/pkg/auth/jwtprovider" + "gitea.privcloud.tech/mtusi2026/internal/pkg/rest/headers" + "gitea.privcloud.tech/mtusi2026/internal/pkg/validator" +) + +func validate(ctx context.Context, v *validator.ValidatorImpl, value any) error { + err := v.ValidateCtx(ctx, value) + if err == nil { + return nil + } + var fieldErrors v10.ValidationErrors + if errors.As(err, &fieldErrors) { + return apperr.Wrap(err, apperr.CodeValidation, "invalid request fields") + } + return apperr.Wrap(err, apperr.CodeInternal, "validation failed") +} + +func Validate(w http.ResponseWriter, r *http.Request, v *validator.ValidatorImpl, value any) bool { + if err := validate(r.Context(), v, value); err != nil { + JSONError(r.Context(), w, err) + return false + } + return true +} + +func JSONResponse(w http.ResponseWriter, status int, body any) { + w.Header().Set(headers.HeaderContentType, "application/json") + w.WriteHeader(status) + + if status == http.StatusNoContent { + return + } + + if err := json.NewEncoder(w).Encode(body); err != nil { + wrapped := apperr.Wrap(err, apperr.CodeInternal, "failed json encode") + slog.Error("response", "error", wrapped) + } +} + +func JSONError(ctx context.Context, w http.ResponseWriter, err error) { + if err == nil { + return + } + + requestId, ok := ctx.Value(headers.HeaderRequestID).(uuid.UUID) + if !ok { + requestId = uuid.New() + } + w.Header().Set(headers.HeaderRequestID, requestId.String()) + + if errors.Is(err, context.DeadlineExceeded) { + err = apperr.Wrap(err, apperr.CodeServiceUnavailable, "request deadline exceeded") + } + appErr := apperr.As(err) + if appErr == nil { + appErr = apperr.As(apperr.New(apperr.CodeInternal, "internal server error")) + } + if appErr.Code().HTTPStatus() == http.StatusUnauthorized && w.Header().Get("WWW-Authenticate") == "" { + w.Header().Set("WWW-Authenticate", `Bearer realm="messenger"`) + } + + slog.Error("response", "error", err, "status", appErr.Code().HTTPStatus(), "requestId", requestId.String()) + + var code contracts.ErrorErrorCode + switch appErr.Code() { + case apperr.CodeBadRequest, apperr.CodeValidation: + code = contracts.InvalidRequest + case apperr.CodeUnauthorized: + code = contracts.Unauthorized + case apperr.CodeNotFound: + code = contracts.NotFound + case apperr.CodeAlreadyExists, apperr.CodeConflict: + code = contracts.Conflict + case apperr.CodePayloadTooLarge: + code = contracts.PayloadTooLarge + case apperr.CodeTooManyRequests: + code = contracts.RateLimited + case apperr.CodeServiceUnavailable, apperr.CodeUnavailable: + code = contracts.Unavailable + case apperr.CodeUnsupportedMediaType: + code = contracts.UnsupportedMediaType + default: + code = contracts.InternalError + } + JSONResponse(w, appErr.Code().HTTPStatus(), contracts.Error{ + Error: struct { + Code contracts.ErrorErrorCode `json:"code"` + Message string `json:"message"` + RequestId string `json:"request_id"` + }{ + Code: code, + Message: appErr.Message(), + RequestId: requestId.String(), + }, + }) +} + +func Decode(w http.ResponseWriter, r *http.Request, object any) bool { + mediaType, params, err := mime.ParseMediaType(r.Header.Get(headers.HeaderContentType)) + if err != nil || mediaType != "application/json" || + (params["charset"] != "" && !strings.EqualFold(params["charset"], "utf-8")) { + JSONError(r.Context(), w, apperr.New(apperr.CodeUnsupportedMediaType, "expected application/json")) + return false + } + decoder := json.NewDecoder(r.Body) + decoder.DisallowUnknownFields() + if err := decoder.Decode(object); err != nil { + JSONError(r.Context(), w, apperr.Wrap(err, apperr.CodeBadRequest, "failed decode")) + return false + } + var trailing any + if err := decoder.Decode(&trailing); err != io.EOF { + JSONError(r.Context(), w, apperr.New(apperr.CodeBadRequest, "expected one JSON object")) + return false + } + return true +} + +func PathParam(r *http.Request, key string) string { + return chi.URLParam(r, key) +} + +func QueryParam(r *http.Request, key string) string { + return r.URL.Query().Get(key) +} + +func HeaderParam(r *http.Request, key string) string { + return r.Header.Get(key) +} + +func ParseIDFromPathParam(ctx context.Context, w http.ResponseWriter, r *http.Request, key string) (uuid.UUID, error) { + rawID := PathParam(r, key) + id, err := uuid.Parse(rawID) + if err != nil { + JSONError(ctx, w, apperr.New(apperr.CodeValidation, fmt.Sprintf("provided invalid %s", key))) + return uuid.Nil, err + } + return id, nil +} + +func ParseIDFromQueryParam(ctx context.Context, w http.ResponseWriter, r *http.Request, key string) (uuid.UUID, error) { + rawID := QueryParam(r, key) + id, err := uuid.Parse(rawID) + if err != nil { + JSONError(ctx, w, apperr.New(apperr.CodeValidation, fmt.Sprintf("provided invalid %s", key))) + return uuid.Nil, err + } + return id, nil +} + +func ParseIDFromHeaderParam(ctx context.Context, w http.ResponseWriter, r *http.Request, key string) (uuid.UUID, error) { + rawID := HeaderParam(r, key) + id, err := uuid.Parse(rawID) + if err != nil { + JSONError(ctx, w, apperr.New(apperr.CodeUnauthorized, "missing or invalid user ID")) + return uuid.Nil, err + } + return id, nil +} + +func Header(r *http.Request, header string) (string, bool) { + return r.Header.Get(header), r.Header.Get(header) != "" +} + +func Token(r *http.Request) (string, bool) { + foundHeader := r.Header.Get(headers.HeaderAuthorization) + if foundHeader == "" { + return "", false + } + + token, found := strings.CutPrefix(foundHeader, "Bearer ") + if !found { + return "", false + } + + return token, true +} + +type sessionContextKey struct{} + +func WithVerifiedSession(r *http.Request, data *dto.UserDataInToken) *http.Request { + return r.WithContext(context.WithValue(r.Context(), sessionContextKey{}, *data)) +} + +func ParseToken(r *http.Request, provider jwtprovider.Provider) (*dto.UserDataInToken, error) { + if data, ok := r.Context().Value(sessionContextKey{}).(dto.UserDataInToken); ok { + return &data, nil + } + foundHeader := r.Header.Get(headers.HeaderAuthorization) + if foundHeader == "" { + return nil, apperr.New(apperr.CodeUnauthorized, "missing authorization header") + } + + token, found := strings.CutPrefix(foundHeader, "Bearer ") + if !found { + return nil, apperr.New(apperr.CodeUnauthorized, "invalid authorization header") + } + + userData, err := provider.ParseVerified(&dto.Token{Payload: token}) + if err != nil { + return nil, apperr.Wrap(err, apperr.CodeUnauthorized, "invalid token") + } + + return userData, nil +} + +func PageParams(w http.ResponseWriter, r *http.Request) (*int, *uuid.UUID, bool) { + query := r.URL.Query() + var limit *int + if values, exists := query["limit"]; exists { + if len(values) != 1 { + JSONError(r.Context(), w, apperr.New(apperr.CodeBadRequest, "invalid limit")) + return nil, nil, false + } + value, err := strconv.Atoi(values[0]) + if err != nil || value < 1 || value > 100 { + JSONError(r.Context(), w, apperr.New(apperr.CodeBadRequest, "invalid limit")) + return nil, nil, false + } + limit = &value + } + var afterID *uuid.UUID + if values, exists := query["cursor"]; exists { + if len(values) != 1 { + JSONError(r.Context(), w, apperr.New(apperr.CodeBadRequest, "invalid cursor")) + return nil, nil, false + } + id, err := uuid.Parse(values[0]) + if err != nil || id == uuid.Nil { + JSONError(r.Context(), w, apperr.New(apperr.CodeBadRequest, "invalid cursor")) + return nil, nil, false + } + afterID = &id + } + return limit, afterID, true +} diff --git a/service/internal/pkg/rest/headers/headers.go b/service/internal/pkg/rest/headers/headers.go new file mode 100644 index 00000000..c92a8dee --- /dev/null +++ b/service/internal/pkg/rest/headers/headers.go @@ -0,0 +1,8 @@ +package headers + +const ( + HeaderContentType = "Content-Type" + HeaderRequestID = "X-Request-Id" + HeaderUserID = "X-User-Id" + HeaderAuthorization = "Authorization" +) diff --git a/service/internal/pkg/rest/middleware/auth.go b/service/internal/pkg/rest/middleware/auth.go new file mode 100644 index 00000000..a39c053b --- /dev/null +++ b/service/internal/pkg/rest/middleware/auth.go @@ -0,0 +1,75 @@ +package middleware + +import ( + "context" + "net/http" + + "gitea.privcloud.tech/mtusi2026/internal/dto" + "gitea.privcloud.tech/mtusi2026/internal/pkg/apperr" + "gitea.privcloud.tech/mtusi2026/internal/pkg/rest/framework" + "gitea.privcloud.tech/mtusi2026/internal/pkg/types/role" +) + +var _ AuthMiddleware = (*authMiddleware)(nil) + +type AuthMiddleware interface { + WithPrivateAccess(next http.Handler) http.Handler + WithAdminAccess(next http.Handler) http.Handler +} + +type SessionVerifier interface { + VerifySession(ctx context.Context, token dto.Token) (*dto.UserDataInToken, error) +} + +type authMiddleware struct { + verifier SessionVerifier +} + +func NewAuthMiddleware(verifier SessionVerifier) AuthMiddleware { + return &authMiddleware{verifier: verifier} +} + +func (a *authMiddleware) WithPrivateAccess(next http.Handler) http.Handler { + return a.privateAccessCheckCallback(next) +} + +func (a *authMiddleware) WithAdminAccess(next http.Handler) http.Handler { + return a.operatorAccessCheckCallback(role.Admin, next) +} + +func (a *authMiddleware) privateAccessCheckCallback(next http.Handler) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + data, err := a.verifySession(r) + if err != nil { + framework.JSONError(r.Context(), w, err) + return + } + + next.ServeHTTP(w, framework.WithVerifiedSession(r, data)) + } +} + +func (a *authMiddleware) operatorAccessCheckCallback(expectedRole role.Role, next http.Handler) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + u, err := a.verifySession(r) + if err != nil { + framework.JSONError(r.Context(), w, err) + return + } + + if u.Role != role.Admin && u.Role != expectedRole { + framework.JSONError(r.Context(), w, apperr.New(apperr.CodeForbidden, "insufficient permissions")) + return + } + + next.ServeHTTP(w, framework.WithVerifiedSession(r, u)) + } +} + +func (a *authMiddleware) verifySession(r *http.Request) (*dto.UserDataInToken, error) { + token, found := framework.Token(r) + if !found || token == "" { + return nil, apperr.New(apperr.CodeUnauthorized, "missing or invalid bearer token") + } + return a.verifier.VerifySession(r.Context(), dto.Token{Payload: token}) +} diff --git a/service/internal/pkg/rest/middleware/auth_test.go b/service/internal/pkg/rest/middleware/auth_test.go new file mode 100644 index 00000000..b62e5d89 --- /dev/null +++ b/service/internal/pkg/rest/middleware/auth_test.go @@ -0,0 +1,73 @@ +package middleware + +import ( + "context" + "errors" + "net/http" + "net/http/httptest" + "testing" + + "gitea.privcloud.tech/mtusi2026/internal/dto" + "gitea.privcloud.tech/mtusi2026/internal/pkg/apperr" + "gitea.privcloud.tech/mtusi2026/internal/pkg/rest/framework" + "github.com/google/uuid" +) + +type sessionVerifier struct { + calls int + data dto.UserDataInToken + err error +} + +func (v *sessionVerifier) VerifySession(context.Context, dto.Token) (*dto.UserDataInToken, error) { + v.calls++ + return &v.data, v.err +} + +func TestVerifiedSessionIsRequestScoped(t *testing.T) { + verifier := &sessionVerifier{data: dto.UserDataInToken{UserID: uuid.New()}} + middleware := NewAuthMiddleware(verifier) + handled := 0 + handler := middleware.WithPrivateAccess(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // A nil provider is intentional: the handler must use the verified request identity. + data, err := framework.ParseToken(r, nil) + if err != nil { + t.Fatal(err) + } + if data.UserID != verifier.data.UserID { + t.Fatal("verified identity changed") + } + handled++ + w.WriteHeader(http.StatusNoContent) + })) + for range 2 { + request := httptest.NewRequest("GET", "/private", nil) + request.Header.Set("Authorization", "Bearer synthetic-token") + recorder := httptest.NewRecorder() + handler.ServeHTTP(recorder, request) + if recorder.Code != 204 { + t.Fatal(recorder.Code) + } + } + if verifier.calls != 2 || handled != 2 { + t.Fatal("session verification must run for every request") + } + verifier.err = apperr.New(apperr.CodeUnauthorized, "revoked session") + request := httptest.NewRequest("GET", "/private", nil) + request.Header.Set("Authorization", "Bearer synthetic-token") + recorder := httptest.NewRecorder() + handler.ServeHTTP(recorder, request) + if recorder.Code != 401 || handled != 2 { + t.Fatal("revocation was bypassed by cached request identity") + } +} + +func TestMissingTokenDoesNotCallSessionStore(t *testing.T) { + verifier := &sessionVerifier{err: errors.New("must not be called")} + handler := NewAuthMiddleware(verifier).WithPrivateAccess(http.HandlerFunc(func(http.ResponseWriter, *http.Request) { t.Fatal("handler called without a token") })) + recorder := httptest.NewRecorder() + handler.ServeHTTP(recorder, httptest.NewRequest("GET", "/private", nil)) + if recorder.Code != 401 || verifier.calls != 0 { + t.Fatal("missing token was accepted") + } +} diff --git a/service/internal/pkg/rest/middleware/instance.go b/service/internal/pkg/rest/middleware/instance.go new file mode 100644 index 00000000..64477054 --- /dev/null +++ b/service/internal/pkg/rest/middleware/instance.go @@ -0,0 +1,12 @@ +package middleware + +import "net/http" + +func Instance(id string) Middleware { + return func(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("X-Instance-Id", id) + next.ServeHTTP(w, r) + }) + } +} diff --git a/service/internal/pkg/rest/middleware/logging.go b/service/internal/pkg/rest/middleware/logging.go new file mode 100644 index 00000000..8ffda7a0 --- /dev/null +++ b/service/internal/pkg/rest/middleware/logging.go @@ -0,0 +1,34 @@ +package middleware + +import ( + "log/slog" + "net/http" + "time" + + "github.com/go-chi/chi/v5/middleware" + + "gitea.privcloud.tech/mtusi2026/internal/pkg/rest/headers" +) + +func LoggingMiddleware() Middleware { + return func(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + started := time.Now() + wrapped := middleware.NewWrapResponseWriter(w, r.ProtoMajor) + next.ServeHTTP(wrapped, r) + + status := wrapped.Status() + if status == 0 { + status = http.StatusOK + } + slog.InfoContext(r.Context(), "http request", + "method", r.Method, + "path", r.URL.Path, + "status", status, + "duration", time.Since(started), + "bytes", wrapped.BytesWritten(), + "request_id", w.Header().Get(headers.HeaderRequestID), + ) + }) + } +} diff --git a/service/internal/pkg/rest/middleware/middleware.go b/service/internal/pkg/rest/middleware/middleware.go new file mode 100644 index 00000000..f54cea97 --- /dev/null +++ b/service/internal/pkg/rest/middleware/middleware.go @@ -0,0 +1,5 @@ +package middleware + +import "net/http" + +type Middleware = func(http.Handler) http.Handler diff --git a/service/internal/pkg/rest/middleware/request-id.go b/service/internal/pkg/rest/middleware/request-id.go new file mode 100644 index 00000000..eed35498 --- /dev/null +++ b/service/internal/pkg/rest/middleware/request-id.go @@ -0,0 +1,22 @@ +package middleware + +import ( + "context" + "net/http" + + "github.com/google/uuid" + + "gitea.privcloud.tech/mtusi2026/internal/pkg/rest/headers" +) + +func RequestIDMiddleware() Middleware { + return func(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + requestID := uuid.New() + w.Header().Set(headers.HeaderRequestID, requestID.String()) + ctx = context.WithValue(ctx, headers.HeaderRequestID, requestID) + next.ServeHTTP(w, r.WithContext(ctx)) + }) + } +} diff --git a/service/internal/pkg/rest/middleware/timeout.go b/service/internal/pkg/rest/middleware/timeout.go new file mode 100644 index 00000000..75421474 --- /dev/null +++ b/service/internal/pkg/rest/middleware/timeout.go @@ -0,0 +1,23 @@ +package middleware + +import ( + "context" + "net/http" + "strings" + "time" +) + +// RequestTimeout bounds database work and waiting for a pooled connection. +func RequestTimeout(timeout time.Duration, attachmentTimeout ...time.Duration) Middleware { + return func(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestTimeout := timeout + if len(attachmentTimeout) > 0 && strings.Contains(r.URL.Path, "/attachments") { + requestTimeout = attachmentTimeout[0] + } + ctx, cancel := context.WithTimeout(r.Context(), requestTimeout) + defer cancel() + next.ServeHTTP(w, r.WithContext(ctx)) + }) + } +} diff --git a/service/internal/pkg/rest/router/router.go b/service/internal/pkg/rest/router/router.go new file mode 100644 index 00000000..201817e2 --- /dev/null +++ b/service/internal/pkg/rest/router/router.go @@ -0,0 +1,131 @@ +package router + +import ( + "net/http" + + "github.com/go-chi/chi/v5" + + "gitea.privcloud.tech/mtusi2026/internal/pkg/rest/middleware" +) + +type Router interface { + Private(method, pattern string, handler http.HandlerFunc, mw ...func(http.Handler) http.Handler) + Public(method, pattern string, handler http.HandlerFunc, mw ...func(http.Handler) http.Handler) + Admin(method, pattern string, handler http.HandlerFunc, mw ...func(http.Handler) http.Handler) + ServeHTTP(w http.ResponseWriter, r *http.Request) + With(mw ...func(http.Handler) http.Handler) Router + Use(mw ...func(http.Handler) http.Handler) Router + Handle(pattern string, handler http.Handler) + Get(pattern string, handler http.HandlerFunc) + Head(pattern string, handler http.HandlerFunc) + Options(pattern string, handler http.HandlerFunc) + Post(pattern string, handler http.HandlerFunc) + Put(pattern string, handler http.HandlerFunc) + Patch(pattern string, handler http.HandlerFunc) + Delete(pattern string, handler http.HandlerFunc) + Connect(pattern string, handler http.HandlerFunc) + Trace(pattern string, handler http.HandlerFunc) +} + +type router struct { + r chi.Router + jwtMw middleware.AuthMiddleware +} + +func New(jwtMw middleware.AuthMiddleware, mw ...middleware.Middleware) Router { + r := chi.NewRouter() + r.Use(mw...) + return &router{ + r: r, + jwtMw: jwtMw, + } +} + +func (c *router) Group(pattern string) Router { + sub := &router{ + r: chi.NewRouter(), + jwtMw: c.jwtMw, + } + c.r.Mount(pattern, sub.r) + return sub +} + +func (c *router) Public(method, pattern string, handler http.HandlerFunc, mw ...func(http.Handler) http.Handler) { + next := c.applyMiddlewares(handler, mw...) + c.r.MethodFunc(method, pattern, next.ServeHTTP) +} + +func (c *router) Private(method, pattern string, handler http.HandlerFunc, mw ...func(http.Handler) http.Handler) { + next := c.applyMiddlewares(handler, mw...) + c.r.MethodFunc(method, pattern, (c.jwtMw.WithPrivateAccess(next)).ServeHTTP) +} + +func (c *router) Admin(method, pattern string, handler http.HandlerFunc, mw ...func(http.Handler) http.Handler) { + next := c.applyMiddlewares(handler, mw...) + c.r.MethodFunc(method, pattern, (c.jwtMw.WithAdminAccess(next)).ServeHTTP) +} + +func (c *router) ServeHTTP(w http.ResponseWriter, r *http.Request) { + c.r.ServeHTTP(w, r) +} + +func (c *router) With(mw ...func(http.Handler) http.Handler) Router { + sub := &router{ + r: c.r.With(mw...), + jwtMw: c.jwtMw, + } + return sub +} + +func (c *router) Use(mw ...func(http.Handler) http.Handler) Router { + c.r.Use(mw...) + return c +} + +func (c *router) Handle(pattern string, handler http.Handler) { + c.r.Handle(pattern, handler) +} + +func (c *router) Get(pattern string, handler http.HandlerFunc) { + c.r.Get(pattern, handler) +} + +func (c *router) Head(pattern string, handler http.HandlerFunc) { + c.r.Head(pattern, handler) +} + +func (c *router) Options(pattern string, handler http.HandlerFunc) { + c.r.Options(pattern, handler) +} + +func (c *router) Post(pattern string, handler http.HandlerFunc) { + c.r.Post(pattern, handler) +} + +func (c *router) Put(pattern string, handler http.HandlerFunc) { + c.r.Put(pattern, handler) +} + +func (c *router) Patch(pattern string, handler http.HandlerFunc) { + c.r.Patch(pattern, handler) +} + +func (c *router) Delete(pattern string, handler http.HandlerFunc) { + c.r.Delete(pattern, handler) +} + +func (c *router) Connect(pattern string, handler http.HandlerFunc) { + c.r.Connect(pattern, handler) +} + +func (c *router) Trace(pattern string, handler http.HandlerFunc) { + c.r.Trace(pattern, handler) +} + +func (c *router) applyMiddlewares(handler http.HandlerFunc, mw ...func(http.Handler) http.Handler) http.Handler { + next := http.Handler(handler) + for _, mwFunc := range mw { + next = mwFunc(next) + } + return next +} diff --git a/service/internal/pkg/types/authprovider/provider.go b/service/internal/pkg/types/authprovider/provider.go new file mode 100644 index 00000000..b05721c4 --- /dev/null +++ b/service/internal/pkg/types/authprovider/provider.go @@ -0,0 +1,82 @@ +package authprovider + +import ( + "encoding/json" + "strings" + + "gitea.privcloud.tech/mtusi2026/internal/pkg/types" +) + +type Type string + +const ( + TypeUndefined Type = "undefined" + TypePassword Type = "password" + TypeEmail Type = "email" + TypePhone Type = "phone" + TypeGoogle Type = "google" + TypeTelegram Type = "telegram" +) + +var allowedTypes = map[Type]struct{}{ + TypePassword: {}, + TypeEmail: {}, + TypePhone: {}, + TypeGoogle: {}, + TypeTelegram: {}, +} + +func ParseType(value string) Type { + t := Type(strings.ToLower(value)) + if !t.IsValid() { + return TypeUndefined + } + return t +} + +func ParseAndValidateType(value string) (Type, error) { + parsed := ParseType(value) + if _, ok := allowedTypes[parsed]; ok { + return parsed, nil + } + return TypeUndefined, types.ErrUnsupportedOTPStatus +} + +func (t Type) String() string { + return string(t) +} + +func (t Type) Validate() error { + if _, ok := allowedTypes[t]; ok { + return nil + } + return types.ErrUnsupportedAuthProviderType +} + +func (t Type) IsValid() bool { + _, ok := allowedTypes[t] + return ok +} + +func (t Type) MarshalJSON() ([]byte, error) { + if err := t.Validate(); err != nil { + return nil, err + } + return json.Marshal(t.String()) +} + +func (t *Type) UnmarshalJSON(data []byte) error { + if t == nil { + return types.ErrUnmarshalAuthProviderTypeIntoNilPointer + } + var value string + if err := json.Unmarshal(data, &value); err != nil { + return err + } + parsed := ParseType(value) + if err := parsed.Validate(); err != nil { + return err + } + *t = parsed + return nil +} diff --git a/service/internal/pkg/types/environment/environment.go b/service/internal/pkg/types/environment/environment.go new file mode 100644 index 00000000..c1bbd5f7 --- /dev/null +++ b/service/internal/pkg/types/environment/environment.go @@ -0,0 +1,77 @@ +package environment + +import ( + "encoding/json" + "strings" + + "gitea.privcloud.tech/mtusi2026/internal/pkg/types" +) + +type ApplicationEnvironment string + +const ( + ApplicationEnvironmentUndefined ApplicationEnvironment = "undefined" + ApplicationEnvironmentDev ApplicationEnvironment = "dev" + ApplicationEnvironmentProd ApplicationEnvironment = "prod" +) + +var allowedEnvironments = map[ApplicationEnvironment]struct{}{ + ApplicationEnvironmentUndefined: {}, + ApplicationEnvironmentDev: {}, + ApplicationEnvironmentProd: {}, +} + +func ParseEnvironment(value string) ApplicationEnvironment { + s := ApplicationEnvironment(strings.ToLower(value)) + if !s.IsValid() { + return ApplicationEnvironmentUndefined + } + return s +} + +func ParseAndValidateEnvironment(status string) (ApplicationEnvironment, error) { + parsed := ParseEnvironment(status) + if _, ok := allowedEnvironments[parsed]; ok { + return parsed, nil + } + return ApplicationEnvironmentUndefined, types.ErrUnsupportedApplicationEnvironment +} + +func (t ApplicationEnvironment) String() string { + return string(t) +} + +func (t ApplicationEnvironment) Validate() error { + if _, ok := allowedEnvironments[t]; ok { + return nil + } + return types.ErrUnsupportedApplicationEnvironment +} + +func (t ApplicationEnvironment) IsValid() bool { + _, ok := allowedEnvironments[t] + return ok +} + +func (t ApplicationEnvironment) MarshalJSON() ([]byte, error) { + if err := t.Validate(); err != nil { + return nil, err + } + return json.Marshal(t.String()) +} + +func (t *ApplicationEnvironment) UnmarshalJSON(data []byte) error { + if t == nil { + return types.ErrUnmarshalApplicationEnvironmentIntoNilPointer + } + var value string + if err := json.Unmarshal(data, &value); err != nil { + return err + } + parsed := ParseEnvironment(value) + if err := parsed.Validate(); err != nil { + return err + } + *t = parsed + return nil +} diff --git a/service/internal/pkg/types/error.go b/service/internal/pkg/types/error.go new file mode 100644 index 00000000..5f523576 --- /dev/null +++ b/service/internal/pkg/types/error.go @@ -0,0 +1,25 @@ +package types + +import "errors" + +var ( + ErrUnsupportedRole = errors.New("unsupported role") + ErrUnsupportedOTPType = errors.New("unsupported OTP type") + ErrUnsupportedAuthProviderType = errors.New("unsupported auth provider type") + ErrUnsupportedContactType = errors.New("unsupported contact type") + ErrUnsupportedOTPStatus = errors.New("unsupported OTP status") + ErrUnsupportedOTPResult = errors.New("unsupported OTP result") + ErrUnsupportedFormStatus = errors.New("unsupported form status") + ErrUnsupportedAuthIdentityStatus = errors.New("unsupported auth identity status") + ErrUnsupportedApplicationEnvironment = errors.New("unsupported application environment") + + ErrUnmarshalRoleIntoNilPointer = errors.New("unmarshal role into nil pointer") + ErrUnmarshalOTPTypeIntoNilPointer = errors.New("unmarshal OTP type into nil pointer") + ErrUnmarshalAuthProviderTypeIntoNilPointer = errors.New("unmarshal auth provider into nil pointer") + ErrUnmarshalContactTypeIntoNilPointer = errors.New("unmarshal contact into nil pointer") + ErrUnmarshalOTPStatusIntoNilPointer = errors.New("unmarshal OTP status into nil pointer") + ErrUnmarshalOTPResultIntoNilPointer = errors.New("unmarshal OTP result into nil pointer") + ErrUnmarshalFormStatusIntoNilPointer = errors.New("unmarshal form status into nil pointer") + ErrUnmarshalAuthIdentityStatusIntoNilPointer = errors.New("unmarshal auth identity status into nil pointer") + ErrUnmarshalApplicationEnvironmentIntoNilPointer = errors.New("unmarshal application environment into nil pointer") +) diff --git a/service/internal/pkg/types/role/role.go b/service/internal/pkg/types/role/role.go new file mode 100644 index 00000000..6f79d782 --- /dev/null +++ b/service/internal/pkg/types/role/role.go @@ -0,0 +1,105 @@ +package role + +import ( + "encoding/json" + "strings" + + "gitea.privcloud.tech/mtusi2026/internal/pkg/types" +) + +type Role string + +const ( + Undefined Role = "undefined" + User Role = "user" + Doctor Role = "doctor" + Manager Role = "manager" + Admin Role = "admin" +) + +var allowedRoles = map[Role]struct{}{ + User: {}, + Doctor: {}, + Manager: {}, + Admin: {}, +} + +func ParseRole(s string) Role { + r := Role(strings.ToLower(s)) + if !r.IsValid() { + return Undefined + } + return r +} + +func ParseAndValidateRole(s string) (Role, error) { + parsed := ParseRole(s) + if _, ok := allowedRoles[parsed]; ok { + return parsed, nil + } + return Undefined, types.ErrUnsupportedRole +} + +func (t Role) String() string { + return string(t) +} + +func (t Role) Validate() error { + if _, ok := allowedRoles[t]; ok { + return nil + } + return types.ErrUnsupportedRole +} + +func (t Role) IsValid() bool { + _, ok := allowedRoles[t] + return ok +} + +func (t Role) MarshalJSON() ([]byte, error) { + if err := t.Validate(); err != nil { + return nil, err + } + return json.Marshal(t.String()) +} + +func (t *Role) UnmarshalJSON(data []byte) error { + if t == nil { + return types.ErrUnmarshalRoleIntoNilPointer + } + var value string + if err := json.Unmarshal(data, &value); err != nil { + return err + } + parsed := ParseRole(value) + if err := parsed.Validate(); err != nil { + return err + } + *t = parsed + return nil +} + +func (t Role) IsAdmin() bool { + return t == Admin +} + +func (t Role) IsManager() bool { + return t == Manager +} + +func (t Role) IsDoctor() bool { + return t == Doctor +} + +func (t Role) IsUser() bool { + return t == User +} + +func (t Role) IsPrivileged() bool { + switch t { + case Doctor, Manager, Admin: + return true + default: + return false + } +} diff --git a/service/internal/pkg/validator/validator.go b/service/internal/pkg/validator/validator.go new file mode 100644 index 00000000..d4ab8aa3 --- /dev/null +++ b/service/internal/pkg/validator/validator.go @@ -0,0 +1,29 @@ +package validator + +import ( + "context" + + "github.com/go-playground/validator/v10" +) + +type TagKey string + +type ValidatorImpl struct { + validator *validator.Validate +} + +func NewValidatorImpl() *ValidatorImpl { + return &ValidatorImpl{validator: validator.New()} +} + +func (v *ValidatorImpl) Register(tag TagKey, fn validator.Func, callValidationEvenIfNull ...bool) error { + return v.validator.RegisterValidation(string(tag), fn, callValidationEvenIfNull...) +} + +func (v *ValidatorImpl) ValidateCtx(ctx context.Context, value any) error { + return v.validator.StructCtx(ctx, value) +} + +func (v *ValidatorImpl) Validate(value any) error { + return v.validator.Struct(value) +} diff --git a/service/internal/service/attachment/reconcile.go b/service/internal/service/attachment/reconcile.go new file mode 100644 index 00000000..0a11258a --- /dev/null +++ b/service/internal/service/attachment/reconcile.go @@ -0,0 +1,205 @@ +package attachment + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "fmt" + "io" + "os" + "time" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/google/uuid" + + "gitea.privcloud.tech/mtusi2026/internal/pkg/objectstore" +) + +type ScanResult struct { + Apply bool `json:"apply"` + Objects int `json:"objects"` + Metadata int `json:"metadata"` + Orphans int `json:"orphans"` + Fresh int `json:"fresh_orphans"` + Deleted int `json:"deleted"` + Missing int `json:"missing"` + Marked int `json:"marked_missing"` + Protected int `json:"referenced_objects"` + S3Requests int `json:"s3_requests"` + DBChecks int `json:"db_checks"` + LockAttempts int `json:"lock_attempts"` + DurationMS int64 `json:"duration_ms"` + GraceSeconds float64 `json:"grace_seconds"` +} + +func (s *Service) Reconcile(ctx context.Context, apply bool) (result ScanResult, err error) { + start := time.Now() + result.Apply = apply + result.GraceSeconds = s.Config.Grace.Seconds() + defer func() { result.DurationMS = time.Since(start).Milliseconds() }() + if err = s.Repo.Pool.Ping(ctx); err != nil { + return + } + result.S3Requests++ + if err = s.Store.Ping(ctx); err != nil { + return + } + pager := s3.NewListObjectsV2Paginator(s.Store.Client, &s3.ListObjectsV2Input{Bucket: &s.Store.Bucket, Prefix: aws.String(Prefix), MaxKeys: aws.Int32(100)}) + for pager.HasMorePages() { + result.S3Requests++ + page, e := pager.NextPage(ctx) + if e != nil { + return result, e + } + for _, obj := range page.Contents { + result.Objects++ + key := aws.ToString(obj.Key) + if !ManagedKey(key) { + continue + } + result.LockAttempts++ + conn, unlock, e := s.Repo.Lock(ctx, key) + if e != nil { + return result, e + } + e = func() error { + defer unlock() + result.DBChecks++ + found, e := s.Repo.Exists(ctx, conn, key) + if e != nil { + return e + } + if found { + result.Protected++ + return nil + } + // Re-read both metadata and object state under the same lock as upload. + result.S3Requests++ + head, e := s.Store.Head(ctx, key) + if objectstore.Missing(e) { + return nil + } + if e != nil { + return e + } + result.Orphans++ + if head.LastModified == nil || time.Since(*head.LastModified) <= s.Config.Grace { + result.Fresh++ + return nil + } + if !apply { + return nil + } + result.DBChecks++ + if _, e = conn.Exec(ctx, "SELECT 1"); e != nil { + return e + } + result.S3Requests++ + if e = s.Store.Delete(ctx, key); e != nil { + return e + } + result.Deleted++ + return nil + }() + if e != nil { + return result, e + } + } + } + after := uuid.Nil + for { + result.DBChecks++ + page, e := s.Repo.Page(ctx, after) + if e != nil { + return result, e + } + if len(page) == 0 { + break + } + for _, a := range page { + result.Metadata++ + after = a.Public.Id + result.LockAttempts++ + conn, unlock, e := s.Repo.Lock(ctx, a.Key) + if e != nil { + return result, e + } + e = func() error { + defer unlock() + result.S3Requests++ + _, e := s.Store.Head(ctx, a.Key) + if e == nil { + return nil + } + if !objectstore.Missing(e) { + return e + } + result.Missing++ + if apply { + result.DBChecks++ + changed, err := s.Repo.MarkMissing(ctx, conn, a.Public.Id, true) + if err != nil { + return err + } + if changed { + result.Marked++ + } + } + return nil + }() + if e != nil { + return result, e + } + } + } + return result, nil +} +func (s *Service) Restore(ctx context.Context, id uuid.UUID, path string) error { + a, e := s.Repo.Get(ctx, id) + if e != nil { + return e + } + f, e := os.Open(path) + if e != nil { + return e + } + defer f.Close() + hash := sha256.New() + size, e := io.Copy(hash, io.LimitReader(f, s.Config.MaxBytes+1)) + if e != nil { + return e + } + if size != int64(a.Public.SizeBytes) || hex.EncodeToString(hash.Sum(nil)) != a.Public.Sha256 { + return fmt.Errorf("original checksum or size mismatch") + } + if e = s.Validate(ctx, f, string(a.Public.ContentType)); e != nil { + return e + } + conn, unlock, e := s.Repo.Lock(ctx, a.Key) + if e != nil { + return e + } + defer unlock() + if _, e = f.Seek(0, 0); e != nil { + return e + } + if e = s.Store.Put(ctx, a.Key, string(a.Public.ContentType), f, size); e != nil { + return e + } + obj, e := s.Store.Get(ctx, a.Key) + if e != nil { + return e + } + defer obj.Body.Close() + hash.Reset() + n, e := io.Copy(hash, io.LimitReader(obj.Body, s.Config.MaxBytes+1)) + if e != nil { + return e + } + if n != size || hex.EncodeToString(hash.Sum(nil)) != a.Public.Sha256 { + return fmt.Errorf("stored original verification failed") + } + _, e = s.Repo.MarkMissing(ctx, conn, id, false) + return e +} diff --git a/service/internal/service/attachment/service.go b/service/internal/service/attachment/service.go new file mode 100644 index 00000000..36c5d713 --- /dev/null +++ b/service/internal/service/attachment/service.go @@ -0,0 +1,162 @@ +package attachment + +import ( + "context" + "crypto/sha256" + "fmt" + "image" + _ "image/jpeg" + _ "image/png" + "io" + "mime" + "net/http" + "os" + "strings" + "time" + "unicode" + "unicode/utf8" + + "github.com/google/uuid" + + "gitea.privcloud.tech/mtusi2026/internal/adapter/repository/storage" + "gitea.privcloud.tech/mtusi2026/internal/config" + "gitea.privcloud.tech/mtusi2026/internal/dto" + "gitea.privcloud.tech/mtusi2026/internal/gen/contracts" + "gitea.privcloud.tech/mtusi2026/internal/pkg/apperr" + "gitea.privcloud.tech/mtusi2026/internal/pkg/objectstore" +) + +const Prefix = "originals/" + +type Membership interface { + GetForMember(context.Context, dto.ChatMember) (*dto.Chat, error) +} +type Service struct { + Repo *storage.AttachmentsRepo + Store *objectstore.Store + Config *config.Attachment + chats Membership + uploads, decoders chan struct{} +} + +func New(repo *storage.AttachmentsRepo, store *objectstore.Store, c *config.Attachment, chats Membership) *Service { + return &Service{Repo: repo, Store: store, Config: c, chats: chats, uploads: make(chan struct{}, c.Uploads), decoders: make(chan struct{}, c.Decoders)} +} +func ValidFilename(s string) bool { + if strings.TrimSpace(s) == "" || !utf8.ValidString(s) || utf8.RuneCountInString(s) < 1 || utf8.RuneCountInString(s) > 255 { + return false + } + for _, r := range s { + if r == '/' || r == '\\' || unicode.IsControl(r) { + return false + } + } + return true +} +func (s *Service) Upload(w http.ResponseWriter, r *http.Request, user, chat uuid.UUID, filename string) (*contracts.Attachment, error) { + ctx := r.Context() + if _, e := s.chats.GetForMember(ctx, dto.ChatMember{UserID: user, ChatID: chat}); e != nil { + return nil, e + } + if !ValidFilename(filename) { + return nil, apperr.New(apperr.CodeBadRequest, "invalid filename") + } + mt, _, e := mime.ParseMediaType(r.Header.Get("Content-Type")) + if e != nil || (mt != "image/png" && mt != "image/jpeg") { + return nil, apperr.New(apperr.CodeUnsupportedMediaType, "expected PNG or JPEG") + } + if r.ContentLength > s.Config.MaxBytes { + return nil, apperr.New(apperr.CodePayloadTooLarge, "upload too large") + } + select { + case s.uploads <- struct{}{}: + defer func() { <-s.uploads }() + default: + return nil, apperr.New(apperr.CodeTooManyRequests, "upload capacity exceeded") + } + f, e := os.CreateTemp("", "attachment-*") + if e != nil { + return nil, e + } + defer os.Remove(f.Name()) + defer f.Close() + r.Body = http.MaxBytesReader(w, r.Body, s.Config.MaxBytes) + hash := sha256.New() + size, e := io.Copy(io.MultiWriter(f, hash), r.Body) + if e != nil { + if _, ok := e.(*http.MaxBytesError); ok { + return nil, apperr.New(apperr.CodePayloadTooLarge, "upload too large") + } + return nil, apperr.New(apperr.CodeBadRequest, "incomplete upload") + } + if e = s.Validate(ctx, f, mt); e != nil { + return nil, e + } + now := time.Now().UTC().Truncate(time.Microsecond) + id := uuid.New() + key := Prefix + id.String() + a := &storage.AttachmentRecord{Key: key, Public: contracts.Attachment{Id: id, ChatId: chat, OwnerId: user, Filename: filename, ContentType: contracts.AttachmentContentType(mt), SizeBytes: int(size), Sha256: fmt.Sprintf("%x", hash.Sum(nil)), Status: "ready", Variants: []contracts.Variant{}, CreatedAt: now, UpdatedAt: now}} + conn, unlock, e := s.Repo.Lock(ctx, key) + if e != nil { + return nil, e + } + defer unlock() + if _, e = f.Seek(0, 0); e != nil { + return nil, e + } + if e = s.Store.Put(ctx, key, mt, f, size); e != nil { + return nil, Unavailable() + } + // Never blindly compensate a failed/ambiguous COMMIT with DELETE: a committed + // row may already reference this key. Reconciliation decides after grace. + if e = s.Repo.Save(ctx, conn, a); e != nil { + return nil, e + } + return &a.Public, nil +} +func (s *Service) Validate(ctx context.Context, f *os.File, mt string) error { + if _, e := f.Seek(0, 0); e != nil { + return e + } + cfg, format, e := image.DecodeConfig(f) + if e != nil || ("image/"+format) != mt { + return apperr.New(apperr.CodeUnsupportedMediaType, "invalid image or MIME mismatch") + } + if cfg.Width <= 0 || cfg.Height <= 0 || int64(cfg.Width) > s.Config.MaxPixels/int64(cfg.Height) { + return apperr.New(apperr.CodePayloadTooLarge, "image pixel limit exceeded") + } + select { + case s.decoders <- struct{}{}: + defer func() { <-s.decoders }() + case <-ctx.Done(): + return ctx.Err() + } + if _, e = f.Seek(0, 0); e != nil { + return e + } + if _, _, e = image.Decode(f); e != nil { + return apperr.New(apperr.CodeUnsupportedMediaType, "corrupt image") + } + return ctx.Err() +} +func (s *Service) Get(ctx context.Context, user, id uuid.UUID) (*storage.AttachmentRecord, error) { + a, e := s.Repo.Get(ctx, id) + if e != nil { + return nil, e + } + if _, e = s.chats.GetForMember(ctx, dto.ChatMember{UserID: user, ChatID: a.Public.ChatId}); e != nil { + return nil, e + } + if a.Missing { + return nil, Unavailable() + } + return a, nil +} +func Unavailable() error { return apperr.New(apperr.CodeUnavailable, "attachment storage unavailable") } +func ManagedKey(key string) bool { + if !strings.HasPrefix(key, Prefix) { + return false + } + id, e := uuid.Parse(strings.TrimPrefix(key, Prefix)) + return e == nil && key == Prefix+id.String() +} diff --git a/service/internal/service/auth/api.go b/service/internal/service/auth/api.go new file mode 100644 index 00000000..6c9ec6ac --- /dev/null +++ b/service/internal/service/auth/api.go @@ -0,0 +1,15 @@ +package auth + +import ( + "context" + + "gitea.privcloud.tech/mtusi2026/internal/dto" +) + +type API interface { + Login(ctx context.Context, req dto.Login) (*dto.LoginResult, error) + VerifyAccessToken(ctx context.Context, req dto.Token) (*dto.UserDataInToken, error) + VerifySession(ctx context.Context, token dto.Token) (*dto.UserDataInToken, error) + RefreshTokens(ctx context.Context, req dto.RefreshToken) (*dto.Tokens, error) + Logout(ctx context.Context, req dto.RefreshToken) error +} diff --git a/service/internal/service/auth/interface.go b/service/internal/service/auth/interface.go new file mode 100644 index 00000000..4b2473d0 --- /dev/null +++ b/service/internal/service/auth/interface.go @@ -0,0 +1,21 @@ +package auth + +import ( + "context" + + "github.com/google/uuid" + "github.com/jackc/pgx/v5" + + "gitea.privcloud.tech/mtusi2026/internal/dto" +) + +type userRepo interface { + GetCredentialsByUsername(ctx context.Context, username string) (*dto.UserCredentials, error) +} + +type refreshTokensRepo interface { + GetByID(ctx context.Context, id uuid.UUID) (*dto.StoredRefreshToken, error) + GetByIDForUpdateTx(ctx context.Context, tx pgx.Tx, id uuid.UUID) (*dto.StoredRefreshToken, error) + CreateTx(ctx context.Context, tx pgx.Tx, req dto.StoredRefreshToken) error + RevokeTx(ctx context.Context, tx pgx.Tx, req dto.RevokeRefreshToken) error +} diff --git a/service/internal/service/auth/svc.go b/service/internal/service/auth/svc.go new file mode 100644 index 00000000..24e1cb8c --- /dev/null +++ b/service/internal/service/auth/svc.go @@ -0,0 +1,267 @@ +package auth + +import ( + "context" + "time" + + "github.com/google/uuid" + "github.com/jackc/pgx/v5" + + "gitea.privcloud.tech/mtusi2026/internal/dto" + "gitea.privcloud.tech/mtusi2026/internal/pkg/apperr" + "gitea.privcloud.tech/mtusi2026/internal/pkg/auth/jwtprovider" + "gitea.privcloud.tech/mtusi2026/internal/pkg/encrypting" + "gitea.privcloud.tech/mtusi2026/internal/pkg/postgres/txmanager" + "gitea.privcloud.tech/mtusi2026/internal/pkg/types/role" +) + +var _ API = (*Auth)(nil) + +type Auth struct { + jwt jwtprovider.Provider + tm *txmanager.TransactionManager + encryptor encrypting.Encryptor + + usersRepo userRepo + refreshTokensRepo refreshTokensRepo +} + +func New( + jwt jwtprovider.Provider, + tm *txmanager.TransactionManager, + encryptor encrypting.Encryptor, + usersRepo userRepo, + refreshTokensRepo refreshTokensRepo, +) *Auth { + return &Auth{ + jwt: jwt, + tm: tm, + encryptor: encryptor, + usersRepo: usersRepo, + refreshTokensRepo: refreshTokensRepo, + } +} + +func (a *Auth) Login(ctx context.Context, req dto.Login) (*dto.LoginResult, error) { + if req.Username == "" || req.Password == "" { + return nil, apperr.New(apperr.CodeUnauthorized, "invalid credentials") + } + + credentials, err := a.usersRepo.GetCredentialsByUsername(ctx, req.Username) + if err != nil { + if apperr.CodeFromError(err) == apperr.CodeNotFound { + return nil, apperr.New(apperr.CodeUnauthorized, "invalid credentials") + } + return nil, err + } + if err = a.encryptor.CompareHashAndSecretUsingArgon2id(ctx, credentials.PasswordHash, req.Password); err != nil { + if ctx.Err() != nil { + return nil, ctx.Err() + } + return nil, apperr.New(apperr.CodeUnauthorized, "invalid credentials") + } + + var ( + tokens *dto.Tokens + data *dto.UserDataInToken + ) + err = a.tm.RunInTransaction(ctx, nil, func(tx pgx.Tx) error { + tokens, err = a.jwt.CreateRefreshAndAccessTokensForUser(credentials.User.ID, role.User) + if err != nil { + return apperr.Wrap(err, apperr.CodeInternal, "failed to generate tokens") + } + data, err = a.verifyToken(tokens.RefreshToken.Token(), false) + if err != nil { + return apperr.Wrap(err, apperr.CodeInternal, "failed to validate generated refresh token") + } + var hash string + hash, err = a.encryptor.EncryptSecretUsingSum256(ctx, tokens.RefreshToken.Payload) + if err != nil { + return apperr.Wrap(err, apperr.CodeInternal, "failed to hash refresh token") + } + stored := dto.StoredRefreshToken{ + ID: data.TokenID, + UserID: data.UserID, + TokenHash: hash, + Role: data.Role, + ExpiresAt: data.ExpiresAt, + CreatedAt: data.IssuedAt, + } + if err = a.refreshTokensRepo.CreateTx(ctx, tx, stored); err != nil { + return err + } + return nil + }) + if err != nil { + return nil, err + } + + return &dto.LoginResult{ + ExpiresAt: data.ExpiresAt, + TokenType: "Bearer", + User: credentials.User, + Tokens: *tokens, + }, nil +} + +func (a *Auth) VerifyAccessToken(ctx context.Context, req dto.Token) (*dto.UserDataInToken, error) { + if err := ctx.Err(); err != nil { + return nil, err + } + return a.verifyToken(&req, true) +} + +func (a *Auth) VerifySession(ctx context.Context, token dto.Token) (*dto.UserDataInToken, error) { + if err := ctx.Err(); err != nil { + return nil, err + } + data, err := a.verifyTokenClaims(&token) + if err != nil { + return nil, err + } + if data.IsAccess { + return nil, apperr.New(apperr.CodeUnauthorized, "invalid session token") + } + + stored, err := a.refreshTokensRepo.GetByID(ctx, data.TokenID) + if err != nil { + if apperr.CodeFromError(err) == apperr.CodeNotFound { + return nil, apperr.New(apperr.CodeUnauthorized, "invalid session") + } + return nil, err + } + if err = a.validateStoredSession(ctx, token, data, stored); err != nil { + return nil, err + } + return data, nil +} + +func (a *Auth) Logout(ctx context.Context, req dto.RefreshToken) error { + data, err := a.verifyToken(req.Token(), false) + if err != nil { + return err + } + + return a.tm.RunInTransaction(ctx, nil, func(tx pgx.Tx) error { + stored, err := a.activeRefreshTokenTx(ctx, tx, req, data) + if err != nil { + return err + } + return a.refreshTokensRepo.RevokeTx(ctx, tx, dto.RevokeRefreshToken{ + ID: stored.ID, + RevokedAt: time.Now(), + }) + }) +} + +func (a *Auth) RefreshTokens(ctx context.Context, req dto.RefreshToken) (*dto.Tokens, error) { + data, err := a.verifyToken(req.Token(), false) + if err != nil { + return nil, err + } + + var tokens *dto.Tokens + err = a.tm.RunInTransaction(ctx, nil, func(tx pgx.Tx) error { + stored, err := a.activeRefreshTokenTx(ctx, tx, req, data) + if err != nil { + return err + } + if err = a.refreshTokensRepo.RevokeTx(ctx, tx, dto.RevokeRefreshToken{ + ID: stored.ID, + RevokedAt: time.Now(), + }); err != nil { + return err + } + tokens, err = a.createTokensTx(ctx, tx, stored.UserID, stored) + return err + }) + if err != nil { + return nil, err + } + return tokens, nil +} + +func (a *Auth) verifyToken(token *dto.Token, isAccess bool) (*dto.UserDataInToken, error) { + data, err := a.verifyTokenClaims(token) + if err != nil { + return nil, err + } + if data.IsAccess != isAccess || data.Role != role.User { + return nil, apperr.New(apperr.CodeUnauthorized, "invalid token") + } + return data, nil +} + +func (a *Auth) verifyTokenClaims(token *dto.Token) (*dto.UserDataInToken, error) { + data, err := a.jwt.ParseVerified(token) + if err != nil { + return nil, apperr.Wrap(err, apperr.CodeUnauthorized, "invalid token") + } + if err = data.Validate(); err != nil { + return nil, apperr.Wrap(err, apperr.CodeUnauthorized, "invalid token claims") + } + now := time.Now() + if data.IssuedAt.After(now) || !data.ExpiresAt.After(now) { + return nil, apperr.New(apperr.CodeUnauthorized, "invalid token") + } + return data, nil +} + +func (a *Auth) activeRefreshTokenTx(ctx context.Context, tx pgx.Tx, token dto.RefreshToken, data *dto.UserDataInToken) (*dto.StoredRefreshToken, error) { + stored, err := a.refreshTokensRepo.GetByIDForUpdateTx(ctx, tx, data.TokenID) + if err != nil { + if apperr.CodeFromError(err) == apperr.CodeNotFound { + return nil, apperr.New(apperr.CodeUnauthorized, "invalid refresh token") + } + return nil, err + } + + // Recheck expiry after acquiring the row lock: a concurrent rotation may have held it. + if err = a.validateStoredSession(ctx, dto.Token(token), data, stored); err != nil { + return nil, err + } + return stored, nil +} + +func (a *Auth) validateStoredSession(ctx context.Context, token dto.Token, data *dto.UserDataInToken, stored *dto.StoredRefreshToken) error { + now := time.Now() + if stored.RevokedAt != nil || !stored.ExpiresAt.After(now) || !data.ExpiresAt.After(now) || + stored.UserID != data.UserID || stored.Role != data.Role { + return apperr.New(apperr.CodeUnauthorized, "invalid session") + } + if err := a.encryptor.CompareHashAndSecretUsingSum256(ctx, stored.TokenHash, token.Payload); err != nil { + return apperr.New(apperr.CodeUnauthorized, "invalid session") + } + return nil +} + +func (a *Auth) createTokensTx(ctx context.Context, tx pgx.Tx, userID uuid.UUID, previous *dto.StoredRefreshToken) (*dto.Tokens, error) { + tokens, err := a.jwt.CreateRefreshAndAccessTokensForUser(userID, role.User) + if err != nil { + return nil, apperr.Wrap(err, apperr.CodeInternal, "failed to generate tokens") + } + data, err := a.verifyToken(tokens.RefreshToken.Token(), false) + if err != nil { + return nil, apperr.Wrap(err, apperr.CodeInternal, "failed to validate generated refresh token") + } + hash, err := a.encryptor.EncryptSecretUsingSum256(ctx, tokens.RefreshToken.Payload) + if err != nil { + return nil, apperr.Wrap(err, apperr.CodeInternal, "failed to hash refresh token") + } + stored := dto.StoredRefreshToken{ + ID: data.TokenID, + UserID: data.UserID, + TokenHash: hash, + Role: data.Role, + ExpiresAt: data.ExpiresAt, + CreatedAt: data.IssuedAt, + } + if previous != nil { + stored.UserAgent = previous.UserAgent + stored.IPAddress = previous.IPAddress + } + if err = a.refreshTokensRepo.CreateTx(ctx, tx, stored); err != nil { + return nil, err + } + return tokens, nil +} diff --git a/service/internal/service/chat/api.go b/service/internal/service/chat/api.go new file mode 100644 index 00000000..cae5a2d4 --- /dev/null +++ b/service/internal/service/chat/api.go @@ -0,0 +1,15 @@ +package chat + +import ( + "context" + + "gitea.privcloud.tech/mtusi2026/internal/dto" +) + +type API interface { + CreateChat(ctx context.Context, req dto.CreateChat) (*dto.Chat, error) + GetForMember(ctx context.Context, req dto.ChatMember) (*dto.Chat, error) + ListForMember(ctx context.Context, req dto.ChatFilters) (*dto.Page[dto.Chat], error) + CreateMessage(ctx context.Context, req dto.CreateMessage) (*dto.Message, error) + ListForChat(ctx context.Context, req dto.MessageFilters) (*dto.Page[dto.Message], error) +} diff --git a/service/internal/service/chat/interface.go b/service/internal/service/chat/interface.go new file mode 100644 index 00000000..dbe71578 --- /dev/null +++ b/service/internal/service/chat/interface.go @@ -0,0 +1,28 @@ +package chat + +import ( + "context" + + "github.com/google/uuid" + "github.com/jackc/pgx/v5" + + "gitea.privcloud.tech/mtusi2026/internal/dto" +) + +type chatsRepo interface { + CreateTx(ctx context.Context, tx pgx.Tx, req dto.Chat) error + GetForMember(ctx context.Context, req dto.ChatMember) (*dto.Chat, error) + ListForMember(ctx context.Context, userID uuid.UUID, page dto.PageQuery) ([]dto.Chat, error) +} + +type messagesRepo interface { + AttachmentsForMessageTx(ctx context.Context, tx pgx.Tx, ids []uuid.UUID) ([]dto.MessageAttachment, error) + CreateIdempotentTx(ctx context.Context, tx pgx.Tx, req dto.CreateMessage) (*dto.Message, error) + GetByIDForChat(ctx context.Context, chatID, messageID uuid.UUID) (*dto.Message, error) + ListForChat(ctx context.Context, chatID uuid.UUID, page dto.PageQuery) ([]dto.Message, error) +} + +type chatsCache interface { + Create(ctx context.Context, req dto.Chat) error + Get(ctx context.Context, chatID uuid.UUID) (*dto.Chat, error) +} diff --git a/service/internal/service/chat/svc.go b/service/internal/service/chat/svc.go new file mode 100644 index 00000000..6dacf961 --- /dev/null +++ b/service/internal/service/chat/svc.go @@ -0,0 +1,286 @@ +package chat + +import ( + "bytes" + "context" + "errors" + "log/slog" + "slices" + "strings" + "time" + "unicode/utf8" + + "github.com/google/uuid" + "github.com/jackc/pgx/v5" + "github.com/redis/go-redis/v9" + "golang.org/x/sync/singleflight" + + "gitea.privcloud.tech/mtusi2026/internal/dto" + "gitea.privcloud.tech/mtusi2026/internal/pkg/apperr" + "gitea.privcloud.tech/mtusi2026/internal/pkg/postgres/txmanager" +) + +var _ API = (*Chat)(nil) + +const chatLoadTimeout = 2 * time.Second + +type Chat struct { + tm *txmanager.TransactionManager + chatsRepo chatsRepo + messagesRepo messagesRepo + chatsCache chatsCache + loads singleflight.Group +} + +func New(tm *txmanager.TransactionManager, chatsRepo chatsRepo, messagesRepo messagesRepo, chatsCache chatsCache) *Chat { + return &Chat{ + tm: tm, + chatsRepo: chatsRepo, + messagesRepo: messagesRepo, + chatsCache: chatsCache, + } +} + +func (c *Chat) CreateChat(ctx context.Context, req dto.CreateChat) (*dto.Chat, error) { + seen := make(map[uuid.UUID]struct{}, len(req.MemberIDs)) + for _, id := range req.MemberIDs { + if id == uuid.Nil { + return nil, apperr.New(apperr.CodeBadRequest, "invalid chat member") + } + if _, exists := seen[id]; exists { + return nil, apperr.New(apperr.CodeBadRequest, "duplicate chat member") + } + seen[id] = struct{}{} + } + + if _, included := seen[req.CreatorID]; !included { + return nil, apperr.New(apperr.CodeBadRequest, "creator must be a chat member") + } + + members := slices.Clone(req.MemberIDs) + slices.SortFunc(members, func(a, b uuid.UUID) int { + return bytes.Compare(a[:], b[:]) + }) + chat := dto.Chat{ + ID: uuid.New(), + Title: req.Title, + MemberIDs: members, + CreatedAt: time.Now().Truncate(time.Microsecond), + } + + if err := c.tm.RunInTransaction(ctx, nil, func(tx pgx.Tx) error { + return c.chatsRepo.CreateTx(ctx, tx, chat) + }); err != nil { + return nil, err + } + + return &chat, nil +} + +func (c *Chat) GetForMember(ctx context.Context, req dto.ChatMember) (*dto.Chat, error) { + if err := ctx.Err(); err != nil { + return nil, err + } + if chat, err := c.cachedChat(ctx, req.ChatID); err == nil { + return chatForMember(chat, req.UserID) + } + + // Repository results depend on membership: never share a user's 404 with + // another user requesting the same chat. + key := req.ChatID.String() + ":" + req.UserID.String() + result := c.loads.DoChan(key, func() (any, error) { + // One client disconnecting must not cancel the other clients' load. + loadCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), chatLoadTimeout) + defer cancel() + + // A previous load may have populated Redis after our initial miss. + if chat, err := c.cachedChat(loadCtx, req.ChatID); err == nil { + return chatForMember(chat, req.UserID) + } + + slog.Info("chat_db_lookup", "chat_id", req.ChatID) + chat, err := c.chatsRepo.GetForMember(loadCtx, req) + if err != nil { + return nil, err + } + if err = c.chatsCache.Create(loadCtx, *chat); err != nil { + slog.Warn("failed to cache chat", "chat_id", req.ChatID, "error", err) + } + return chat, nil + }) + + select { + case <-ctx.Done(): + return nil, ctx.Err() + case loaded := <-result: + if err := ctx.Err(); err != nil { + return nil, err + } + if loaded.Err != nil { + return nil, loaded.Err + } + return chatForMember(loaded.Val.(*dto.Chat), req.UserID) + } +} + +func (c *Chat) cachedChat(ctx context.Context, chatID uuid.UUID) (*dto.Chat, error) { + chat, err := c.chatsCache.Get(ctx, chatID) + switch { + case err == nil: + slog.Info("chat_cache_hit", "chat_id", chatID) + case errors.Is(err, redis.Nil): + slog.Info("chat_cache_miss", "chat_id", chatID) + default: + slog.Warn("chat_cache_read_error", "chat_id", chatID, "error", err) + } + return chat, err +} + +func chatForMember(chat *dto.Chat, userID uuid.UUID) (*dto.Chat, error) { + if !slices.Contains(chat.MemberIDs, userID) { + return nil, apperr.New(apperr.CodeNotFound, "chat not found") + } + return chat, nil +} + +func (c *Chat) ListForMember(ctx context.Context, req dto.ChatFilters) (*dto.Page[dto.Chat], error) { + limit, err := pageLimit(req.Limit) + if err != nil { + return nil, err + } + + query := dto.PageQuery{Limit: limit + 1} + if req.AfterID != nil { + cursor, err := c.GetForMember(ctx, dto.ChatMember{UserID: req.UserID, ChatID: *req.AfterID}) + if err != nil { + return nil, cursorError(err) + } + query.After = &dto.PagePosition{CreatedAt: cursor.CreatedAt, ID: cursor.ID} + } + + items, err := c.chatsRepo.ListForMember(ctx, req.UserID, query) + if err != nil { + return nil, err + } + + return makePage(items, limit, func(chat dto.Chat) uuid.UUID { return chat.ID }), nil +} + +func (c *Chat) CreateMessage(ctx context.Context, req dto.CreateMessage) (*dto.Message, error) { + if _, err := c.GetForMember(ctx, dto.ChatMember{UserID: req.SenderID, ChatID: req.ChatID}); err != nil { + return nil, err + } + + if err := validateMessage(req); err != nil { + return nil, err + } + req.AttachmentIDs = append([]uuid.UUID{}, req.AttachmentIDs...) + + var message *dto.Message + err := c.tm.RunInTransaction(ctx, nil, func(tx pgx.Tx) error { + if len(req.AttachmentIDs) > 0 { + attachments, err := c.messagesRepo.AttachmentsForMessageTx(ctx, tx, req.AttachmentIDs) + if err != nil { + return err + } + if len(attachments) != len(req.AttachmentIDs) { + return apperr.New(apperr.CodeNotFound, "attachment not found") + } + for _, attachment := range attachments { + if attachment.ChatID != req.ChatID { + return apperr.New(apperr.CodeNotFound, "attachment not found") + } + if attachment.Missing { + return apperr.New(apperr.CodeUnavailable, "attachment storage unavailable") + } + } + } + + var err error + message, err = c.messagesRepo.CreateIdempotentTx(ctx, tx, req) + return err + }) + return message, err +} + +func (c *Chat) ListForChat(ctx context.Context, req dto.MessageFilters) (*dto.Page[dto.Message], error) { + if _, err := c.GetForMember(ctx, dto.ChatMember{UserID: req.UserID, ChatID: req.ChatID}); err != nil { + return nil, err + } + + limit, err := pageLimit(req.Limit) + if err != nil { + return nil, err + } + + query := dto.PageQuery{Limit: limit + 1} + if req.AfterID != nil { + cursor, err := c.messagesRepo.GetByIDForChat(ctx, req.ChatID, *req.AfterID) + if err != nil { + return nil, cursorError(err) + } + query.After = &dto.PagePosition{CreatedAt: cursor.CreatedAt, ID: cursor.ID} + } + + items, err := c.messagesRepo.ListForChat(ctx, req.ChatID, query) + if err != nil { + return nil, err + } + + return makePage(items, limit, func(message dto.Message) uuid.UUID { return message.ID }), nil +} + +func pageLimit(limit *int) (int, error) { + size := 50 + if limit != nil { + size = *limit + } + if size < 1 || size > 100 { + return 0, apperr.New(apperr.CodeBadRequest, "invalid limit") + } + return size, nil +} + +func cursorError(err error) error { + if apperr.CodeFromError(err) == apperr.CodeNotFound { + return apperr.New(apperr.CodeBadRequest, "invalid cursor") + } + return err +} + +func makePage[T any](items []T, limit int, id func(T) uuid.UUID) *dto.Page[T] { + if items == nil { + items = make([]T, 0) + } + page := dto.Page[T]{Items: items} + if len(items) > limit { + page.Items = items[:limit] + cursor := id(page.Items[limit-1]) + page.NextCursorID = &cursor + } + return &page +} + +// Content rules belong to the service so non-HTTP callers cannot bypass them. +func validateMessage(req dto.CreateMessage) error { + if !utf8.ValidString(req.Text) || strings.ContainsRune(req.Text, '\x00') || utf8.RuneCountInString(req.Text) > 4000 { + return apperr.New(apperr.CodeBadRequest, "invalid message text") + } + if len(req.AttachmentIDs) > 10 { + return apperr.New(apperr.CodeBadRequest, "too many attachments") + } + seen := make(map[uuid.UUID]struct{}, len(req.AttachmentIDs)) + for _, id := range req.AttachmentIDs { + if id == uuid.Nil { + return apperr.New(apperr.CodeBadRequest, "invalid attachment id") + } + if _, ok := seen[id]; ok { + return apperr.New(apperr.CodeBadRequest, "duplicate attachment id") + } + seen[id] = struct{}{} + } + if strings.TrimSpace(req.Text) == "" && len(req.AttachmentIDs) == 0 { + return apperr.New(apperr.CodeBadRequest, "message requires text or an attachment") + } + return nil +} diff --git a/service/internal/service/user/api.go b/service/internal/service/user/api.go new file mode 100644 index 00000000..65329cf2 --- /dev/null +++ b/service/internal/service/user/api.go @@ -0,0 +1,15 @@ +package user + +import ( + "context" + + "github.com/google/uuid" + + "gitea.privcloud.tech/mtusi2026/internal/dto" +) + +type API interface { + Create(ctx context.Context, req dto.CreateUser) (*dto.User, error) + GetByID(ctx context.Context, id uuid.UUID) (*dto.User, error) + GetByUsername(ctx context.Context, username string) (*dto.User, error) +} diff --git a/service/internal/service/user/interface.go b/service/internal/service/user/interface.go new file mode 100644 index 00000000..4eb78b5e --- /dev/null +++ b/service/internal/service/user/interface.go @@ -0,0 +1,15 @@ +package user + +import ( + "context" + + "github.com/google/uuid" + + "gitea.privcloud.tech/mtusi2026/internal/dto" +) + +type userRepo interface { + Create(ctx context.Context, req dto.CreateStoredUser) error + GetByID(ctx context.Context, id uuid.UUID) (*dto.User, error) + GetByUsername(ctx context.Context, username string) (*dto.User, error) +} diff --git a/service/internal/service/user/svc.go b/service/internal/service/user/svc.go new file mode 100644 index 00000000..f6070a91 --- /dev/null +++ b/service/internal/service/user/svc.go @@ -0,0 +1,67 @@ +package user + +import ( + "context" + "time" + + "github.com/google/uuid" + + "gitea.privcloud.tech/mtusi2026/internal/dto" + "gitea.privcloud.tech/mtusi2026/internal/pkg/apperr" + "gitea.privcloud.tech/mtusi2026/internal/pkg/encrypting" +) + +var ( + _ API = (*User)(nil) +) + +type User struct { + encryptor encrypting.Encryptor + usersRepo userRepo +} + +func New(encryptor encrypting.Encryptor, usersRepo userRepo) *User { + return &User{encryptor: encryptor, usersRepo: usersRepo} +} + +func (u *User) Create(ctx context.Context, req dto.CreateUser) (*dto.User, error) { + hash, err := u.encryptor.EncryptSecretUsingArgon2id(ctx, req.Password) + if err != nil { + return nil, apperr.Wrap(err, apperr.CodeInternal, "failed to hash password") + } + + user := dto.User{ + ID: uuid.New(), + Username: req.Username, + DisplayName: req.DisplayName, + CreatedAt: time.Now().Truncate(time.Microsecond), + } + if err = u.usersRepo.Create(ctx, dto.CreateStoredUser{ + ID: user.ID, + Username: user.Username, + DisplayName: user.DisplayName, + PasswordHash: hash, + CreatedAt: user.CreatedAt, + UpdatedAt: user.CreatedAt, + }); err != nil { + return nil, err + } + + return &user, nil +} + +func (u *User) GetByID(ctx context.Context, id uuid.UUID) (*dto.User, error) { + user, err := u.usersRepo.GetByID(ctx, id) + if err != nil { + return nil, err + } + return user, nil +} + +func (u *User) GetByUsername(ctx context.Context, username string) (*dto.User, error) { + user, err := u.usersRepo.GetByUsername(ctx, username) + if err != nil { + return nil, err + } + return user, nil +} diff --git a/service/migrations/001_create_users_table.sql b/service/migrations/001_create_users_table.sql new file mode 100644 index 00000000..60f730ad --- /dev/null +++ b/service/migrations/001_create_users_table.sql @@ -0,0 +1,18 @@ +CREATE TABLE IF NOT EXISTS users +( + id UUID PRIMARY KEY, + name VARCHAR(100) NOT NULL, + username VARCHAR(32) NOT NULL, + password_hash TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + + CONSTRAINT users_username_unique UNIQUE (username), + CONSTRAINT users_username_format CHECK (username ~ '^[a-z0-9_]{3,32}$'), + CONSTRAINT users_name_not_blank CHECK (name !~ '^[[:space:]]*$'), + CONSTRAINT users_password_hash_not_empty CHECK (password_hash <> '') +); + +---- create above / drop below ---- + +DROP TABLE IF EXISTS users; diff --git a/service/migrations/002_create_refresh_tokens_table.sql b/service/migrations/002_create_refresh_tokens_table.sql new file mode 100644 index 00000000..cbc73082 --- /dev/null +++ b/service/migrations/002_create_refresh_tokens_table.sql @@ -0,0 +1,19 @@ +CREATE TABLE IF NOT EXISTS refresh_tokens +( + id UUID PRIMARY KEY, + user_id UUID NOT NULL REFERENCES users (id), + token_hash TEXT NOT NULL, + user_agent TEXT, + ip_address INET, + role VARCHAR(16) NOT NULL, + expires_at TIMESTAMPTZ NOT NULL, + revoked_at TIMESTAMPTZ, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW() +); + +CREATE INDEX IF NOT EXISTS idx_refresh_user_id ON refresh_tokens (user_id); +CREATE INDEX IF NOT EXISTS idx_refresh_expires ON refresh_tokens (expires_at); + +---- create above / drop below ---- + +DROP TABLE IF EXISTS refresh_tokens; \ No newline at end of file diff --git a/service/migrations/003_create_chats_table.sql b/service/migrations/003_create_chats_table.sql new file mode 100644 index 00000000..b45df916 --- /dev/null +++ b/service/migrations/003_create_chats_table.sql @@ -0,0 +1,14 @@ +CREATE TABLE IF NOT EXISTS chats +( + id UUID PRIMARY KEY, + title VARCHAR(100) NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + + CONSTRAINT chats_title_not_blank CHECK (title !~ '^[[:space:]]*$') +); + +CREATE INDEX IF NOT EXISTS idx_chats_created_at_id ON chats (created_at, id); + +---- create above / drop below ---- + +DROP TABLE IF EXISTS chats; diff --git a/service/migrations/004_create_chat_members_table.sql b/service/migrations/004_create_chat_members_table.sql new file mode 100644 index 00000000..64131981 --- /dev/null +++ b/service/migrations/004_create_chat_members_table.sql @@ -0,0 +1,15 @@ +CREATE TABLE IF NOT EXISTS chat_members +( + chat_id UUID NOT NULL, + user_id UUID NOT NULL, + + PRIMARY KEY (chat_id, user_id), + CONSTRAINT chat_members_chat_fk FOREIGN KEY (chat_id) REFERENCES chats (id) ON DELETE NO ACTION, + CONSTRAINT chat_members_user_fk FOREIGN KEY (user_id) REFERENCES users (id) ON DELETE NO ACTION +); + +CREATE INDEX IF NOT EXISTS idx_chat_members_user_id_chat_id ON chat_members (user_id, chat_id); + +---- create above / drop below ---- + +DROP TABLE IF EXISTS chat_members; diff --git a/service/migrations/005_create_messages_table.sql b/service/migrations/005_create_messages_table.sql new file mode 100644 index 00000000..1ab0d7dc --- /dev/null +++ b/service/migrations/005_create_messages_table.sql @@ -0,0 +1,18 @@ +CREATE TABLE IF NOT EXISTS messages +( + id UUID PRIMARY KEY, + chat_id UUID NOT NULL, + sender_id UUID NOT NULL, + text VARCHAR(4000) NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + + CONSTRAINT messages_text_not_blank CHECK (text !~ '^[[:space:]]*$'), + CONSTRAINT messages_sender_member_fk FOREIGN KEY (chat_id, sender_id) + REFERENCES chat_members (chat_id, user_id) ON DELETE NO ACTION +); + +CREATE INDEX IF NOT EXISTS idx_messages_chat_id_created_at_id ON messages (chat_id, created_at, id); + +---- create above / drop below ---- + +DROP TABLE IF EXISTS messages; diff --git a/service/migrations/006_message_idempotency.sql b/service/migrations/006_message_idempotency.sql new file mode 100644 index 00000000..8e48e9ba --- /dev/null +++ b/service/migrations/006_message_idempotency.sql @@ -0,0 +1,14 @@ +CREATE TABLE message_idempotency ( + user_id UUID NOT NULL REFERENCES users(id), + method TEXT NOT NULL CHECK (method = 'POST'), + chat_id UUID NOT NULL REFERENCES chats(id), + key TEXT NOT NULL CHECK (key ~ '^[A-Za-z0-9._:-]{1,128}$'), + request_hash BYTEA NOT NULL, + message_id UUID NOT NULL REFERENCES messages(id), + expires_at TIMESTAMPTZ NOT NULL, + PRIMARY KEY (user_id, method, chat_id, key) +); +CREATE INDEX message_idempotency_expiry ON message_idempotency(expires_at); + +---- create above / drop below ---- +DROP TABLE message_idempotency; diff --git a/service/migrations/007_attachments.sql b/service/migrations/007_attachments.sql new file mode 100644 index 00000000..050c6864 --- /dev/null +++ b/service/migrations/007_attachments.sql @@ -0,0 +1,44 @@ +CREATE TABLE attachments +( + id UUID PRIMARY KEY, + chat_id UUID NOT NULL REFERENCES chats (id), + owner_id UUID NOT NULL, + filename TEXT NOT NULL, + object_key TEXT NOT NULL UNIQUE, + content_type TEXT NOT NULL CHECK (content_type IN ('image/png', 'image/jpeg')), + size_bytes BIGINT NOT NULL CHECK (size_bytes BETWEEN 1 AND 10485760), + sha256 TEXT NOT NULL CHECK (sha256 ~ '^[a-f0-9]{64}$'), + integrity_missing BOOLEAN NOT NULL DEFAULT false, + created_at TIMESTAMPTZ NOT NULL, + updated_at TIMESTAMPTZ NOT NULL, + UNIQUE (id, chat_id), + FOREIGN KEY (chat_id, owner_id) REFERENCES chat_members (chat_id, user_id) +); +CREATE INDEX attachments_scan ON attachments (id); +ALTER TABLE messages + DROP CONSTRAINT messages_text_not_blank; +ALTER TABLE messages + ADD CONSTRAINT messages_id_chat UNIQUE (id, chat_id); +CREATE TABLE message_attachments +( + message_id UUID NOT NULL, + chat_id UUID NOT NULL, + attachment_id UUID NOT NULL, + position SMALLINT NOT NULL CHECK (position BETWEEN 0 AND 9), + PRIMARY KEY (message_id, attachment_id), + UNIQUE (message_id, position), + FOREIGN KEY (message_id, chat_id) REFERENCES messages (id, chat_id), + FOREIGN KEY (attachment_id, chat_id) REFERENCES attachments (id, chat_id) +); +ALTER TABLE message_idempotency + ADD COLUMN hash_version SMALLINT NOT NULL DEFAULT 1; + +---- create above / drop below ---- +ALTER TABLE message_idempotency + DROP COLUMN hash_version; +DROP TABLE message_attachments; +ALTER TABLE messages + DROP CONSTRAINT messages_id_chat; +ALTER TABLE messages + ADD CONSTRAINT messages_text_not_blank CHECK (text !~ '^[[:space:]]*$'); +DROP TABLE attachments; diff --git a/service/migrations/embed.go b/service/migrations/embed.go new file mode 100644 index 00000000..eb054216 --- /dev/null +++ b/service/migrations/embed.go @@ -0,0 +1,8 @@ +package migrations + +import ( + "embed" +) + +//go:embed *.sql +var Migrations embed.FS diff --git a/service/tests/api_test.go b/service/tests/api_test.go new file mode 100644 index 00000000..db9fdd01 --- /dev/null +++ b/service/tests/api_test.go @@ -0,0 +1,285 @@ +package tests + +import ( + "encoding/json" + "fmt" + "github.com/google/uuid" + "io" + "net/http" + "net/url" + "os" + "strings" + "testing" + "time" +) + +type apiClient struct { + baseURL string + prefix string + tokens map[string]string +} + +type apiResponse struct { + status int + requestID string + instanceID string + body []byte +} + +func newAPI(t *testing.T) *apiClient { + t.Helper() + base := os.Getenv("TEST_BASE_URL") + if base == "" { + base = "http://localhost:18085" + } + return &apiClient{baseURL: strings.TrimRight(base, "/"), prefix: uuid.NewString()[:8] + "_", tokens: map[string]string{}} +} +func (api *apiClient) request(t *testing.T, method, path, userID, body string) apiResponse { + return api.requestKey(t, method, path, userID, body, uuid.NewString()) +} +func (api *apiClient) requestKey(t *testing.T, method, path, userID, body, key string) apiResponse { + t.Helper() + response, err := api.send(method, path, userID, body, key) + if err != nil { + t.Fatal(err) + } + return response +} +func (api *apiClient) send(method, path, userID, body, key string) (apiResponse, error) { + if path == "/api/v1/users" { + var fields map[string]any + if json.Unmarshal([]byte(body), &fields) == nil { + if name, ok := fields["username"].(string); ok { + fields["username"] = api.prefix + name + data, err := json.Marshal(fields) + if err != nil { + return apiResponse{}, err + } + body = string(data) + } + } + } + req, err := http.NewRequest(method, api.baseURL+path, strings.NewReader(body)) + if err != nil { + return apiResponse{}, err + } + if body != "" { + req.Header.Set("Content-Type", "application/json") + } + if userID != "" { + req.Header.Set("Authorization", "Bearer "+api.tokens[userID]) + } + if key != "" { + req.Header.Set("Idempotency-Key", key) + } + response, err := (&http.Client{Timeout: 5 * time.Second}).Do(req) + if err != nil { + return apiResponse{}, fmt.Errorf("running lab5 stand required: %w", err) + } + defer response.Body.Close() + data, err := io.ReadAll(response.Body) + if err != nil { + return apiResponse{}, err + } + requestID, instanceID := response.Header.Get("X-Request-Id"), response.Header.Get("X-Instance-Id") + if requestID == "" || instanceID == "" { + return apiResponse{}, fmt.Errorf("%s %s: HTTP %d missing response identity", method, path, response.StatusCode) + } + return apiResponse{status: response.StatusCode, requestID: requestID, instanceID: instanceID, body: data}, nil +} + +func jsonString(t *testing.T, value any) string { + t.Helper() + data, err := json.Marshal(value) + if err != nil { + t.Fatalf("encode request: %v", err) + } + return string(data) +} + +func decodeOK(t *testing.T, resp apiResponse, wantStatus int, target any) { + t.Helper() + if resp.status != wantStatus { + t.Fatalf("status = %d, want %d; body: %s", resp.status, wantStatus, resp.body) + } + if err := json.Unmarshal(resp.body, target); err != nil { + t.Fatalf("decode response %s: %v", resp.body, err) + } +} + +func assertError(t *testing.T, resp apiResponse, wantStatus int, wantCode string) { + t.Helper() + if resp.status != wantStatus { + t.Fatalf("status = %d, want %d; body: %s", resp.status, wantStatus, resp.body) + } + var body struct { + Error struct { + Code string `json:"code"` + Message string `json:"message"` + RequestID string `json:"request_id"` + } `json:"error"` + } + if err := json.Unmarshal(resp.body, &body); err != nil { + t.Fatalf("decode error response %s: %v", resp.body, err) + } + if body.Error.Code != wantCode || body.Error.Message == "" || body.Error.RequestID != resp.requestID { + t.Fatalf("invalid error response: header request ID %q, body %s", resp.requestID, resp.body) + } +} + +func (api *apiClient) createUser(t *testing.T, username string) string { + t.Helper() + resp := api.request(t, http.MethodPost, "/api/v1/users", "", jsonString(t, map[string]string{ + "username": username, "display_name": username, "password": "Synthetic-password-123", + })) + var user struct { + ID string `json:"id"` + } + decodeOK(t, resp, http.StatusCreated, &user) + if user.ID == "" { + t.Fatal("created user has no ID") + } + req, _ := http.NewRequest(http.MethodPost, api.baseURL+"/api/v1/auth/sessions", nil) + req.SetBasicAuth(api.prefix+username, "Synthetic-password-123") + sessionResp, err := (&http.Client{Timeout: 5 * time.Second}).Do(req) + if err != nil { + t.Fatal(err) + } + defer sessionResp.Body.Close() + var session struct { + Token string `json:"token"` + } + if err = json.NewDecoder(sessionResp.Body).Decode(&session); err != nil || sessionResp.StatusCode != 201 || session.Token == "" { + t.Fatalf("login failed: status %d, error %v", sessionResp.StatusCode, err) + } + api.tokens[user.ID] = session.Token + return user.ID +} + +func (api *apiClient) createChat(t *testing.T, creatorID string, memberIDs ...string) string { + t.Helper() + resp := api.request(t, http.MethodPost, "/api/v1/chats", creatorID, jsonString(t, map[string]any{ + "title": "Test chat", "member_ids": memberIDs, + })) + var chat struct { + ID string `json:"id"` + } + decodeOK(t, resp, http.StatusCreated, &chat) + if chat.ID == "" { + t.Fatal("created chat has no ID") + } + return chat.ID +} + +func TestInvalidJSON(t *testing.T) { + api := newAPI(t) + for _, tc := range []struct { + name string + body string + }{ + {name: "malformed", body: `{"username":`}, + {name: "wrong type", body: `{"username":42,"display_name":"Alice"}`}, + {name: "unknown field", body: `{"username":"alice","display_name":"Alice","admin":true}`}, + } { + t.Run(tc.name, func(t *testing.T) { + resp := api.request(t, http.MethodPost, "/api/v1/users", "", tc.body) + assertError(t, resp, http.StatusBadRequest, "invalid_request") + }) + } +} + +func TestForeignChatIsHidden(t *testing.T) { + api := newAPI(t) + alice := api.createUser(t, "alice") + bob := api.createUser(t, "bob") + eve := api.createUser(t, "eve") + chatID := api.createChat(t, alice, alice, bob) + + for _, tc := range []struct { + method string + path string + body string + }{ + {method: http.MethodGet, path: "/api/v1/chats/" + chatID}, + {method: http.MethodGet, path: "/api/v1/chats/" + chatID + "/messages"}, + {method: http.MethodPost, path: "/api/v1/chats/" + chatID + "/messages", body: `{"text":"secret"}`}, + } { + resp := api.request(t, tc.method, tc.path, eve, tc.body) + assertError(t, resp, http.StatusNotFound, "not_found") + } +} + +func TestEmptyMessage(t *testing.T) { + api := newAPI(t) + alice := api.createUser(t, "alice") + chatID := api.createChat(t, alice, alice) + path := "/api/v1/chats/" + chatID + "/messages" + for _, body := range []string{`{"text":""}`, `{"text":" \t\n "}`} { + resp := api.request(t, http.MethodPost, path, alice, body) + assertError(t, resp, http.StatusBadRequest, "invalid_request") + } +} + +func TestDuplicateUsername(t *testing.T) { + api := newAPI(t) + api.createUser(t, "alice") + resp := api.request(t, http.MethodPost, "/api/v1/users", "", `{"username":"alice","display_name":"Another Alice","password":"Synthetic-password-123"}`) + assertError(t, resp, http.StatusConflict, "conflict") +} + +func TestMessagePagination(t *testing.T) { + api := newAPI(t) + alice := api.createUser(t, "alice") + chatID := api.createChat(t, alice, alice) + wantIDs := make(map[string]bool) + for i := range 5 { + resp := api.request(t, http.MethodPost, "/api/v1/chats/"+chatID+"/messages", alice, + jsonString(t, map[string]string{"text": fmt.Sprintf("message %d", i)})) + var message struct { + ID string `json:"id"` + } + decodeOK(t, resp, http.StatusCreated, &message) + if message.ID == "" || wantIDs[message.ID] { + t.Fatalf("invalid or duplicate created message ID %q", message.ID) + } + wantIDs[message.ID] = true + } + + seen := make(map[string]bool) + cursor := "" + pages := 0 + for { + path := "/api/v1/chats/" + chatID + "/messages?limit=2" + if cursor != "" { + path += "&cursor=" + url.QueryEscape(cursor) + } + resp := api.request(t, http.MethodGet, path, alice, "") + var page struct { + Items []struct { + ID string `json:"id"` + } `json:"items"` + NextCursor *string `json:"next_cursor"` + } + decodeOK(t, resp, http.StatusOK, &page) + pages++ + if pages > 3 || len(page.Items) == 0 || len(page.Items) > 2 { + t.Fatalf("unexpected page %d: %s", pages, resp.body) + } + for _, message := range page.Items { + if !wantIDs[message.ID] || seen[message.ID] { + t.Fatalf("unexpected or repeated message ID %q on page %d", message.ID, pages) + } + seen[message.ID] = true + } + if page.NextCursor == nil { + break + } + if *page.NextCursor == "" || *page.NextCursor == cursor { + t.Fatalf("invalid next cursor on page %d: %s", pages, resp.body) + } + cursor = *page.NextCursor + } + if pages != 3 || len(seen) != len(wantIDs) { + t.Fatalf("got %d pages and %d unique messages, want 3 pages and %d messages", pages, len(seen), len(wantIDs)) + } +} diff --git a/service/tests/attachments_test.go b/service/tests/attachments_test.go new file mode 100644 index 00000000..e9a5860f --- /dev/null +++ b/service/tests/attachments_test.go @@ -0,0 +1,204 @@ +package tests + +import ( + "bytes" + "crypto/sha256" + "encoding/binary" + "encoding/hex" + "encoding/json" + "hash/crc32" + "image" + "image/jpeg" + "io" + "net/http" + "net/url" + "os" + "strings" + "sync" + "testing" + "time" + + "github.com/google/uuid" +) + +func (api *apiClient) upload(t *testing.T, chat, user, name, mime string, data []byte, chunked bool) apiResponse { + t.Helper() + req, e := http.NewRequest("POST", api.baseURL+"/api/v1/chats/"+chat+"/attachments?filename="+url.QueryEscape(name), bytes.NewReader(data)) + if e != nil { + t.Fatal(e) + } + if chunked { + req.ContentLength = -1 + } + req.Header.Set("Content-Type", mime) + if user != "" { + req.Header.Set("Authorization", "Bearer "+api.tokens[user]) + } + resp, e := (&http.Client{Timeout: 20 * time.Second}).Do(req) + if e != nil { + t.Fatal(e) + } + defer resp.Body.Close() + body, e := io.ReadAll(resp.Body) + if e != nil { + t.Fatal(e) + } + if resp.Header.Get("X-Instance-Id") == "" { + t.Fatal("missing instance id") + } + return apiResponse{status: resp.StatusCode, body: body, requestID: resp.Header.Get("X-Request-Id"), instanceID: resp.Header.Get("X-Instance-Id")} +} +func fixturePNG(t *testing.T) []byte { + t.Helper() + b, e := os.ReadFile("../../tests/fixtures/sample.png") + if e != nil { + t.Fatal(e) + } + return b +} +func TestAttachmentContractAndErrors(t *testing.T) { + api := newAPI(t) + alice := api.createUser(t, "alice") + bob := api.createUser(t, "bob") + eve := api.createUser(t, "eve") + chat := api.createChat(t, alice, alice, bob) + other := api.createChat(t, alice, alice) + data := fixturePNG(t) + var att struct { + ID string `json:"id"` + Sha string `json:"sha256"` + Size int `json:"size_bytes"` + } + decodeOK(t, api.upload(t, chat, alice, "Файл.png", "image/png", data, false), 201, &att) + hash := sha256.Sum256(data) + if att.Sha != hex.EncodeToString(hash[:]) || att.Size != len(data) { + t.Fatal("original metadata mismatch") + } + content := "/api/v1/attachments/" + att.ID + "/content" + got := api.request(t, "GET", content, bob, "") + if got.status != 200 || !bytes.Equal(got.body, data) { + t.Fatal("download changed bytes") + } + for _, suffix := range []string{"", "/content"} { + assertError(t, api.request(t, "GET", "/api/v1/attachments/"+att.ID+suffix, eve, ""), 404, "not_found") + } + assertError(t, api.upload(t, chat, eve, "file.png", "image/png", data, false), 404, "not_found") + assertError(t, api.upload(t, chat, "", "file.png", "image/png", data, false), 401, "unauthorized") + for _, name := range []string{"", " ", "../a.png", "a/b.png", "a\\b.png", "bad\x00.png", "bad\n.png", strings.Repeat("я", 256)} { + assertError(t, api.upload(t, chat, alice, name, "image/png", data, false), 400, "invalid_request") + } + for _, tc := range []struct { + name, mime string + data []byte + }{ + {"fake", "image/png", []byte("not an image")}, + {"mime mismatch", "image/jpeg", data}, + {"wrong type", "application/octet-stream", data}, + {"truncated", "image/png", data[:80]}, + } { + t.Run(tc.name, func(t *testing.T) { + assertError(t, api.upload(t, chat, alice, "test.png", tc.mime, tc.data, false), 415, "unsupported_media_type") + }) + } + bomb := bytes.Clone(data) + binary.BigEndian.PutUint32(bomb[16:20], 40000001) + binary.BigEndian.PutUint32(bomb[20:24], 1) + binary.BigEndian.PutUint32(bomb[29:33], crc32.ChecksumIEEE(bomb[12:29])) + assertError(t, api.upload(t, chat, alice, "bomb.png", "image/png", bomb, false), 413, "payload_too_large") + // A syntactically valid ancillary PNG chunk reaches the exact byte boundary + // without allocating an enormous decoded image. + padding := make([]byte, 10485760-len(data)-12) + chunk := append([]byte("paDd"), padding...) + boundary := append([]byte{}, data[:len(data)-12]...) + boundary = binary.BigEndian.AppendUint32(boundary, uint32(len(padding))) + boundary = append(boundary, chunk...) + boundary = binary.BigEndian.AppendUint32(boundary, crc32.ChecksumIEEE(chunk)) + boundary = append(boundary, data[len(data)-12:]...) + var boundaryAtt map[string]any + decodeOK(t, api.upload(t, chat, alice, "boundary.png", "image/png", boundary, false), 201, &boundaryAtt) + + oversized := make([]byte, 10485761) + copy(oversized, data) + for _, chunked := range []bool{false, true} { + assertError(t, api.upload(t, chat, alice, "large.png", "image/png", oversized, chunked), 413, "payload_too_large") + } + var jpg bytes.Buffer + if e := jpeg.Encode(&jpg, image.NewRGBA(image.Rect(0, 0, 32, 24)), nil); e != nil { + t.Fatal(e) + } + var jpegAtt map[string]any + decodeOK(t, api.upload(t, chat, bob, "file.jpg", "image/jpeg", jpg.Bytes(), false), 201, &jpegAtt) + imageOnly := `{"text":"","attachment_ids":["` + att.ID + `"]}` + var message map[string]any + key := uuid.NewString() + decodeOK(t, api.requestKey(t, "POST", "/api/v1/chats/"+chat+"/messages", bob, imageOnly, key), 201, &message) + var replay map[string]any + decodeOK(t, api.requestKey(t, "POST", "/api/v1/chats/"+chat+"/messages", bob, imageOnly, key), 201, &replay) + if jsonString(t, message) != jsonString(t, replay) { + t.Fatal("attachment retry changed message") + } + assertError(t, api.requestKey(t, "POST", "/api/v1/chats/"+chat+"/messages", bob, `{"text":"changed","attachment_ids":["`+att.ID+`"]}`, key), 409, "conflict") + assertError(t, api.request(t, "POST", "/api/v1/chats/"+other+"/messages", alice, imageOnly), 404, "not_found") + for _, body := range []string{`{"text":"","attachment_ids":["` + uuid.NewString() + `"]}`} { + assertError(t, api.request(t, "POST", "/api/v1/chats/"+chat+"/messages", alice, body), 404, "not_found") + } + for _, body := range []string{`{"text":""}`, `{"text":" "}`, `{"attachment_ids":["` + att.ID + `"]}`, `{"text":"ok","attachment_ids":null}`, `{"text":"ok","attachment_ids":"bad"}`, `{"text":"ok","attachment_ids":["` + att.ID + `","` + att.ID + `"]}`} { + assertError(t, api.request(t, "POST", "/api/v1/chats/"+chat+"/messages", alice, body), 400, "invalid_request") + } + assertError(t, api.request(t, "GET", content+"?variant=thumbnail", bob, ""), 400, "invalid_request") +} +func TestConcurrentAttachmentMessage(t *testing.T) { + api := newAPI(t) + user := api.createUser(t, "alice") + chat := api.createChat(t, user, user) + var att struct { + ID string `json:"id"` + } + decodeOK(t, api.upload(t, chat, user, "sample.png", "image/png", fixturePNG(t), false), 201, &att) + key := uuid.NewString() + body := `{"text":"","attachment_ids":["` + att.ID + `"]}` + type result struct { + resp apiResponse + err error + } + ch := make(chan result, 16) + var wg sync.WaitGroup + for range 16 { + wg.Go(func() { + resp, e := api.send("POST", "/api/v1/chats/"+chat+"/messages", user, body, key) + ch <- result{resp, e} + }) + } + wg.Wait() + close(ch) + seen := map[string]bool{} + var id string + for item := range ch { + if item.err != nil { + t.Fatal(item.err) + } + var msg struct { + ID string `json:"id"` + IDs []string `json:"attachment_ids"` + } + decodeOK(t, item.resp, 201, &msg) + if id != "" && id != msg.ID { + t.Fatal("duplicate attachment message") + } + id = msg.ID + seen[item.resp.instanceID] = true + if len(msg.IDs) != 1 || msg.IDs[0] != att.ID { + t.Fatal("lost attachment") + } + } + if len(seen) < 2 { + t.Fatal("did not exercise two replicas") + } + var page struct { + Items []json.RawMessage `json:"items"` + } + decodeOK(t, api.request(t, "GET", "/api/v1/chats/"+chat+"/messages", user, ""), 200, &page) + if len(page.Items) != 1 { + t.Fatal("duplicate history") + } +} diff --git a/service/tests/concurrency_test.go b/service/tests/concurrency_test.go new file mode 100644 index 00000000..c48ec8c1 --- /dev/null +++ b/service/tests/concurrency_test.go @@ -0,0 +1,161 @@ +package tests + +import ( + "encoding/json" + "fmt" + "github.com/google/uuid" + "io" + "net/http" + "net/url" + "strings" + "sync" + "testing" + "time" +) + +type createdMessage struct { + ID string `json:"id"` + SenderID string `json:"sender_id"` + Text string `json:"text"` +} + +func postMessage(client *http.Client, endpoint, userID, text string) (createdMessage, error) { + body, err := json.Marshal(struct { + Text string `json:"text"` + }{Text: text}) + if err != nil { + return createdMessage{}, err + } + req, err := http.NewRequest(http.MethodPost, endpoint, strings.NewReader(string(body))) + if err != nil { + return createdMessage{}, err + } + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Authorization", "Bearer "+userID) + req.Header.Set("Idempotency-Key", uuid.NewString()) + resp, err := client.Do(req) + if err != nil { + return createdMessage{}, err + } + defer resp.Body.Close() + data, err := io.ReadAll(resp.Body) + if err != nil { + return createdMessage{}, err + } + if resp.StatusCode != http.StatusCreated { + return createdMessage{}, fmt.Errorf("POST status %d: %s", resp.StatusCode, data) + } + if resp.Header.Get("X-Request-Id") == "" { + return createdMessage{}, fmt.Errorf("POST response has no X-Request-Id") + } + var message createdMessage + if err := json.Unmarshal(data, &message); err != nil { + return createdMessage{}, fmt.Errorf("decode POST response: %w", err) + } + return message, nil +} + +func TestConcurrentMessages(t *testing.T) { + const clients = 10 + const messagesPerClient = 10 + const total = clients * messagesPerClient + + api := newAPI(t) + users := make([]string, clients) + for i := range users { + users[i] = api.createUser(t, fmt.Sprintf("user%02d", i)) + } + chatID := api.createChat(t, users[0], users...) + endpoint := api.baseURL + "/api/v1/chats/" + chatID + "/messages" + + type result struct { + message createdMessage + wantSenderID string + wantText string + err error + } + results := make(chan result, total) + start := make(chan struct{}) + var ready sync.WaitGroup + var workers sync.WaitGroup + ready.Add(clients) + workers.Add(clients) + for i, userID := range users { + go func(workerID int, userID string) { + defer workers.Done() + client := &http.Client{Timeout: 5 * time.Second} + defer client.CloseIdleConnections() + ready.Done() + <-start + for j := range messagesPerClient { + text := fmt.Sprintf("client-%02d-message-%02d", workerID, j) + message, err := postMessage(client, endpoint, api.tokens[userID], text) + results <- result{message: message, wantSenderID: userID, wantText: text, err: err} + } + }(i, userID) + } + ready.Wait() + close(start) + workers.Wait() + close(results) + + created := make(map[string]createdMessage, total) + count := 0 + for item := range results { + count++ + if item.err != nil { + t.Errorf("send %q: %v", item.wantText, item.err) + continue + } + if item.message.ID == "" || item.message.SenderID != item.wantSenderID || item.message.Text != item.wantText { + t.Errorf("unexpected response for %q: %+v", item.wantText, item.message) + continue + } + if _, duplicate := created[item.message.ID]; duplicate { + t.Errorf("duplicate message ID %q", item.message.ID) + continue + } + created[item.message.ID] = item.message + } + if count != total || len(created) != total { + t.Fatalf("sent %d requests, created %d unique messages; want %d", count, len(created), total) + } + + seen := make(map[string]bool, total) + cursor := "" + pages := 0 + for { + path := "/api/v1/chats/" + chatID + "/messages?limit=17" + if cursor != "" { + path += "&cursor=" + url.QueryEscape(cursor) + } + resp := api.request(t, http.MethodGet, path, users[0], "") + var page struct { + Items []createdMessage `json:"items"` + NextCursor *string `json:"next_cursor"` + } + decodeOK(t, resp, http.StatusOK, &page) + pages++ + if pages > total || len(page.Items) == 0 || len(page.Items) > 17 { + t.Fatalf("unexpected page %d: %s", pages, resp.body) + } + for _, message := range page.Items { + want, exists := created[message.ID] + if !exists || seen[message.ID] || message != want { + t.Fatalf("unexpected or repeated message on page %d: %+v", pages, message) + } + seen[message.ID] = true + } + if page.NextCursor == nil { + break + } + if *page.NextCursor == "" || *page.NextCursor == cursor { + t.Fatalf("invalid cursor on page %d: %s", pages, resp.body) + } + cursor = *page.NextCursor + } + if len(seen) != total { + t.Fatalf("history has %d unique messages across %d pages; want %d", len(seen), pages, total) + } + t.Logf("%d clients sent %d messages; history contains %d unique messages across %d pages", clients, total, len(seen), pages) +} diff --git a/service/tests/idempotency_test.go b/service/tests/idempotency_test.go new file mode 100644 index 00000000..9b317fd2 --- /dev/null +++ b/service/tests/idempotency_test.go @@ -0,0 +1,104 @@ +package tests + +import ( + "fmt" + "github.com/google/uuid" + "net/http" + "sync" + "testing" +) + +func TestIdempotencyContract(t *testing.T) { + api := newAPI(t) + alice := api.createUser(t, "alice") + bob := api.createUser(t, "bob") + eve := api.createUser(t, "eve") + chat := api.createChat(t, alice, alice, bob) + other := api.createChat(t, alice, alice) + path := "/api/v1/chats/" + chat + "/messages" + for _, key := range []string{"", "bad key", "кириллица", string(make([]byte, 129))} { + // NUL cannot be sent by net/http: use an overlength ASCII key instead. + if len(key) == 129 { + key = fmt.Sprintf("%0129d", 1) + } + assertError(t, api.requestKey(t, "POST", path, alice, `{"text":"hello"}`, key), 400, "invalid_request") + } + key := uuid.NewString() + first := api.requestKey(t, "POST", path, alice, `{"text":"hello"}`, key) + var want map[string]any + decodeOK(t, first, 201, &want) + var got map[string]any + decodeOK(t, api.requestKey(t, "POST", path, alice, `{ "text" : "hello" }`, key), 201, &got) + if jsonString(t, want) != jsonString(t, got) { + t.Fatal("replay changed message") + } + assertError(t, api.requestKey(t, "POST", path, alice, `{"text":"changed"}`, key), 409, "conflict") + assertError(t, api.requestKey(t, "POST", path, eve, `{"text":"hello"}`, key), 404, "not_found") + for _, tc := range []struct{ user, path string }{{bob, path}, {alice, "/api/v1/chats/" + other + "/messages"}} { + var msg map[string]any + decodeOK(t, api.requestKey(t, "POST", tc.path, tc.user, `{"text":"hello"}`, key), 201, &msg) + if msg["id"] == want["id"] { + t.Fatal("scope not isolated") + } + } + resp := api.request(t, http.MethodDelete, "/api/v1/auth/sessions/current", alice, "") + if resp.status != 204 { + t.Fatal(resp.status) + } + assertError(t, api.requestKey(t, "POST", path, alice, `{"text":"hello"}`, key), 401, "unauthorized") +} +func TestConcurrentIdempotencyAcrossReplicas(t *testing.T) { + api := newAPI(t) + user := api.createUser(t, "alice") + chat := api.createChat(t, user, user) + path := "/api/v1/chats/" + chat + "/messages" + key := uuid.NewString() + const count = 32 + type result struct { + response apiResponse + err error + } + responses := make(chan result, count) + start := make(chan struct{}) + var wg sync.WaitGroup + for range count { + wg.Go(func() { + <-start + response, err := api.send("POST", path, user, `{"text":"one logical message"}`, key) + responses <- result{response: response, err: err} + }) + } + close(start) + wg.Wait() + close(responses) + instances := map[string]bool{} + var id string + for result := range responses { + if result.err != nil { + t.Fatal(result.err) + } + resp := result.response + var msg struct { + ID string `json:"id"` + } + decodeOK(t, resp, 201, &msg) + if id != "" && id != msg.ID { + t.Fatal("duplicate side effect") + } + id = msg.ID + instances[resp.instanceID] = true + } + if len(instances) < 2 { + t.Fatalf("need at least 2 real API replicas, observed %d", len(instances)) + } + var page struct { + Items []struct { + ID string `json:"id"` + } `json:"items"` + } + decodeOK(t, api.request(t, "GET", path, user, ""), 200, &page) + if len(page.Items) != 1 || page.Items[0].ID != id { + t.Fatalf("history contains %d messages", len(page.Items)) + } + t.Logf("%d requests across %d replicas created exactly one message", count, len(instances)) +} -- 2.54.0 From 2198368a44319c893effe72e1ba04a527631fdb5 Mon Sep 17 00:00:00 2001 From: Grbisba Date: Fri, 13 Nov 2026 12:48:18 +0300 Subject: [PATCH 2/2] =?UTF-8?q?=D0=A0=D0=B5=D0=B0=D0=BB=D0=B8=D0=B7=D0=B0?= =?UTF-8?q?=D1=86=D0=B8=D1=8F=20=D0=BD=D0=B0=205?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .dockerignore | 3 + .env.example | 10 +- .github/workflows/application.yml | 28 + .gitignore | 2 + Dockerfile | 3 + Makefile | 19 +- REPORT.md | 178 ++++++- compose.faults.yaml | 22 + compose.yaml | 54 ++ evidence/broker-persistence.json | 10 + evidence/build.json | 14 + evidence/checks.txt | 201 ++++++++ evidence/images.json | 53 ++ evidence/pipeline-final.json | 42 ++ evidence/pipeline.json | 337 ++++++++++++ evidence/storage.json | 24 + infra/imagemagick-policy.xml | 13 + scripts/bootstrap-broker.py | 17 + scripts/bootstrap.sh | 2 + scripts/restart.sh | 6 +- service/cmd/app/main.go | 10 + service/cmd/app/pipeline.go | 138 +++++ service/cmd/app/storage.go | 8 + service/go.mod | 1 + service/go.sum | 2 + service/internal/adapter/broker/rabbitmq.go | 170 ++++++ .../controller/http/attachmentHandler.go | 56 +- .../repository/storage/attachment_scan.go | 7 +- .../adapter/repository/storage/attachments.go | 30 +- .../repository/storage/idempotency_test.go | 12 +- .../adapter/repository/storage/jobs.go | 134 +++++ .../adapter/repository/storage/messages.go | 90 ++-- .../adapter/repository/storage/outbox.go | 52 ++ service/internal/config/pipeline.go | 48 ++ service/internal/dto/attachment.go | 47 ++ service/internal/dto/message.go | 7 + service/internal/gen/contracts/types.gen.go | 16 +- service/internal/pkg/fault/fault.go | 64 +++ service/internal/pkg/imageprocessor/image.go | 83 +++ service/internal/pkg/objectstore/s3.go | 8 + .../internal/service/attachment/interface.go | 51 ++ .../internal/service/attachment/pipeline.go | 486 ++++++++++++++++++ .../internal/service/attachment/reconcile.go | 37 +- .../internal/service/attachment/service.go | 73 +-- service/internal/service/chat/idempotency.go | 54 ++ service/internal/service/chat/interface.go | 7 +- service/internal/service/chat/svc.go | 5 +- service/internal/service/outbox/api.go | 12 + service/internal/service/outbox/models.go | 115 +++++ service/internal/service/outbox/status.go | 16 + service/internal/service/outbox/svc.go | 114 ++++ service/internal/service/outbox/transport.go | 7 + service/migrations/008_async_pipeline.sql | 41 ++ service/migrations/009_variant_manifest.sql | 8 + service/tests/attachments_test.go | 32 +- student-tests/README.md | 14 + student-tests/broker_persistence.py | 47 ++ student-tests/images.py | 53 ++ student-tests/pipeline.py | 179 +++++++ student-tests/storage.py | 62 +++ 60 files changed, 3253 insertions(+), 181 deletions(-) create mode 100644 .github/workflows/application.yml create mode 100644 compose.faults.yaml create mode 100644 evidence/broker-persistence.json create mode 100644 evidence/build.json create mode 100644 evidence/checks.txt create mode 100644 evidence/images.json create mode 100644 evidence/pipeline-final.json create mode 100644 evidence/pipeline.json create mode 100644 evidence/storage.json create mode 100644 infra/imagemagick-policy.xml create mode 100644 scripts/bootstrap-broker.py create mode 100644 service/cmd/app/pipeline.go create mode 100644 service/internal/adapter/broker/rabbitmq.go create mode 100644 service/internal/adapter/repository/storage/jobs.go create mode 100644 service/internal/adapter/repository/storage/outbox.go create mode 100644 service/internal/config/pipeline.go create mode 100644 service/internal/pkg/fault/fault.go create mode 100644 service/internal/pkg/imageprocessor/image.go create mode 100644 service/internal/service/attachment/interface.go create mode 100644 service/internal/service/attachment/pipeline.go create mode 100644 service/internal/service/chat/idempotency.go create mode 100644 service/internal/service/outbox/api.go create mode 100644 service/internal/service/outbox/models.go create mode 100644 service/internal/service/outbox/status.go create mode 100644 service/internal/service/outbox/svc.go create mode 100644 service/internal/service/outbox/transport.go create mode 100644 service/migrations/008_async_pipeline.sql create mode 100644 service/migrations/009_variant_manifest.sql create mode 100644 student-tests/README.md create mode 100644 student-tests/broker_persistence.py create mode 100644 student-tests/images.py create mode 100644 student-tests/pipeline.py create mode 100644 student-tests/storage.py diff --git a/.dockerignore b/.dockerignore index bc16e556..19e41e74 100644 --- a/.dockerignore +++ b/.dockerignore @@ -18,3 +18,6 @@ __pycache__ node_modules evidence .DS_Store + +.lab5-faults +bin diff --git a/.env.example b/.env.example index ecbfeb2a..b0567a98 100644 --- a/.env.example +++ b/.env.example @@ -30,9 +30,11 @@ RECONCILE_GRACE_PERIOD=1h # SeaweedFS image and persistent bucket bootstrap are pinned in compose.yaml. # API has four upload slots and one decoder per replica, GOMEMLIMIT=384MiB. -# Planned lab5 async pipeline settings; the ported lab4 application does not use these yet. -BROKER_URL_FILE=/run/secrets/broker_url +# Async lab5 pipeline. Broker credentials are generated by bootstrap. +BROKER_PASSWORD=replace-with-local-broker-password WORKER_CONCURRENCY=2 +WORKER_PREFETCH=2 JOB_MAX_ATTEMPTS=5 -JOB_TIMEOUT_SECONDS=60 -STAGING_RETENTION_SECONDS=86400 +JOB_TIMEOUT=60s +JOB_LEASE=90s +STAGING_RETENTION=24h diff --git a/.github/workflows/application.yml b/.github/workflows/application.yml new file mode 100644 index 00000000..0ae34271 --- /dev/null +++ b/.github/workflows/application.yml @@ -0,0 +1,28 @@ +name: Application +on: [push, pull_request] +permissions: + contents: read +jobs: + application: + runs-on: ubuntu-latest + timeout-minutes: 25 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 + with: + go-version: '1.27.2' + cache-dependency-path: service/go.sum + - run: make check build vet + - run: make up + - run: make test test-go test-storage test-images test-staging test-broker-persistence + - run: make test-persistence ACTION_SCRIPT=scripts/restart.sh + - run: make test-failover ACTION_SCRIPT=scripts/stop-one.sh + - run: make restore + - run: make test-pipeline + - uses: actions/upload-artifact@v4 + if: always() + with: + name: pipeline-evidence + path: evidence/pipeline.json + - run: make down + if: always() diff --git a/.gitignore b/.gitignore index 73a022b1..01540a78 100644 --- a/.gitignore +++ b/.gitignore @@ -31,3 +31,5 @@ coverage/ # Keep small, sanitized evidence summaries in evidence/ under version control. # Inspect git diff --cached: ignore rules are not a secret scanner. /.ignore/ + +.lab5-faults/ diff --git a/Dockerfile b/Dockerfile index 38980df9..a953e599 100644 --- a/Dockerfile +++ b/Dockerfile @@ -16,6 +16,9 @@ RUN CGO_ENABLED=0 GOOS=$TARGETOS GOARCH=$TARGETARCH \ FROM alpine:3.23.6@sha256:85fe1e81d6758c208f3e1eed4338a1997e19d4be002d4dd32d3100c9a8c010a0 +RUN apk add --no-cache imagemagick=7.1.2.35-r0 imagemagick-webp=7.1.2.35-r0 imagemagick-jpeg=7.1.2.35-r0 +COPY infra/imagemagick-policy.xml /etc/ImageMagick-7/policy.xml + COPY --from=build --chmod=0555 /out/messenger /usr/local/bin/messenger ENV APP_HOST=0.0.0.0 \ diff --git a/Makefile b/Makefile index db8c42a0..55b3c450 100644 --- a/Makefile +++ b/Makefile @@ -13,8 +13,8 @@ OAPI_CODEGEN ?= $(shell $(GO) env GOPATH)/bin/oapi-codegen .PHONY: help check validate test load test-persistence test-failover openapi-generate certs key-gen create-dir gen-private-key gen-public-key gen-key gen-crt help: - @echo "up/down: lab5 stand (ported lab4 application); build/vet: Go checks" - @echo "test: lab5 API smoke (requires async implementation); test-go/test-storage: existing Go checks" + @echo "up/down: lab5 async image pipeline; build/vet: Go checks" + @echo "test: lab5 API smoke; test-go/test-storage: Go checks" @echo "restart/restore/rolling-update: infrastructure actions; benchmark/load: synthetic traffic" @echo "check/validate: lab5 contracts; test-persistence/test-failover: explicit ACTION_SCRIPT hooks" @echo "reconcile/reconcile-apply: storage consistency" @@ -111,3 +111,18 @@ reconcile: reconcile-apply: ./scripts/compose.sh run --rm -T operator storage reconcile --apply + +.PHONY: test-pipeline +test-pipeline: + $(PYTHON) student-tests/pipeline.py --base-url "$(BASE_URL)" --repeats 3 --output evidence/pipeline.json + +.PHONY: test-images test-staging +test-images: + $(PYTHON) student-tests/images.py + +test-staging: + $(PYTHON) student-tests/storage.py + +.PHONY: test-broker-persistence +test-broker-persistence: + $(PYTHON) student-tests/broker_persistence.py diff --git a/REPORT.md b/REPORT.md index e98f75e9..489af1ec 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,37 +1,173 @@ # Отчёт — лабораторная 5 -- Автор, группа: TODO -- Целевая оценка; выбранный трек 5, если нужен: TODO -- Commit/PR: TODO после публикации -- Использованные источники, библиотеки и помощь ИИ: TODO +- Автор, группа: не предоставлены владельцем проекта. +- Целевая оценка: **5**, трек 2 — доказанная идемпотентность versioned pipeline. +- Ревизия: базовый commit и состояние рабочего дерева записываются в `evidence/pipeline.json`; решение пока не закоммичено. +- Использованы Go, pgx, AWS SDK, RabbitMQ `amqp091-go`, ImageMagick; ## Запуск из чистого clone -TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных. +Требуются Docker Compose и для локальных Go-проверок Go 1.27+. Контейнерная сборка использует Go 1.27.2. Образы закреплены версиями/digest в `compose.yaml` и `Dockerfile`; ImageMagick и модули JPEG/WebP — 7.1.2.35-r0. -## Архитектура и решения +```sh +make check build vet +make up +curl --fail http://localhost:18085/health/ready +make test +make test-go +make test-storage +make test-images test-staging test-broker-persistence +make test-persistence ACTION_SCRIPT=scripts/restart.sh +make test-failover ACTION_SCRIPT=scripts/stop-one.sh +make restore +make test-pipeline +make down +``` -TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab4, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации. +Полная валидация схем: + +```sh +python3 -m venv .venv +.venv/bin/python -m pip install -r requirements-dev.txt +make validate PYTHON=.venv/bin/python +``` + +Bootstrap сохраняет существующие локальные настройки и генерирует синтетические credentials PostgreSQL, Redis, S3 и RabbitMQ. `.env.lab5` и `secrets/lab5` игнорируются Git. Миграции выполняются отдельным одноразовым процессом, bucket создаётся идемпотентно. Внешний endpoint — только `127.0.0.1:18085`; внутренние хранилища портов на host не публикуют. API-реплик по умолчанию две. + +`make down` сохраняет volumes. **Намеренное удаление учебных данных**: `./scripts/compose.sh down --volumes`; эта команда не входит в обычные проверки и автоматически не выполняется. + +## Архитектура и границы + +```mermaid +flowchart LR + Client --> LB[HAProxy] + LB --> API[API x2] + API --> Staging[Private S3 staging] + API --> DB[PostgreSQL: metadata + jobs + outbox] + DB --> Dispatcher + Dispatcher --> Broker[RabbitMQ durable queue] + Broker --> Worker + Worker --> Results[Private S3 versioned results] + Worker --> DB +``` + +Handler разбирает HTTP, проверяет поля и ограничивает входной поток. Attachment service принимает DTO и reader, проверяет доступ и целостность изображения, управляет транзакцией и состояниями. Репозитории выполняют операции хранения; HTTP, retry-политик и преобразования изображений в SQL/триггерах нет. Канонизация тела сообщения, сравнение идемпотентных запросов и создание Message перенесены в chat service. + +До `202` весь проверенный оригинал уже записан в S3 staging, а metadata, job и outbox — в одной завершённой транзакции PostgreSQL. При неопределённом COMMIT нет слепого компенсирующего DELETE. Ответ содержит сохранённый снимок `queued`, даже если быстрый worker уже начал работу. + +Staging находится в том же S3-сервисе, что конечные объекты: это разделение prefix и жизненных циклов, а не обещание отложить всю сетевую передачу файла. Проверка корректности изображения при upload включает ограниченное декодирование; resize/crop/WebP и публикация конечного оригинала выполняются отдельно в worker. + +API readiness проверяет PostgreSQL и S3. Недоступность broker/worker не блокирует транзакционный приём через outbox. Worker и dispatcher переподключаются после отказа зависимостей. Redis остаётся необязательным кэшем, очередь в него не помещается. + +### Инфраструктура + +| Роль → service | Версия | Проверка | Volume / сеть / credentials | +| --- | --- | --- | --- | +| API → api, ≥2 | собственная сборка Go 1.27.2 | `/health/ready` | без локального durable state; default + storage; signing/S3/broker files, DB env | +| PostgreSQL → postgres | 17.11, digest в Compose | pg_isready | postgres_data; default; локальный пароль env | +| Кэш → redis | 8.2.11, digest в Compose | authenticated PING | redis_data, AOF; default; локальный пароль env | +| LB → lb | HAProxy 3.2, digest в Compose | backend readiness | без volume данных; default; публичный localhost порт | +| S3 → s3 | SeaweedFS 4.52, digest в Compose | cluster status; bootstrap проверяет запись | s3_data; internal storage; credentials JSON file | +| Broker → broker | RabbitMQ 4.2.0, digest в Compose | rabbitmq-diagnostics check_running | broker_data; default; синтетические credentials | +| Worker → worker | образ приложения + ImageMagick | процесс/reconnect, состояние jobs | эфемерные ограниченные temp files; default + storage; secret files | +| Dispatcher → dispatcher | образ приложения | процесс/reconnect, состояние outbox | без локального durable state; default; broker secret file | + +У worker/dispatcher отключён унаследованный HTTP healthcheck: HTTP-сервер они не запускают. Состояние `running` само по себе не подтверждает прогресс; проверять нужно jobs/outbox и end-to-end сценарий. + +### Доставка и повторы + +- Durable RabbitMQ queue, persistent messages, publisher confirms и mandatory routing. Имя broker фиксировано, чтобы новый контейнер открывал ту же persistent node database; readiness ждёт запуска RabbitMQ application, а не только Erlang node. Consumer ack выполняется после устойчивого результата, постановки следующей попытки либо записи terminal failed. +- Сообщения соответствуют `jobs.schema.json`: UUID, версия, attempt, timestamp, опциональный trace carrier. Байтов файла, URL и credentials в них нет. Worker читает доверенные storage metadata из PostgreSQL. +- `job_id` стабилен для вложения и версии. Redelivery сама не увеличивает attempt; управляемый retry/recovery увеличивает его. По умолчанию пять попыток, exponential backoff с jitter 0.5–30 секунд в зависимости от номера попытки. +- Publisher имеет 20-секундный deadline на самом socket (одного контекста PublishWithContext недостаточно для ограничения сетевой записи). Dispatcher захватывает одну запись с lease 30 секунд, публикует её и фиксирует результат с проверкой token. Просроченный dispatcher не может изменить запись нового владельца. После пяти неудачных публикаций остаётся dead_letter-запись и наблюдаемый failed job. +- Worker: concurrency=2, prefetch=2, timeout=60s, lease=90s. Lease recovery работает независимо от consumer. Неактивный lease не продлевается: работа дольше лимита повторяется ограниченно. +- Постоянная ошибка заканчивается немедленно (`invalid_image`, `source_missing`, `source_corrupt`); временная повторяется и после исчерпания бюджета даёт `retry_exhausted`. API получает только безопасный error_code. +- Реестр failed — PostgreSQL. Replay явный, только для текущего failed job с доступным source; записывается аудит, номер attempt продолжает расти, выдаётся новый бюджет из пяти попыток. Уже успешную job replay не запускает. + +```sh +./scripts/compose.sh run --rm -T operator jobs list +./scripts/compose.sh run --rm -T operator jobs replay --id JOB_UUID +./scripts/compose.sh run --rm -T operator outbox replay --id JOB_UUID +./scripts/compose.sh run --rm -T operator jobs reprocess --id ATTACHMENT_UUID --pipeline-version 2 +``` + +`outbox replay` восстанавливает логическую failed job вместе с новой outbox-записью; ID аргумента — job, не отдельная доставка. Старые dead_letter-записи сохраняются для диагностики и не переводят новую попытку обратно в failed. + +### Версии и инварианты + +Manifest имеет PK `(attachment_id, pipeline_version, variant)`. Варианты публикуются одной DB-транзакцией только после записи всех объектов. Завершение проверяет текущую версию вложения, состояние job, token и действующий lease. Ready не откатывается из-за старой доставки. + +Объектные ключи включают вложение, версию, variant и SHA-256. Повтор одинакового эффекта использует тот же ключ; другая версия пишет в свой namespace. Поэтому устаревшая попытка не перезапишет новые байты S3, даже когда БД отклоняет её завершение. Незавершённые объекты возможны и удаляются как сироты после grace; они не становятся опубликованным вторым набором вариантов. + +Версия 1 использует WebP quality 80, версия 2 — 85. Остальные параметры одинаковы. Reprocess доступен для ready-вложения и строго большей поддерживаемой версии; на время новой работы статус queued и content=409. Сохраняются прежние manifest, новый оригинал побайтно совпадает с исходником. + +### Изображения и cleanup + +Thumbnail: JPEG, вписан в 256×256 без увеличения, размеры округляются вниз с минимумом 1 px. Crop: центральный квадрат JPEG 128×128 с разрешённым увеличением. Optimized: WebP внутри 1280×1280 без увеличения. Применяется EXIF orientation, alpha компонуется на белый, метаданные производных удаляются. JPEG quality=85. Не обещается уменьшение каждого файла. + +ImageMagick запускается без shell, отдельным дочерним процессом с отменой по timeout. Политика запрещает delegates/filters и разрешает только нужные coders; memory=256MiB, map=0, disk=256MiB, thread=1, time=60s, list-length=4. Worker ограничен 1 CPU/1GiB; его локальные файлы — только восстанавливаемые временные данные. API принимает максимум четыре одновременных upload и выполняет одну проверку декодирования на реплику. Пиксели проверяются до полной распаковки. Превышение ресурсов заканчивается ошибкой, а не бесконечным retry. + +Staging удаляется после устойчивого ready; failed source хранится 24h. Источники, которыми являются исторические опубликованные results, физически сохраняются. После удаления failed source replay отклоняется; оператор может восстановить оригинал с проверкой размера/SHA и затем явно повторить job. Reconcile защищает ссылки metadata/manifest и активные jobs, удаляет сироты только после grace=1h (>2×upload timeout). Старые originals lab4 переводятся миграцией на первую версию pipeline без изменения ID и ссылок сообщений. ## Доказательства критериев -| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | -| --- | --- | --- | --- | -| Публичный smoke | make test | TODO | TODO | -| Остальные пункты уровня 3 | TODO | TODO | TODO | -| Все пункты уровня 4, если заявлен | TODO | TODO | TODO | -| Выбранный трек 5, если заявлен | TODO | TODO | TODO | +Фактический вывод команд сохраняется отдельно от описания реализации. `evidence/pipeline.json` содержит повторяемые сценарии, UUID, длительности и результат каждого запуска; `evidence/images.json` — проверки изображений. Проверки защиты staging, удаления после ready, retention failed и восстановления исходника — `evidence/storage.json`. Остальные результаты — `evidence/checks.txt`. -TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены. +| Критерий | Команда / наблюдение | Доказательство | +| --- | --- | --- | +| Структура, схемы, сборка и анализ | make check; make validate; make build; make vet | checks.txt | +| Публичный контракт и предыдущие лабораторные | make test; make test-go; make test-storage | checks.txt | +| Сессия и данные после перезапуска; отказ API | test-persistence / test-failover с действительными hooks | checks.txt | +| 202 при остановленном worker, сообщение с queued-вложением, рестарт stores/broker и ready | worker_down_and_store_restart | pipeline.json | +| Опубликованное сообщение переживает пересоздание broker с тем же volume при остановленном dispatcher | make test-broker-persistence; hostname=broker сохраняет имя узла | broker-persistence.json | +| Thumbnail/crop/WebP, orientation, alpha, реальные размеры и hashes | student-tests/images.py | images.json | +| Сбой до/после S3 side effect и до/после COMMIT результата, до ack | crash_before_objects / after_original_put / after_objects / after_result_commit / before_ack | pipeline.json | +| Confirm получен, outbox ещё processing; повтор публикации безопасен | crash_confirm_before_outbox_mark | pipeline.json | +| Ready redelivery — no-op | ready_redelivery, 3 копии | pipeline.json | +| Реальный lease expiry, более новая попытка/версия, возобновление старого worker и поздняя старая доставка | lease_reorder_and_stale_version | pipeline.json | +| Staging защищён при queued; после ready удалён; failed retention и restore проверены | make test-staging | storage.json | +| Retry ограничен, failed наблюдаем, replay явный | bounded_retry_and_explicit_replay / permanent_error | pipeline.json | +| Сбой после staging и разрыв COMMIT/publish | api_crash_after_staging / api_crash_after_upload_commit | pipeline.json | -## Эксперименты и отказы +## Эксперимент трека 2 -TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения. +**Гипотеза:** атомарное сохранение намерения, durable доставка, fencing в БД и разделение версий/контента в S3 сохраняют принятые задания и предотвращают публикацию устаревшего эффекта при проверяемых crash/reorder/duplicate сценариях. -| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | -| --- | --- | --- | --- | -| TODO | TODO | TODO | TODO | +Методика: `make test-pipeline` включает исключительно для эксперимента `compose.faults.yaml`. Директивы в игнорируемом `.lab5-faults/` адресуются по job/attachment ID и точке сбоя. Барьер отмечает достижение точки; crash завершает процесс с кодом 86, restart policy поднимает его снова. Pause применяется один раз: другая попытка может завершиться после реального истечения lease. Общая директория не содержит токенов и очищается при завершении. -## Ограничения и следующие шаги +В эксперименте timeout=5s, lease=8s, concurrency/prefetch=2; остальные параметры не ускоряются искусственным изменением lease в БД. Окно после publisher confirm использует обычный outbox lease=30s. Для гарантированного повторного publish тест повторно делает свою outbox-запись pending; это внесение дубля, а не штатная операция пользователя. Для COMMIT/publish dispatcher заранее остановлен, проверяется pending-запись. Для crash до COMMIT отсутствие `202` не трактуется как потеря принятой работы. -TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза. +Каждый сценарий выполняется три раза с новыми синтетическими пользователями/вложениями. Основной файл — публичный PNG fixture 320×200, 111817 bytes; дополнительных параллельных upload в fault-эксперименте нет. Измеряется длительность сценария (включает Docker stop/start и проверки), а не чистая latency обработки. Точность таймеров — monotonic clock клиента; нагрузочный максимум этим экспериментом не измеряется. + +Окружение выполнения: Apple M4 Pro, 14 CPU, host RAM 24GiB; Docker Desktop arm64, 14 CPU и около 7.75GiB RAM. CPU/RAM контейнеров ограничены Compose. Исследование проводится на одном Docker-host с сохранением volumes, без симуляции потери диска. + + +### Наблюдавшиеся результаты + +Все **39/39** сценариев трека завершились PASS (13 сценариев × 3 повтора). Интервалы ниже включают действия Docker и проверки; это не измерение чистого processing latency. + +| Сценарий | Минимум–максимум, с | +| --- | --- | +| worker_down_and_store_restart | 63.864–66.563 | +| crash_before_objects | 10.888–12.269 | +| crash_after_original_put | 11.222–12.171 | +| crash_after_objects | 11.174–11.340 | +| crash_after_result_commit | 1.973–2.088 | +| crash_before_ack | 2.017–2.098 | +| ready_redelivery | 2.435–3.002 | +| crash_confirm_before_outbox_mark | 33.988–36.021 | +| lease_reorder_and_stale_version | 14.652–15.225 | +| bounded_retry_and_explicit_replay | 13.737–15.238 | +| permanent_error | 2.552–2.699 | +| api_crash_after_staging | 0.144–0.817 | +| api_crash_after_upload_commit | 2.141–2.328 | + +После эксперимента отдельная проверка retention обнаружила и позволила исправить гонку восстановления исходника: Restore обновляет возраст source, а Cleanup берёт тот же advisory lock. Эта корректировка проверена `make test-staging`. После добавления socket deadline повторно выполнены confirm/crash, lease + stale version и COMMIT/publish на итоговом образе: **3/3 PASS**, см. `evidence/pipeline-final.json`. Image ID исходного полного fault-прогона сохранён в pipeline.json, итогового — в pipeline-final.json. + +## Ограничения + +At-least-once доставка сохраняется: publish/confirm/COMMIT/ack не образуют одну распределённую транзакцию. Доказывается идемпотентность опубликованного эффекта в перечисленных сценариях, а не exactly-once доставка и не отсутствие любых возможных отказов. + +Один PostgreSQL, S3, broker, LB и host — отдельные точки отказа. Durable очередь здесь не означает кластер RabbitMQ. Невосстановимая потеря volumes не покрывается; backup/restore инфраструктуры не реализован. После окончательной ошибки нужен операторский replay; при длительном отказе брокера принятые задания могут стать наблюдаемыми failed. Накопление outbox/jobs не ограничено глобальной квотой: ограничены upload, worker concurrency/prefetch и ресурсы, но отдельный трек перегрузки не заявляется. + +Источники: [RabbitMQ reliability](https://www.rabbitmq.com/docs/reliability), [confirms/acks](https://www.rabbitmq.com/docs/confirms), [Go client](https://github.com/rabbitmq/amqp091-go), [ImageMagick resource policy](https://imagemagick.org/resources/). diff --git a/compose.faults.yaml b/compose.faults.yaml new file mode 100644 index 00000000..75f47d85 --- /dev/null +++ b/compose.faults.yaml @@ -0,0 +1,22 @@ +# Explicit isolated experiment profile. Never enabled by make up. +services: + api: + environment: + LAB5_FAULTS: "1" + volumes: + - ./.lab5-faults:/faults + worker: + environment: + LAB5_FAULTS: "1" + JOB_TIMEOUT: 5s + JOB_LEASE: 8s + volumes: + - ./.lab5-faults:/faults + dispatcher: + environment: + LAB5_FAULTS: "1" + volumes: + - ./.lab5-faults:/faults + operator: + environment: + LAB5_FAULTS: "1" diff --git a/compose.yaml b/compose.yaml index 217cce67..71bc0cac 100644 --- a/compose.yaml +++ b/compose.yaml @@ -40,6 +40,13 @@ x-api: &api S3_BUCKET: ${S3_BUCKET:-messenger-attachments} S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret + BROKER_URL_FILE: /run/secrets/broker_url + WORKER_CONCURRENCY: ${WORKER_CONCURRENCY:-2} + WORKER_PREFETCH: ${WORKER_PREFETCH:-2} + JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5} + JOB_TIMEOUT: ${JOB_TIMEOUT:-60s} + JOB_LEASE: ${JOB_LEASE:-90s} + STAGING_RETENTION: ${STAGING_RETENTION:-24h} MAX_UPLOAD_BYTES: ${MAX_UPLOAD_BYTES:-10485760} MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS:-40000000} UPLOAD_TIMEOUT: ${UPLOAD_TIMEOUT:-120s} @@ -47,6 +54,7 @@ x-api: &api GOMEMLIMIT: 384MiB networks: [default, storage] volumes: + - ./secrets/lab5/broker_url:/run/secrets/broker_url:ro - ./secrets/lab5/private_key.pem:/run/certs/private_key.pem:ro - ./secrets/lab5/public_key.pem:/run/certs/public_key.pem:ro - ./secrets/lab5/s3_api_access:/run/secrets/s3_access:ro @@ -78,6 +86,50 @@ services: depends_on: postgres: condition: service_healthy + broker: + hostname: broker + image: rabbitmq:4.2.0-alpine@sha256:32962eea9bf99928c2fc541a7defb78c7feca9df0434c5dfb7b539d61e1af524 + restart: unless-stopped + cpus: 0.5 + mem_limit: 512m + environment: + RABBITMQ_DEFAULT_USER: messenger + RABBITMQ_DEFAULT_PASS: ${BROKER_PASSWORD:?Run scripts/bootstrap.sh} + volumes: + - broker_data:/var/lib/rabbitmq + healthcheck: + test: [CMD, rabbitmq-diagnostics, -q, check_running] + interval: 3s + timeout: 5s + retries: 30 + stop_grace_period: 30s + worker: + <<: *api + command: [worker] + cpus: 1 + mem_limit: 1g + stop_grace_period: 10s + healthcheck: + disable: true + depends_on: + migrate: + condition: service_completed_successfully + s3-init: + condition: service_completed_successfully + broker: + condition: service_healthy + dispatcher: + <<: *api + command: [dispatcher] + cpus: 0.25 + mem_limit: 128m + healthcheck: + disable: true + depends_on: + migrate: + condition: service_completed_successfully + broker: + condition: service_healthy lb: image: ${HAPROXY_IMAGE:-haproxy:3.2-alpine@sha256:96ec3c75a9264ba94e733eeacb44051ed6e7b79bb77a73c2c36f236b6423f1f0} restart: unless-stopped @@ -168,6 +220,7 @@ services: - ./secrets/lab5/s3_admin_access:/run/secrets/s3_access:ro - ./secrets/lab5/s3_admin_secret:/run/secrets/s3_secret:ro - ./tests/fixtures:/fixtures:ro + - ./secrets/lab5/broker_url:/run/secrets/broker_url:ro healthcheck: disable: true depends_on: {} @@ -176,6 +229,7 @@ volumes: postgres_data: redis_data: s3_data: + broker_data: networks: storage: diff --git a/evidence/broker-persistence.json b/evidence/broker-persistence.json new file mode 100644 index 00000000..84aa504c --- /dev/null +++ b/evidence/broker-persistence.json @@ -0,0 +1,10 @@ +{ + "result": "PASS", + "attachment_id": "cb1e8982-980d-4286-8224-335b644a5fc5", + "broker_hostname": "broker", + "old_container_id": "16848707eae7bdfb195da43c912ad66399678cc0a48d100bea119619f7e62009", + "new_container_id": "0dfeb1f83d39adc4de9c7798d65b5f125927fe95ae050e4a17e8a5238713b81b", + "message_present_before_and_after": true, + "outbox_published_before_replacement": true, + "dispatcher_stopped_during_replacement": true +} diff --git a/evidence/build.json b/evidence/build.json new file mode 100644 index 00000000..bdc85a14 --- /dev/null +++ b/evidence/build.json @@ -0,0 +1,14 @@ +{ + "date_utc": "2026-11-13T09:29:53.527687+00:00", + "base_commit": "ce035c85282e75513ea36cb429f4b8d775398d62", + "working_tree": "uncommitted solution", + "runtime_source_sha256": "cadfe8fc0d834a598d26e03d1c0348c907ac6fe9a4574aa959f156dfd69488d6", + "hash_method": "Sorted relative path + NUL + file content + NUL, for service/**/*.go, migrations/*.sql, go.mod, go.sum, Dockerfile and ImageMagick policy", + "image_id": "sha256:dc259d5fbee064cac712f13a42c074c2df34ddc74957266d41a75d6f5e631b4f", + "go_local": "go version go1.27.0 darwin/arm64", + "docker_server": "29.7.2", + "compose": "Docker Compose version v5.3.1", + "docker_resources": "cpus=14 memory_bytes=8320954368 arch=aarch64", + "hardware": "Apple M4 Pro, 14 CPUs, 24 GiB host RAM", + "source_reuse": "Adapted core outbox from supplied mamont-optics project; original project left unchanged" +} diff --git a/evidence/checks.txt b/evidence/checks.txt new file mode 100644 index 00000000..faa5a766 --- /dev/null +++ b/evidence/checks.txt @@ -0,0 +1,201 @@ +Lab5 verification, UTC 2026-11-13T09:29:53.527687+00:00 +All commands below completed with exit code 0. +Go output excerpts omit SQL, runtime logs and synthetic request bodies. + +$ make check build vet; make validate PYTHON=.venv/bin/python +python3 scripts/check_contract.py +PASS: offline skeleton structure, local refs and Python syntax; application NOT tested. +cd service && go build ./... +cd service && go build -o ../bin/benchmark ./cmd/benchmark +cd service && go vet ./... +.venv/bin/python scripts/validate_openapi.py +PASS: full OpenAPI 3.1 and JSON Schema validation; application NOT tested. + +$ make test +python3 tests/smoke.py --base-url "http://localhost:18085" +PASS: lab5 public API scenario; infrastructure and grading evidence remain separate. + +$ make test-go (race enabled) +? gitea.privcloud.tech/mtusi2026/cmd/app [no test files] +=== RUN TestLostPostResponseIsNotRetried +--- PASS: TestLostPostResponseIsNotRetried (0.00s) +PASS +ok gitea.privcloud.tech/mtusi2026/cmd/benchmark 1.461s +? gitea.privcloud.tech/mtusi2026/internal/adapter/broker [no test files] +? gitea.privcloud.tech/mtusi2026/internal/adapter/controller/http [no test files] +? gitea.privcloud.tech/mtusi2026/internal/adapter/repository/cache/outside [no test files] +=== RUN TestIdempotencyTransactionAndExpiry +PASS +ok gitea.privcloud.tech/mtusi2026/internal/adapter/repository/storage 1.653s +? gitea.privcloud.tech/mtusi2026/internal/config [no test files] +? gitea.privcloud.tech/mtusi2026/internal/dto [no test files] +? gitea.privcloud.tech/mtusi2026/internal/gen/contracts [no test files] +? gitea.privcloud.tech/mtusi2026/internal/pkg/apperr [no test files] +? gitea.privcloud.tech/mtusi2026/internal/pkg/auth/jwtprovider [no test files] +? gitea.privcloud.tech/mtusi2026/internal/pkg/cache [no test files] +=== RUN TestSessionFingerprints +--- PASS: TestSessionFingerprints (0.02s) +PASS +ok gitea.privcloud.tech/mtusi2026/internal/pkg/encrypting 1.908s +? gitea.privcloud.tech/mtusi2026/internal/pkg/fault [no test files] +? gitea.privcloud.tech/mtusi2026/internal/pkg/imageprocessor [no test files] +? gitea.privcloud.tech/mtusi2026/internal/pkg/migrator [no test files] +? gitea.privcloud.tech/mtusi2026/internal/pkg/objectstore [no test files] +? gitea.privcloud.tech/mtusi2026/internal/pkg/postgres [no test files] +? gitea.privcloud.tech/mtusi2026/internal/pkg/postgres/txmanager [no test files] +? gitea.privcloud.tech/mtusi2026/internal/pkg/rest/framework [no test files] +? gitea.privcloud.tech/mtusi2026/internal/pkg/rest/headers [no test files] +=== RUN TestVerifiedSessionIsRequestScoped +--- PASS: TestVerifiedSessionIsRequestScoped (0.00s) +=== RUN TestMissingTokenDoesNotCallSessionStore +--- PASS: TestMissingTokenDoesNotCallSessionStore (0.00s) +PASS +ok gitea.privcloud.tech/mtusi2026/internal/pkg/rest/middleware 2.467s +? gitea.privcloud.tech/mtusi2026/internal/pkg/rest/router [no test files] +? gitea.privcloud.tech/mtusi2026/internal/pkg/types [no test files] +? gitea.privcloud.tech/mtusi2026/internal/pkg/types/authprovider [no test files] +? gitea.privcloud.tech/mtusi2026/internal/pkg/types/environment [no test files] +? gitea.privcloud.tech/mtusi2026/internal/pkg/types/role [no test files] +? gitea.privcloud.tech/mtusi2026/internal/pkg/validator [no test files] +? gitea.privcloud.tech/mtusi2026/internal/service/attachment [no test files] +? gitea.privcloud.tech/mtusi2026/internal/service/auth [no test files] +? gitea.privcloud.tech/mtusi2026/internal/service/chat [no test files] +? gitea.privcloud.tech/mtusi2026/internal/service/outbox [no test files] +? gitea.privcloud.tech/mtusi2026/internal/service/user [no test files] +? gitea.privcloud.tech/mtusi2026/migrations [no test files] +=== RUN TestInvalidJSON +=== RUN TestInvalidJSON/malformed +=== RUN TestInvalidJSON/wrong_type +=== RUN TestInvalidJSON/unknown_field +--- PASS: TestInvalidJSON (0.01s) + --- PASS: TestInvalidJSON/malformed (0.01s) + --- PASS: TestInvalidJSON/wrong_type (0.00s) + --- PASS: TestInvalidJSON/unknown_field (0.00s) +=== RUN TestForeignChatIsHidden +--- PASS: TestForeignChatIsHidden (0.54s) +=== RUN TestEmptyMessage +--- PASS: TestEmptyMessage (0.21s) +=== RUN TestDuplicateUsername +--- PASS: TestDuplicateUsername (0.28s) +=== RUN TestMessagePagination +--- PASS: TestMessagePagination (0.22s) +=== RUN TestAttachmentContractAndErrors +=== RUN TestAttachmentContractAndErrors/fake +=== RUN TestAttachmentContractAndErrors/mime_mismatch +=== RUN TestAttachmentContractAndErrors/wrong_type +=== RUN TestAttachmentContractAndErrors/truncated +--- PASS: TestAttachmentContractAndErrors (1.05s) + --- PASS: TestAttachmentContractAndErrors/fake (0.00s) + --- PASS: TestAttachmentContractAndErrors/mime_mismatch (0.00s) + --- PASS: TestAttachmentContractAndErrors/wrong_type (0.00s) + --- PASS: TestAttachmentContractAndErrors/truncated (0.00s) +=== RUN TestConcurrentAttachmentMessage +--- PASS: TestConcurrentAttachmentMessage (0.25s) +=== RUN TestConcurrentMessages +--- PASS: TestConcurrentMessages (1.84s) +=== RUN TestIdempotencyContract +--- PASS: TestIdempotencyContract (0.45s) +=== RUN TestConcurrentIdempotencyAcrossReplicas +--- PASS: TestConcurrentIdempotencyAcrossReplicas (0.22s) +PASS +ok gitea.privcloud.tech/mtusi2026/tests 7.261s + +$ make test-storage (isolated database) +=== RUN TestIdempotencyTransactionAndExpiry +=== RUN TestIdempotencyTransactionAndExpiry/service_owns_message_content_rules +=== RUN TestIdempotencyTransactionAndExpiry/service_owns_message_content_rules/empty +=== RUN TestIdempotencyTransactionAndExpiry/service_owns_message_content_rules/whitespace +=== RUN TestIdempotencyTransactionAndExpiry/service_owns_message_content_rules/invalid_UTF8 +=== RUN TestIdempotencyTransactionAndExpiry/service_owns_message_content_rules/nul +=== RUN TestIdempotencyTransactionAndExpiry/service_owns_message_content_rules/too_long +=== RUN TestIdempotencyTransactionAndExpiry/service_owns_message_content_rules/duplicate +=== RUN TestIdempotencyTransactionAndExpiry/service_owns_message_content_rules/too_many +=== RUN TestIdempotencyTransactionAndExpiry/service_owns_message_content_rules/nil_id +=== RUN TestIdempotencyTransactionAndExpiry/service_owns_message_content_rules/unknown +=== RUN TestIdempotencyTransactionAndExpiry/service_owns_message_content_rules/foreign_chat +=== RUN TestIdempotencyTransactionAndExpiry/service_owns_message_content_rules/missing_object +=== RUN TestIdempotencyTransactionAndExpiry/object_lock_cancellation_does_not_leak_a_pooled_lock +--- PASS: TestIdempotencyTransactionAndExpiry (0.12s) + --- PASS: TestIdempotencyTransactionAndExpiry/service_owns_message_content_rules (0.02s) + --- PASS: TestIdempotencyTransactionAndExpiry/service_owns_message_content_rules/empty (0.00s) + --- PASS: TestIdempotencyTransactionAndExpiry/service_owns_message_content_rules/whitespace (0.00s) + --- PASS: TestIdempotencyTransactionAndExpiry/service_owns_message_content_rules/invalid_UTF8 (0.00s) + --- PASS: TestIdempotencyTransactionAndExpiry/service_owns_message_content_rules/nul (0.00s) + --- PASS: TestIdempotencyTransactionAndExpiry/service_owns_message_content_rules/too_long (0.00s) + --- PASS: TestIdempotencyTransactionAndExpiry/service_owns_message_content_rules/duplicate (0.00s) + --- PASS: TestIdempotencyTransactionAndExpiry/service_owns_message_content_rules/too_many (0.00s) + --- PASS: TestIdempotencyTransactionAndExpiry/service_owns_message_content_rules/nil_id (0.00s) + --- PASS: TestIdempotencyTransactionAndExpiry/service_owns_message_content_rules/unknown (0.00s) + --- PASS: TestIdempotencyTransactionAndExpiry/service_owns_message_content_rules/foreign_chat (0.00s) + --- PASS: TestIdempotencyTransactionAndExpiry/service_owns_message_content_rules/missing_object (0.00s) + --- PASS: TestIdempotencyTransactionAndExpiry/object_lock_cancellation_does_not_leak_a_pooled_lock (0.05s) +PASS + +$ make test-images +PASS: image geometry, EXIF, alpha, real formats, checksums, ACL and negative inputs + +$ make test-staging +PASS: active source protected, ready staging removed, failed retention enforced, restore/replay verified + +$ make test-broker-persistence +PASS: confirmed message survived broker container replacement with the same volume and stable node identity + +$ make test-persistence ACTION_SCRIPT=scripts/restart.sh +python3 tests/resilience.py --mode persistence --action-script "scripts/restart.sh" --base-url "http://localhost:18085" +Executing explicit persistence hook; hook output suppressed to avoid leaking secrets. +PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence. + +$ make test-failover ACTION_SCRIPT=scripts/stop-one.sh +python3 tests/resilience.py --mode failover --action-script "scripts/stop-one.sh" --base-url "http://localhost:18085" +Executing explicit failover hook; hook output suppressed to avoid leaking secrets. +{"observed_instances_before": 2, "successful_writes": 133, "failed_attempts": 1, "first_recovery_seconds": 1.273} +PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. + +Actual process state before persistence hook +/mtusi2026lab5-api-1 pid=63585 started=2026-11-13T09:13:37.287792805Z +/mtusi2026lab5-api-2 pid=63443 started=2026-11-13T09:13:37.185017014Z +/mtusi2026lab5-broker-1 pid=16156 started=2026-11-13T09:06:10.180568751Z +/mtusi2026lab5-dispatcher-1 pid=63444 started=2026-11-13T09:13:37.187137055Z +/mtusi2026lab5-lb-1 pid=6588 started=2026-11-13T08:49:29.974451052Z +/mtusi2026lab5-postgres-1 pid=15734 started=2026-11-13T09:06:09.241419334Z +/mtusi2026lab5-redis-1 pid=15771 started=2026-11-13T09:06:09.289178667Z +/mtusi2026lab5-s3-1 pid=17699 started=2026-11-13T09:06:34.353438762Z +/mtusi2026lab5-worker-1 pid=66553 started=2026-11-13T09:14:11.339788001Z + +Actual process state after persistence hook +/mtusi2026lab5-api-1 pid=73635 started=2026-11-13T09:15:52.601737132Z +/mtusi2026lab5-api-2 pid=73510 started=2026-11-13T09:15:52.490649465Z +/mtusi2026lab5-broker-1 pid=70234 started=2026-11-13T09:14:51.740755173Z +/mtusi2026lab5-dispatcher-1 pid=73512 started=2026-11-13T09:15:52.489731548Z +/mtusi2026lab5-lb-1 pid=73700 started=2026-11-13T09:15:52.696327173Z +/mtusi2026lab5-postgres-1 pid=70113 started=2026-11-13T09:14:50.775761506Z +/mtusi2026lab5-redis-1 pid=70044 started=2026-11-13T09:14:50.718397506Z +/mtusi2026lab5-s3-1 pid=71476 started=2026-11-13T09:15:16.065232545Z +/mtusi2026lab5-worker-1 pid=73511 started=2026-11-13T09:15:52.490188673Z + +Actual state after API failover, before restoring stopped replica +api mtusi2026lab5-api-1 running healthy +api mtusi2026lab5-api-2 exited +broker mtusi2026lab5-broker-1 running healthy +dispatcher mtusi2026lab5-dispatcher-1 running +lb mtusi2026lab5-lb-1 running healthy +migrate mtusi2026lab5-migrate-1 exited +postgres mtusi2026lab5-postgres-1 running healthy +redis mtusi2026lab5-redis-1 running healthy +s3 mtusi2026lab5-s3-1 running healthy +s3-init mtusi2026lab5-s3-init-1 exited +worker mtusi2026lab5-worker-1 running + +Effective container resource limits +/mtusi2026lab5-api-1 memory_bytes=536870912 cpu_nano=1000000000 +/mtusi2026lab5-api-2 memory_bytes=536870912 cpu_nano=1000000000 +/mtusi2026lab5-broker-1 memory_bytes=536870912 cpu_nano=500000000 +/mtusi2026lab5-dispatcher-1 memory_bytes=134217728 cpu_nano=250000000 +/mtusi2026lab5-lb-1 memory_bytes=134217728 cpu_nano=500000000 +/mtusi2026lab5-postgres-1 memory_bytes=1073741824 cpu_nano=2000000000 +/mtusi2026lab5-redis-1 memory_bytes=134217728 cpu_nano=250000000 +/mtusi2026lab5-s3-1 memory_bytes=536870912 cpu_nano=1000000000 +/mtusi2026lab5-worker-1 memory_bytes=1073741824 cpu_nano=1000000000 + +make test-pipeline: 39/39 PASS; details and repetitions in pipeline.json +Final image targeted fault checks: 3/3 PASS; details in pipeline-final.json diff --git a/evidence/images.json b/evidence/images.json new file mode 100644 index 00000000..717ac91a --- /dev/null +++ b/evidence/images.json @@ -0,0 +1,53 @@ +{ + "results": [ + { + "case": "small", + "result": "PASS", + "thumbnail": [ + 16, + 8 + ] + }, + { + "case": "floor", + "result": "PASS", + "thumbnail": [ + 256, + 76 + ] + }, + { + "case": "alpha", + "result": "PASS", + "thumbnail": [ + 40, + 20 + ] + }, + { + "case": "orientation", + "result": "PASS", + "thumbnail": [ + 128, + 256 + ] + }, + { + "case": "thin", + "result": "PASS", + "thumbnail": [ + 1, + 256 + ] + }, + { + "case": "center_crop", + "result": "PASS", + "thumbnail": [ + 256, + 85 + ] + } + ], + "negative_cases": 4 +} diff --git a/evidence/pipeline-final.json b/evidence/pipeline-final.json new file mode 100644 index 00000000..e4293fd0 --- /dev/null +++ b/evidence/pipeline-final.json @@ -0,0 +1,42 @@ +{ + "date_utc": "2026-11-13T09:24:08.989020+00:00", + "commit": "ce035c85282e75513ea36cb429f4b8d775398d62", + "working_tree": "implementation changes uncommitted", + "repeats": 1, + "worker": { + "concurrency": 2, + "prefetch": 2, + "timeout_seconds": 5, + "lease_seconds": 8, + "memory": "1GiB", + "cpus": 1 + }, + "results": [ + { + "scenario": "crash_confirm_before_outbox_mark", + "run": 1, + "result": "PASS", + "seconds": 33.527, + "job_id": "ce8a7658-c069-429a-b655-cb051f922e14" + }, + { + "scenario": "lease_reorder_and_stale_version", + "run": 1, + "result": "PASS", + "seconds": 14.623, + "attachment_id": "6e709809-9ca2-4fb8-b7f6-155a3aace89c", + "old_job_id": "421fa4dc-3704-4f87-a490-3110f062459a", + "pipeline_version": 2, + "manifest_rows": 8 + }, + { + "scenario": "api_crash_after_upload_commit", + "run": 1, + "result": "PASS", + "seconds": 2.122, + "attachment_id": "293e92d6-4253-4fd9-8bad-719a488f6bdf", + "accepted_commit": true + } + ], + "image_id": "sha256:dc259d5fbee064cac712f13a42c074c2df34ddc74957266d41a75d6f5e631b4f" +} diff --git a/evidence/pipeline.json b/evidence/pipeline.json new file mode 100644 index 00000000..946326fc --- /dev/null +++ b/evidence/pipeline.json @@ -0,0 +1,337 @@ +{ + "date_utc": "2026-11-13T09:09:05.756165+00:00", + "commit": "ce035c85282e75513ea36cb429f4b8d775398d62", + "working_tree": "implementation changes uncommitted", + "repeats": 3, + "worker": { + "concurrency": 2, + "prefetch": 2, + "timeout_seconds": 5, + "lease_seconds": 8, + "memory": "1GiB", + "cpus": 1 + }, + "results": [ + { + "scenario": "worker_down_and_store_restart", + "run": 1, + "result": "PASS", + "seconds": 66.563, + "attachment_id": "912b15f8-2ddb-4cbc-8db7-526540256749", + "job_id": "4913bc34-1533-4140-a0dc-c53efa5be0dd", + "postgres_restarted": true + }, + { + "scenario": "crash_before_objects", + "run": 1, + "result": "PASS", + "seconds": 10.888, + "job_id": "bfdb496b-f2e0-4836-9559-1a9b07ed21c4", + "manifest_rows": 4 + }, + { + "scenario": "crash_after_original_put", + "run": 1, + "result": "PASS", + "seconds": 12.171, + "job_id": "8468e628-5eef-42c2-9a86-d050d74cb672", + "manifest_rows": 4 + }, + { + "scenario": "crash_after_objects", + "run": 1, + "result": "PASS", + "seconds": 11.174, + "job_id": "4c587b0f-fa79-4d9e-a3c7-4904db1e80d3", + "manifest_rows": 4 + }, + { + "scenario": "crash_after_result_commit", + "run": 1, + "result": "PASS", + "seconds": 1.973, + "job_id": "419c9eeb-6467-4138-b05e-92f1bc929df8", + "manifest_rows": 4 + }, + { + "scenario": "crash_before_ack", + "run": 1, + "result": "PASS", + "seconds": 2.017, + "job_id": "face4c1b-5597-4b24-8f61-a0e756e601de", + "manifest_rows": 4 + }, + { + "scenario": "ready_redelivery", + "run": 1, + "result": "PASS", + "seconds": 2.435, + "job_id": "514c616d-367c-4693-8d33-60079b2ced87", + "copies": 3 + }, + { + "scenario": "crash_confirm_before_outbox_mark", + "run": 1, + "result": "PASS", + "seconds": 33.996, + "job_id": "71f7c962-009e-4982-ae7b-e7f24fdcdc7f" + }, + { + "scenario": "lease_reorder_and_stale_version", + "run": 1, + "result": "PASS", + "seconds": 14.652, + "attachment_id": "52a2e241-60a1-4443-8ef7-2e8e3e860577", + "old_job_id": "1c5824ca-774e-4230-9764-3e0938f4efeb", + "pipeline_version": 2, + "manifest_rows": 8 + }, + { + "scenario": "bounded_retry_and_explicit_replay", + "run": 1, + "result": "PASS", + "seconds": 14.47, + "job_id": "8b8f8975-f4c8-45cf-80e9-00626c4e5c36", + "failed_attempt": 5, + "replayed_attempt": 6 + }, + { + "scenario": "permanent_error", + "run": 1, + "result": "PASS", + "seconds": 2.699, + "job_id": "e1948ce8-b194-4f66-8008-6e733ad42959" + }, + { + "scenario": "api_crash_after_staging", + "run": 1, + "result": "PASS", + "seconds": 0.817, + "attachment_id": "2d831ec0-5918-4290-8e26-3c2bec3cc0fd", + "accepted_commit": false + }, + { + "scenario": "api_crash_after_upload_commit", + "run": 1, + "result": "PASS", + "seconds": 2.328, + "attachment_id": "f0da85c6-003c-4d79-ba01-f3f961064741", + "accepted_commit": true + }, + { + "scenario": "worker_down_and_store_restart", + "run": 2, + "result": "PASS", + "seconds": 66.013, + "attachment_id": "64517fe4-6419-4257-bbf5-a47a88b51068", + "job_id": "873b9510-7ada-4dc0-bdfb-3e204ae2bc3d", + "postgres_restarted": true + }, + { + "scenario": "crash_before_objects", + "run": 2, + "result": "PASS", + "seconds": 11.241, + "job_id": "1c0ea968-87d2-45df-bb9b-d8b45c68dc1d", + "manifest_rows": 4 + }, + { + "scenario": "crash_after_original_put", + "run": 2, + "result": "PASS", + "seconds": 11.222, + "job_id": "748017a5-8f09-4741-8e35-35e4dbcb3c55", + "manifest_rows": 4 + }, + { + "scenario": "crash_after_objects", + "run": 2, + "result": "PASS", + "seconds": 11.34, + "job_id": "489d2edd-b008-48fc-ba35-c9f64ee05ba2", + "manifest_rows": 4 + }, + { + "scenario": "crash_after_result_commit", + "run": 2, + "result": "PASS", + "seconds": 2.088, + "job_id": "0ed57d8a-3fcc-42a4-a31f-2f02ae4f5f5f", + "manifest_rows": 4 + }, + { + "scenario": "crash_before_ack", + "run": 2, + "result": "PASS", + "seconds": 2.026, + "job_id": "e9a8270c-3132-460a-a888-ee8722798e35", + "manifest_rows": 4 + }, + { + "scenario": "ready_redelivery", + "run": 2, + "result": "PASS", + "seconds": 2.704, + "job_id": "b4f7d69b-3de5-4350-97bf-0e880ed94b0e", + "copies": 3 + }, + { + "scenario": "crash_confirm_before_outbox_mark", + "run": 2, + "result": "PASS", + "seconds": 33.988, + "job_id": "5f7a89af-a045-423c-8c70-47b19615b577" + }, + { + "scenario": "lease_reorder_and_stale_version", + "run": 2, + "result": "PASS", + "seconds": 14.694, + "attachment_id": "a1230ca9-8001-4bb4-b648-51446f646cbf", + "old_job_id": "2f1e1fbf-856d-4605-873c-b23011072a4e", + "pipeline_version": 2, + "manifest_rows": 8 + }, + { + "scenario": "bounded_retry_and_explicit_replay", + "run": 2, + "result": "PASS", + "seconds": 13.737, + "job_id": "737209e2-5d3c-4693-8394-fc8173b88ee2", + "failed_attempt": 5, + "replayed_attempt": 6 + }, + { + "scenario": "permanent_error", + "run": 2, + "result": "PASS", + "seconds": 2.552, + "job_id": "e5e6dc90-9ea5-4d7c-aa11-c6c275d69793" + }, + { + "scenario": "api_crash_after_staging", + "run": 2, + "result": "PASS", + "seconds": 0.153, + "attachment_id": "ed8e2936-47fa-4717-9eb3-e63292c8a0ad", + "accepted_commit": false + }, + { + "scenario": "api_crash_after_upload_commit", + "run": 2, + "result": "PASS", + "seconds": 2.159, + "attachment_id": "a739f237-2278-421c-aa55-6024e717ad89", + "accepted_commit": true + }, + { + "scenario": "worker_down_and_store_restart", + "run": 3, + "result": "PASS", + "seconds": 63.864, + "attachment_id": "95176e6d-301a-4725-9861-7133507184eb", + "job_id": "e5f8b9fa-e460-4441-92b0-2af2ac2540bf", + "postgres_restarted": true + }, + { + "scenario": "crash_before_objects", + "run": 3, + "result": "PASS", + "seconds": 12.269, + "job_id": "67c70b6e-ea42-4c1d-894c-d2234a70bf03", + "manifest_rows": 4 + }, + { + "scenario": "crash_after_original_put", + "run": 3, + "result": "PASS", + "seconds": 11.402, + "job_id": "386f9b0d-1b1f-42b6-95a0-a13d6b97f118", + "manifest_rows": 4 + }, + { + "scenario": "crash_after_objects", + "run": 3, + "result": "PASS", + "seconds": 11.259, + "job_id": "9c639085-f040-4989-90ba-54376c64630d", + "manifest_rows": 4 + }, + { + "scenario": "crash_after_result_commit", + "run": 3, + "result": "PASS", + "seconds": 2.068, + "job_id": "73fe51b1-32eb-4c93-b8b0-0ed920a6641d", + "manifest_rows": 4 + }, + { + "scenario": "crash_before_ack", + "run": 3, + "result": "PASS", + "seconds": 2.098, + "job_id": "2947da70-b658-48c0-9aa2-905bc0927781", + "manifest_rows": 4 + }, + { + "scenario": "ready_redelivery", + "run": 3, + "result": "PASS", + "seconds": 3.002, + "job_id": "4f67af5a-1014-4f91-9a68-54739f2edde7", + "copies": 3 + }, + { + "scenario": "crash_confirm_before_outbox_mark", + "run": 3, + "result": "PASS", + "seconds": 36.021, + "job_id": "c4135c71-64e7-44ba-9e4f-bf8f7489a459" + }, + { + "scenario": "lease_reorder_and_stale_version", + "run": 3, + "result": "PASS", + "seconds": 15.225, + "attachment_id": "3bdce3c7-d7d5-49fb-9691-96e626317ae1", + "old_job_id": "dd3d35b5-14fe-420b-a650-e740bc43489c", + "pipeline_version": 2, + "manifest_rows": 8 + }, + { + "scenario": "bounded_retry_and_explicit_replay", + "run": 3, + "result": "PASS", + "seconds": 15.238, + "job_id": "444a1bf1-42ed-4123-9ec7-7f614cd6d90f", + "failed_attempt": 5, + "replayed_attempt": 6 + }, + { + "scenario": "permanent_error", + "run": 3, + "result": "PASS", + "seconds": 2.604, + "job_id": "d21ee0e0-702d-4459-8fd6-5e5f62c6c11d" + }, + { + "scenario": "api_crash_after_staging", + "run": 3, + "result": "PASS", + "seconds": 0.144, + "attachment_id": "963d2034-021f-492f-81f3-d31431a2bc0f", + "accepted_commit": false + }, + { + "scenario": "api_crash_after_upload_commit", + "run": 3, + "result": "PASS", + "seconds": 2.141, + "attachment_id": "ea757ed8-98a8-4f3b-993e-556347380e26", + "accepted_commit": true + } + ], + "tested_image_id": "sha256:0b720b9bab11e4a762e8b180b26c290a43503c4db2db3436c201cb08725f24c0", + "runtime_source_sha256": "e6460140119a22471f992cd02c721b55463d0810a6a8e50333e193d674c7c79a", + "subsequent_fix": "Restore refreshes retention timestamp; Cleanup shares its source lock. Publisher has a physical socket deadline. Verified by storage.py and the additional final-image fault scenarios in pipeline-final.json; job processing, lease and manifest logic unchanged." +} diff --git a/evidence/storage.json b/evidence/storage.json new file mode 100644 index 00000000..8a953a03 --- /dev/null +++ b/evidence/storage.json @@ -0,0 +1,24 @@ +{ + "results": [ + { + "scenario": "queued_staging_protected", + "result": "PASS", + "attachment_id": "f2098db8-c76b-4eff-bb37-921f4e66b98b" + }, + { + "scenario": "staging_deleted_after_ready", + "result": "PASS", + "attachment_id": "f2098db8-c76b-4eff-bb37-921f4e66b98b" + }, + { + "scenario": "failed_retention_and_missing_source_replay", + "result": "PASS", + "age_simulated_hours": 25 + }, + { + "scenario": "verified_restore_then_replay", + "result": "PASS", + "job_id": "8d6f4591-e10f-4616-b387-fe7ec5020a84" + } + ] +} diff --git a/infra/imagemagick-policy.xml b/infra/imagemagick-policy.xml new file mode 100644 index 00000000..decc7175 --- /dev/null +++ b/infra/imagemagick-policy.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + diff --git a/scripts/bootstrap-broker.py b/scripts/bootstrap-broker.py new file mode 100644 index 00000000..a1d7e471 --- /dev/null +++ b/scripts/bootstrap-broker.py @@ -0,0 +1,17 @@ +#!/usr/bin/env python3 +"""Create synthetic local broker credentials, preserving existing settings.""" +from pathlib import Path +import secrets +from urllib.parse import quote +root=Path(__file__).resolve().parents[1] +env=root/'.env.lab5' +values=dict(line.split('=',1) for line in env.read_text().splitlines() if '=' in line and not line.startswith('#')) +password=values.get('BROKER_PASSWORD') +if not password: + password=secrets.token_hex(24) + with env.open('a') as f: f.write('\nBROKER_PASSWORD='+password+'\n') +p=root/'secrets/lab5/broker_url' +if p.exists(): p.chmod(0o600) +p.write_text('amqp://messenger:'+quote(password,safe='')+'@broker:5672/\n') +p.chmod(0o444) +print('Local broker credentials ready.') diff --git a/scripts/bootstrap.sh b/scripts/bootstrap.sh index 4c09f781..f67bf842 100755 --- a/scripts/bootstrap.sh +++ b/scripts/bootstrap.sh @@ -21,3 +21,5 @@ chmod 444 secrets/lab5/*.pem printf '%s\n' 'Local lab5 settings and synthetic keys ready; existing files preserved.' python3 scripts/bootstrap-storage.py + +python3 scripts/bootstrap-broker.py diff --git a/scripts/restart.sh b/scripts/restart.sh index 24699183..324c1ef6 100755 --- a/scripts/restart.sh +++ b/scripts/restart.sh @@ -1,7 +1,7 @@ #!/bin/sh set -eu cd "$(dirname "$0")/.." -./scripts/compose.sh stop api lb -./scripts/compose.sh restart postgres redis s3 +./scripts/compose.sh stop api lb worker dispatcher +./scripts/compose.sh restart postgres redis s3 broker ./scripts/compose.sh run --rm -T s3-init -./scripts/compose.sh up -d --wait --wait-timeout 90 api lb +./scripts/compose.sh up -d --wait --wait-timeout 90 api lb worker dispatcher diff --git a/service/cmd/app/main.go b/service/cmd/app/main.go index 804126da..353039d2 100644 --- a/service/cmd/app/main.go +++ b/service/cmd/app/main.go @@ -27,6 +27,7 @@ import ( "gitea.privcloud.tech/mtusi2026/internal/service/attachment" "gitea.privcloud.tech/mtusi2026/internal/service/auth" "gitea.privcloud.tech/mtusi2026/internal/service/chat" + "gitea.privcloud.tech/mtusi2026/internal/service/outbox" "gitea.privcloud.tech/mtusi2026/internal/service/user" "github.com/google/uuid" stdhttp "net/http" @@ -56,6 +57,9 @@ func run() error { if err != nil { return err } + if len(os.Args) > 1 && (os.Args[1] == "worker" || os.Args[1] == "dispatcher" || os.Args[1] == "jobs" || os.Args[1] == "outbox") { + return runPipeline(cfg, attachmentConfig, os.Args[1:]) + } if len(os.Args) > 1 && os.Args[1] == "storage" { return runStorage(cfg, attachmentConfig, os.Args[2:]) } @@ -91,6 +95,12 @@ func run() error { chatSvc := chat.New(tm, chatsRepo, messageRepo, chatCache) objectStore := objectstore.New(attachmentConfig) attachmentSvc := attachment.New(&storage.AttachmentsRepo{Pool: pgx}, objectStore, attachmentConfig, chatSvc) + pipelineConfig, err := config.LoadPipeline() + if err != nil { + return err + } + outboxSvc := outbox.New(&storage.OutboxRepo{Pool: pgx}, tm, nil) + attachmentSvc.Configure(tm, outboxSvc, pipelineConfig) httpRouter := router.New( middleware.NewAuthMiddleware(authSvc), diff --git a/service/cmd/app/pipeline.go b/service/cmd/app/pipeline.go new file mode 100644 index 00000000..1cbabe45 --- /dev/null +++ b/service/cmd/app/pipeline.go @@ -0,0 +1,138 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "flag" + "gitea.privcloud.tech/mtusi2026/internal/adapter/broker" + "gitea.privcloud.tech/mtusi2026/internal/adapter/repository/storage" + "gitea.privcloud.tech/mtusi2026/internal/config" + "gitea.privcloud.tech/mtusi2026/internal/pkg/objectstore" + "gitea.privcloud.tech/mtusi2026/internal/pkg/postgres" + "gitea.privcloud.tech/mtusi2026/internal/pkg/postgres/txmanager" + "gitea.privcloud.tech/mtusi2026/internal/service/attachment" + "gitea.privcloud.tech/mtusi2026/internal/service/outbox" + "github.com/google/uuid" + "github.com/jackc/pgx/v5" + "log/slog" + "os" + "os/signal" + "syscall" + "time" +) + +func runPipeline(cfg *config.Config, a *config.Attachment, args []string) error { + c, e := config.LoadPipeline() + if e != nil { + return e + } + ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) + defer stop() + pool := postgres.New(cfg.Postgres) + defer pool.Close() + tm := txmanager.New(pool) + repo := &storage.AttachmentsRepo{Pool: pool} + rabbit := &broker.Rabbit{URL: c.BrokerURL} + ob := outbox.New(&storage.OutboxRepo{Pool: pool}, tm, rabbit) + svc := attachment.New(repo, objectstore.New(a), a, nil) + svc.Configure(tm, ob, c) + ob.OnDead = svc.PublishFailed + if args[0] == "jobs" || args[0] == "outbox" { + if len(args) < 2 { + return errors.New("operator action required") + } + fs := flag.NewFlagSet(args[1], flag.ContinueOnError) + idText := fs.String("id", "", "job ID (attachment ID for reprocess)") + version := fs.Int("pipeline-version", 2, "new pipeline version") + if e = fs.Parse(args[2:]); e != nil { + return e + } + if args[1] == "list" { + jobs, e := repo.ListJobs(ctx) + if e != nil { + return e + } + return json.NewEncoder(os.Stdout).Encode(jobs) + } + id, e := uuid.Parse(*idText) + if e != nil { + return e + } + switch args[1] { + case "redeliver": + if os.Getenv("LAB5_FAULTS") != "1" { + return errors.New("redeliver is restricted to the explicit fault profile") + } + var payload []byte + e = tm.RunInTransaction(ctx, nil, func(tx pgx.Tx) error { + j, e := repo.JobTx(ctx, tx, id) + if e != nil { + return e + } + payload, e = json.Marshal(j.Message) + return e + }) + if e == nil { + e = rabbit.Publish(ctx, outbox.DeliveryMessage{ID: uuid.New(), AggregateID: id, Payload: payload}) + } + case "replay": + e = svc.Replay(ctx, id) + case "reprocess": + e = svc.Reprocess(ctx, id, *version) + default: + return errors.New("unknown operator action") + } + if e != nil { + return e + } + return json.NewEncoder(os.Stdout).Encode(map[string]any{"accepted": true, "id": id}) + } + if c.BrokerURL == "" { + return errors.New("BROKER_URL_FILE is required") + } + if args[0] == "worker" { + // Recovery continues while the consumer is disconnected or a delivery is paused. + recoveryDone := make(chan struct{}) + defer func() { stop(); <-recoveryDone }() + go func() { + defer close(recoveryDone) + tick := time.NewTicker(time.Second) + defer tick.Stop() + for { + select { + case <-ctx.Done(): + return + case <-tick.C: + if e := svc.Recover(ctx); e != nil { + slog.Warn("job_recovery_failed") + } + if e := svc.Cleanup(ctx); e != nil { + slog.Warn("source_cleanup_failed") + } + } + } + }() + for ctx.Err() == nil { + if e = rabbit.Consume(ctx, c.Concurrency, c.Prefetch, svc.Process); e != nil && ctx.Err() == nil { + slog.Warn("consumer_reconnecting") + } + select { + case <-ctx.Done(): + case <-time.After(time.Second): + } + } + return nil + } + for ctx.Err() == nil { + _, e = ob.ProcessOnce(ctx) + if e != nil { + slog.Warn("dispatcher_retry") + } + select { + case <-ctx.Done(): + case <-time.After(100 * time.Millisecond): + } + } + return nil +} diff --git a/service/cmd/app/storage.go b/service/cmd/app/storage.go index daeb6a1d..09d459c9 100644 --- a/service/cmd/app/storage.go +++ b/service/cmd/app/storage.go @@ -20,7 +20,9 @@ import ( "gitea.privcloud.tech/mtusi2026/internal/config" "gitea.privcloud.tech/mtusi2026/internal/pkg/objectstore" "gitea.privcloud.tech/mtusi2026/internal/pkg/postgres" + "gitea.privcloud.tech/mtusi2026/internal/pkg/postgres/txmanager" "gitea.privcloud.tech/mtusi2026/internal/service/attachment" + "gitea.privcloud.tech/mtusi2026/internal/service/outbox" ) func runStorage(cfg *config.Config, c *config.Attachment, args []string) error { @@ -93,6 +95,12 @@ func runStorage(cfg *config.Config, c *config.Attachment, args []string) error { pool := postgres.New(cfg.Postgres) defer pool.Close() svc := attachment.New(&storage.AttachmentsRepo{Pool: pool}, store, c, nil) + jobs, e := config.LoadPipeline() + if e != nil { + return e + } + tm := txmanager.New(pool) + svc.Configure(tm, outbox.New(&storage.OutboxRepo{Pool: pool}, tm, nil), jobs) switch args[0] { case "reconcile": fs := flag.NewFlagSet("reconcile", flag.ContinueOnError) diff --git a/service/go.mod b/service/go.mod index b816a855..ce30f283 100644 --- a/service/go.mod +++ b/service/go.mod @@ -52,6 +52,7 @@ require ( github.com/mitchellh/copystructure v1.2.0 // indirect github.com/mitchellh/reflectwalk v1.0.2 // indirect github.com/pelletier/go-toml/v2 v2.4.3-0.20260624044523-35f78d5908f1 // indirect + github.com/rabbitmq/amqp091-go v1.10.0 // indirect github.com/sagikazarmark/locafero v0.11.0 // indirect github.com/shopspring/decimal v1.4.0 // indirect github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8 // indirect diff --git a/service/go.sum b/service/go.sum index 9c564cfe..d7b4c21d 100644 --- a/service/go.sum +++ b/service/go.sum @@ -94,6 +94,8 @@ github.com/oapi-codegen/runtime v1.4.2/go.mod h1:GwV7hC2hviaMzj+ITfHVRESK5J2W/Ge github.com/pelletier/go-toml/v2 v2.4.3-0.20260624044523-35f78d5908f1 h1:gIZQQ2+wbsdQGWLPDSdJFptR25tBCjSC0KbSIQZ1rgE= github.com/pelletier/go-toml/v2 v2.4.3-0.20260624044523-35f78d5908f1/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rabbitmq/amqp091-go v1.10.0 h1:STpn5XsHlHGcecLmMFCtg7mqq0RnD+zFr4uzukfVhBw= +github.com/rabbitmq/amqp091-go v1.10.0/go.mod h1:Hy4jKW5kQART1u+JkDTF9YYOQUHXqMuhrgxOEeS7G4o= github.com/redis/go-redis/v9 v9.23.0 h1:/TTVdJa8BRNsybwwGGnmiYktHNlUXtbOQxlWpHBPL3s= github.com/redis/go-redis/v9 v9.23.0/go.mod h1:EWP2UUk+XDaI1s5nQB5mablZN6vccXlmBQTyfnF87D0= github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= diff --git a/service/internal/adapter/broker/rabbitmq.go b/service/internal/adapter/broker/rabbitmq.go new file mode 100644 index 00000000..508a2511 --- /dev/null +++ b/service/internal/adapter/broker/rabbitmq.go @@ -0,0 +1,170 @@ +package broker + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "gitea.privcloud.tech/mtusi2026/internal/dto" + "gitea.privcloud.tech/mtusi2026/internal/pkg/fault" + "gitea.privcloud.tech/mtusi2026/internal/service/outbox" + "github.com/google/uuid" + amqp "github.com/rabbitmq/amqp091-go" + "io" + "log/slog" + "net" + "regexp" + "time" +) + +const Queue = "attachments.v1" + +var tracePattern = regexp.MustCompile(`^00-[0-9a-f]{32}-[0-9a-f]{16}-[0-9a-f]{2}$`) + +type Rabbit struct{ URL string } + +func (r *Rabbit) connect(deadline time.Duration) (*amqp.Connection, *amqp.Channel, error) { + dial := amqp.DefaultDial(5 * time.Second) + if deadline > 0 { + dial = func(network, address string) (net.Conn, error) { + c, e := net.DialTimeout(network, address, 5*time.Second) + if e != nil { + return nil, e + } + if e = c.SetDeadline(time.Now().Add(deadline)); e != nil { + c.Close() + return nil, e + } + return c, nil + } + } + conn, e := amqp.DialConfig(r.URL, amqp.Config{Heartbeat: 10 * time.Second, Dial: dial}) + if e != nil { + return nil, nil, errors.New("broker connection unavailable") + } + ch, e := conn.Channel() + if e != nil { + conn.Close() + return nil, nil, errors.New("broker channel unavailable") + } + _, e = ch.QueueDeclare(Queue+".invalid", true, false, false, false, nil) + if e == nil { + e = ch.ExchangeDeclare("attachments.invalid", "direct", true, false, false, false, nil) + } + if e == nil { + e = ch.QueueBind(Queue+".invalid", "invalid", "attachments.invalid", false, nil) + } + if e == nil { + _, e = ch.QueueDeclare(Queue, true, false, false, false, amqp.Table{"x-dead-letter-exchange": "attachments.invalid", "x-dead-letter-routing-key": "invalid"}) + } + if e != nil { + ch.Close() + conn.Close() + return nil, nil, errors.New("broker topology unavailable") + } + return conn, ch, nil +} +func (r *Rabbit) Publish(ctx context.Context, m outbox.DeliveryMessage) error { + conn, ch, e := r.connect(20 * time.Second) + if e != nil { + return e + } + defer conn.Close() + defer ch.Close() + if e = ch.Confirm(false); e != nil { + return e + } + confirms := ch.NotifyPublish(make(chan amqp.Confirmation, 1)) + returns := ch.NotifyReturn(make(chan amqp.Return, 1)) + var job dto.JobMessage + if e = json.Unmarshal(m.Payload, &job); e != nil { + return e + } + fault.Hit(ctx, "before_publish", job.JobID) + if e = ch.PublishWithContext(ctx, "", Queue, true, false, amqp.Publishing{DeliveryMode: amqp.Persistent, ContentType: "application/json", MessageId: m.ID.String(), Body: m.Payload}); e != nil { + return errors.New("broker publish failed") + } + select { + case c, ok := <-confirms: + if !ok || !c.Ack { + return errors.New("broker did not confirm") + } + select { + case <-returns: + return errors.New("unroutable job") + default: + } + case <-time.After(15 * time.Second): + return errors.New("confirm timeout") + case <-ctx.Done(): + return ctx.Err() + } + fault.Hit(ctx, "after_publish", job.JobID) + return nil +} +func (r *Rabbit) Consume(ctx context.Context, concurrency, prefetch int, process func(context.Context, dto.JobMessage) error) error { + conn, ch, e := r.connect(0) + if e != nil { + return e + } + defer conn.Close() + defer ch.Close() + if e = ch.Qos(prefetch, 0, false); e != nil { + return e + } + deliveries, e := ch.Consume(Queue, "worker-"+uuid.NewString(), false, false, false, false, nil) + if e != nil { + return e + } + done := make(chan struct{}) + go func() { + select { + case <-ctx.Done(): + conn.Close() + case <-done: + } + }() + defer close(done) + tasks := make(chan amqp.Delivery) + finished := make(chan struct{}, concurrency) + for range concurrency { + go func() { + defer func() { finished <- struct{}{} }() + for d := range tasks { + var m dto.JobMessage + dec := json.NewDecoder(bytes.NewReader(d.Body)) + dec.DisallowUnknownFields() + e := dec.Decode(&m) + if e != nil || dec.Decode(new(any)) != io.EOF || m.SchemaVersion != 1 || m.Type != "attachment.process.v1" || m.JobID == uuid.Nil || m.AttachmentID == uuid.Nil || m.Attempt < 1 || m.PipelineVersion < 1 || m.CreatedAt.IsZero() || len(m.Tracestate) > 512 || (m.Traceparent != "" && !tracePattern.MatchString(m.Traceparent)) { + _ = d.Reject(false) + continue + } + if e = process(ctx, m); e != nil { + slog.Warn("job_not_acknowledged", "job_id", m.JobID, "reason", "processing_or_dependency_unavailable") + select { + case <-ctx.Done(): + case <-time.After(200 * time.Millisecond): + } + _ = d.Nack(false, true) + continue + } + fault.Hit(ctx, "before_ack", m.JobID) + _ = d.Ack(false) + } + }() + } + for d := range deliveries { + select { + case tasks <- d: + case <-ctx.Done(): + } + } + close(tasks) + for range concurrency { + <-finished + } + if ctx.Err() != nil { + return ctx.Err() + } + return errors.New("broker consumer disconnected") +} diff --git a/service/internal/adapter/controller/http/attachmentHandler.go b/service/internal/adapter/controller/http/attachmentHandler.go index 577723c4..d286d539 100644 --- a/service/internal/adapter/controller/http/attachmentHandler.go +++ b/service/internal/adapter/controller/http/attachmentHandler.go @@ -1,12 +1,14 @@ package http import ( + "gitea.privcloud.tech/mtusi2026/internal/dto" "gitea.privcloud.tech/mtusi2026/internal/pkg/apperr" "gitea.privcloud.tech/mtusi2026/internal/pkg/auth/jwtprovider" "gitea.privcloud.tech/mtusi2026/internal/pkg/rest/framework" "gitea.privcloud.tech/mtusi2026/internal/pkg/rest/router" "gitea.privcloud.tech/mtusi2026/internal/service/attachment" "io" + "mime" "net/http" "strconv" "strings" @@ -41,7 +43,20 @@ func (h *AttachmentHandler) Upload(w http.ResponseWriter, r *http.Request) { framework.JSONError(r.Context(), w, apperr.New(apperr.CodeBadRequest, "expected one filename")) return } - a, e := h.svc.Upload(w, r, user.UserID, id, values[0]) + if !attachment.ValidFilename(values[0]) { + framework.JSONError(r.Context(), w, apperr.New(apperr.CodeBadRequest, "invalid filename")) + return + } + mt, _, e := mime.ParseMediaType(r.Header.Get("Content-Type")) + if e != nil || (mt != "image/png" && mt != "image/jpeg") { + framework.JSONError(r.Context(), w, apperr.New(apperr.CodeUnsupportedMediaType, "expected PNG or JPEG")) + return + } + if r.ContentLength > h.svc.Config.MaxBytes { + framework.JSONError(r.Context(), w, apperr.New(apperr.CodePayloadTooLarge, "upload too large")) + return + } + a, e := h.svc.Upload(r.Context(), dto.UploadAttachment{UserID: user.UserID, ChatID: id, Filename: values[0], ContentType: mt, Body: io.LimitReader(r.Body, h.svc.Config.MaxBytes+1)}) if e != nil { if apperr.CodeFromError(e) == apperr.CodeTooManyRequests { w.Header().Set("Retry-After", "1") @@ -49,7 +64,7 @@ func (h *AttachmentHandler) Upload(w http.ResponseWriter, r *http.Request) { framework.JSONError(r.Context(), w, e) return } - framework.JSONResponse(w, http.StatusCreated, a) + framework.JSONResponse(w, http.StatusAccepted, a) } func (h *AttachmentHandler) Get(w http.ResponseWriter, r *http.Request) { user, e := framework.ParseToken(r, h.provider) @@ -61,27 +76,32 @@ func (h *AttachmentHandler) Get(w http.ResponseWriter, r *http.Request) { if e != nil { return } - a, e := h.svc.Get(r.Context(), user.UserID, id) + + if !strings.HasSuffix(r.URL.Path, "/content") { + a, e := h.svc.Get(r.Context(), user.UserID, id) + if e != nil { + framework.JSONError(r.Context(), w, e) + return + } + framework.JSONResponse(w, 200, a.Public) + return + } + v := r.URL.Query().Get("variant") + if v == "" { + v = "original" + } + if v != "original" && v != "thumbnail" && v != "crop" && v != "optimized" { + framework.JSONError(r.Context(), w, apperr.New(apperr.CodeBadRequest, "invalid variant")) + return + } + obj, e := h.svc.Content(r.Context(), user.UserID, id, v) if e != nil { framework.JSONError(r.Context(), w, e) return } - if !strings.HasSuffix(r.URL.Path, "/content") { - framework.JSONResponse(w, 200, a.Public) - return - } - if v := r.URL.Query().Get("variant"); v != "" && v != "original" { - framework.JSONError(r.Context(), w, apperr.New(apperr.CodeBadRequest, "invalid variant")) - return - } - obj, e := h.svc.Store.Get(r.Context(), a.Key) - if e != nil { - framework.JSONError(r.Context(), w, attachment.Unavailable()) - return - } defer obj.Body.Close() - w.Header().Set("Content-Type", string(a.Public.ContentType)) - w.Header().Set("Content-Length", strconv.Itoa(a.Public.SizeBytes)) + w.Header().Set("Content-Type", obj.ContentType) + w.Header().Set("Content-Length", strconv.FormatInt(obj.Size, 10)) w.Header().Set("X-Content-Type-Options", "nosniff") // Once streaming has started a transport failure terminates the response; no JSON // error can be appended to an image. Content-Length exposes incomplete downloads. diff --git a/service/internal/adapter/repository/storage/attachment_scan.go b/service/internal/adapter/repository/storage/attachment_scan.go index 15fa2134..5a105372 100644 --- a/service/internal/adapter/repository/storage/attachment_scan.go +++ b/service/internal/adapter/repository/storage/attachment_scan.go @@ -25,10 +25,15 @@ func (r *AttachmentsRepo) Page(ctx context.Context, after uuid.UUID) ([]*Attachm } func (r *AttachmentsRepo) Exists(ctx context.Context, c *pgxpool.Conn, key string) (bool, error) { var found bool - e := c.QueryRow(ctx, "SELECT EXISTS(SELECT 1 FROM attachments WHERE object_key=$1)", key).Scan(&found) + e := c.QueryRow(ctx, "SELECT EXISTS(SELECT 1 FROM attachment_variants WHERE object_key=$1 UNION ALL SELECT 1 FROM attachments WHERE object_key=$1 OR staging_key=$1 OR variant_keys @> jsonb_build_object('thumbnail',$1::text) OR variant_keys @> jsonb_build_object('crop',$1::text) OR variant_keys @> jsonb_build_object('optimized',$1::text))", key).Scan(&found) return found, e } func (r *AttachmentsRepo) MarkMissing(ctx context.Context, c *pgxpool.Conn, id uuid.UUID, missing bool) (bool, error) { tag, e := c.Exec(ctx, "UPDATE attachments SET integrity_missing=$2,updated_at=clock_timestamp() WHERE id=$1 AND integrity_missing<>$2", id, missing) return tag.RowsAffected() == 1, e } + +func (r *AttachmentsRepo) CheckConn(ctx context.Context, c *pgxpool.Conn) error { + _, e := c.Exec(ctx, "SELECT 1") + return e +} diff --git a/service/internal/adapter/repository/storage/attachments.go b/service/internal/adapter/repository/storage/attachments.go index 6d35c74c..0a3b2bbe 100644 --- a/service/internal/adapter/repository/storage/attachments.go +++ b/service/internal/adapter/repository/storage/attachments.go @@ -8,40 +8,34 @@ import ( "github.com/jackc/pgx/v5" "github.com/jackc/pgx/v5/pgxpool" - "gitea.privcloud.tech/mtusi2026/internal/gen/contracts" + "encoding/json" + "gitea.privcloud.tech/mtusi2026/internal/dto" "gitea.privcloud.tech/mtusi2026/internal/pkg/apperr" ) -type AttachmentRecord struct { - Public contracts.Attachment - Key string - Missing bool -} +type AttachmentRecord = dto.AttachmentRecord type AttachmentsRepo struct{ Pool *pgxpool.Pool } -const AttachmentColumns = "id,chat_id,owner_id,filename,content_type,size_bytes,sha256,created_at,updated_at,object_key,integrity_missing" +const AttachmentColumns = "id,chat_id,owner_id,filename,content_type,size_bytes,sha256,created_at,updated_at,object_key,integrity_missing,status,error_code,pipeline_version,staging_key,variants,variant_keys" func ScanAttachment(row pgx.Row) (*AttachmentRecord, error) { a := new(AttachmentRecord) p := &a.Public - if err := row.Scan(&p.Id, &p.ChatId, &p.OwnerId, &p.Filename, &p.ContentType, &p.SizeBytes, &p.Sha256, &p.CreatedAt, &p.UpdatedAt, &a.Key, &a.Missing); err != nil { + var variants, keys []byte + if err := row.Scan(&p.Id, &p.ChatId, &p.OwnerId, &p.Filename, &p.ContentType, &p.SizeBytes, &p.Sha256, &p.CreatedAt, &p.UpdatedAt, &a.Key, &a.Missing, &p.Status, &p.ErrorCode, &a.PipelineVersion, &a.StagingKey, &variants, &keys); err != nil { return nil, apperr.WrapPostgres(err, "attachment unavailable") } - p.Status = "ready" - p.Variants = []contracts.Variant{} + if err := json.Unmarshal(variants, &p.Variants); err != nil { + return nil, err + } + if err := json.Unmarshal(keys, &a.VariantKeys); err != nil { + return nil, err + } return a, nil } func (r *AttachmentsRepo) Get(ctx context.Context, id uuid.UUID) (*AttachmentRecord, error) { return ScanAttachment(r.Pool.QueryRow(ctx, "SELECT "+AttachmentColumns+" FROM attachments WHERE id=$1", id)) } -func (r *AttachmentsRepo) Save(ctx context.Context, conn *pgxpool.Conn, a *AttachmentRecord) error { - p := a.Public - _, e := conn.Exec(ctx, `INSERT INTO attachments(id,chat_id,owner_id,filename,content_type,size_bytes,sha256,created_at,updated_at,object_key) VALUES($1,$2,$3,$4,$5,$6,$7,$8,$9,$10)`, p.Id, p.ChatId, p.OwnerId, p.Filename, p.ContentType, p.SizeBytes, p.Sha256, p.CreatedAt, p.UpdatedAt, a.Key) - if e != nil { - return apperr.WrapPostgres(e, "failed to save attachment") - } - return nil -} // Session locks span S3 work and the database commit. A lost connection releases // the lock; the mandatory grace period also exceeds the maximum upload lifetime. diff --git a/service/internal/adapter/repository/storage/idempotency_test.go b/service/internal/adapter/repository/storage/idempotency_test.go index 2d151e25..db5164cc 100644 --- a/service/internal/adapter/repository/storage/idempotency_test.go +++ b/service/internal/adapter/repository/storage/idempotency_test.go @@ -58,7 +58,7 @@ func TestIdempotencyTransactionAndExpiry(t *testing.T) { if err != nil { t.Fatal(err) } - if _, err = repo.CreateIdempotentTx(ctx, tx, req); err != nil { + if _, err = chatsvc.CreateIdempotentTx(ctx, tx, repo, req); err != nil { t.Fatal(err) } if err = tx.Rollback(ctx); err != nil { @@ -75,7 +75,7 @@ func TestIdempotencyTransactionAndExpiry(t *testing.T) { if err != nil { t.Fatal(err) } - first, err := repo.CreateIdempotentTx(ctx, tx, req) + first, err := chatsvc.CreateIdempotentTx(ctx, tx, repo, req) if err != nil { t.Fatal(err) } @@ -91,7 +91,7 @@ func TestIdempotencyTransactionAndExpiry(t *testing.T) { if err != nil { t.Fatal(err) } - second, err := repo.CreateIdempotentTx(ctx, tx, req) + second, err := chatsvc.CreateIdempotentTx(ctx, tx, repo, req) if err != nil { t.Fatal(err) } @@ -115,7 +115,7 @@ func TestIdempotencyTransactionAndExpiry(t *testing.T) { if err != nil { t.Fatal(err) } - if _, err = repo.CreateIdempotentTx(ctx, tx, req); err == nil { + if _, err = chatsvc.CreateIdempotentTx(ctx, tx, repo, req); err == nil { t.Fatal("expected injected failure") } if err = tx.Rollback(ctx); err != nil { @@ -134,7 +134,7 @@ func TestIdempotencyTransactionAndExpiry(t *testing.T) { if err != nil { t.Fatal(err) } - if _, err = repo.CreateIdempotentTx(ctx, tx, req); err != nil { + if _, err = chatsvc.CreateIdempotentTx(ctx, tx, repo, req); err != nil { t.Fatal("transient failure was cached", err) } if err = tx.Commit(ctx); err != nil { @@ -257,7 +257,7 @@ func TestIdempotencyTransactionAndExpiry(t *testing.T) { if err != nil { t.Fatal(err) } - _, err = repo.CreateIdempotentTx(ctx, tx, dto.CreateMessage{ChatID: chat, SenderID: user, IdempotencyKey: uuid.NewString(), Text: "link rollback", AttachmentIDs: []uuid.UUID{own, uuid.New()}}) + _, err = chatsvc.CreateIdempotentTx(ctx, tx, repo, dto.CreateMessage{ChatID: chat, SenderID: user, IdempotencyKey: uuid.NewString(), Text: "link rollback", AttachmentIDs: []uuid.UUID{own, uuid.New()}}) if err == nil { t.Fatal("expected missing attachment FK failure") } diff --git a/service/internal/adapter/repository/storage/jobs.go b/service/internal/adapter/repository/storage/jobs.go new file mode 100644 index 00000000..4bb9648d --- /dev/null +++ b/service/internal/adapter/repository/storage/jobs.go @@ -0,0 +1,134 @@ +package storage + +import ( + "context" + "encoding/json" + "gitea.privcloud.tech/mtusi2026/internal/dto" + "github.com/google/uuid" + "github.com/jackc/pgx/v5" + "time" +) + +func (r *AttachmentsRepo) Ping(ctx context.Context) error { return r.Pool.Ping(ctx) } +func (r *AttachmentsRepo) GetTx(ctx context.Context, tx pgx.Tx, id uuid.UUID) (*dto.AttachmentRecord, error) { + return ScanAttachment(tx.QueryRow(ctx, "SELECT "+AttachmentColumns+" FROM attachments WHERE id=$1 FOR UPDATE", id)) +} +func (r *AttachmentsRepo) InsertTx(ctx context.Context, tx pgx.Tx, a *dto.AttachmentRecord) error { + p := a.Public + _, e := tx.Exec(ctx, `INSERT INTO attachments(id,chat_id,owner_id,filename,content_type,size_bytes,sha256,created_at,updated_at,object_key,staging_key,status) VALUES($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$10,$11)`, p.Id, p.ChatId, p.OwnerId, p.Filename, p.ContentType, p.SizeBytes, p.Sha256, p.CreatedAt, p.UpdatedAt, a.StagingKey, p.Status) + return e +} +func (r *AttachmentsRepo) UpdateTx(ctx context.Context, tx pgx.Tx, a *dto.AttachmentRecord) error { + variants, e := json.Marshal(a.Public.Variants) + if e != nil { + return e + } + keys, e := json.Marshal(a.VariantKeys) + if e != nil { + return e + } + _, e = tx.Exec(ctx, `UPDATE attachments SET status=$2,error_code=$3,pipeline_version=$4,staging_key=$5,object_key=$6,variants=$7,variant_keys=$8,updated_at=$9,integrity_missing=$10 WHERE id=$1`, a.Public.Id, a.Public.Status, a.Public.ErrorCode, a.PipelineVersion, a.StagingKey, a.Key, variants, keys, a.Public.UpdatedAt, a.Missing) + return e +} + +const jobColumns = `id,attachment_id,pipeline_version,attempt,created_at,state,token,lease_until,next_attempt_at,attempt_limit,error_code,updated_at` + +func scanJob(row pgx.Row) (*dto.AttachmentJob, error) { + j := new(dto.AttachmentJob) + m := &j.Message + err := row.Scan(&m.JobID, &m.AttachmentID, &m.PipelineVersion, &m.Attempt, &m.CreatedAt, &j.State, &j.Token, &j.LeaseUntil, &j.NextAttemptAt, &j.AttemptLimit, &j.ErrorCode, &j.UpdatedAt) + m.SchemaVersion = 1 + m.Type = "attachment.process.v1" + return j, err +} +func (r *AttachmentsRepo) JobTx(ctx context.Context, tx pgx.Tx, id uuid.UUID) (*dto.AttachmentJob, error) { + return scanJob(tx.QueryRow(ctx, `SELECT `+jobColumns+` FROM attachment_jobs WHERE id=$1 FOR UPDATE`, id)) +} +func (r *AttachmentsRepo) InsertJobTx(ctx context.Context, tx pgx.Tx, j *dto.AttachmentJob) error { + m := j.Message + _, e := tx.Exec(ctx, `INSERT INTO attachment_jobs(`+jobColumns+`) VALUES($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12)`, m.JobID, m.AttachmentID, m.PipelineVersion, m.Attempt, m.CreatedAt, j.State, j.Token, j.LeaseUntil, j.NextAttemptAt, j.AttemptLimit, j.ErrorCode, j.UpdatedAt) + return e +} +func (r *AttachmentsRepo) SaveJobTx(ctx context.Context, tx pgx.Tx, j *dto.AttachmentJob) error { + _, e := tx.Exec(ctx, `UPDATE attachment_jobs SET attempt=$2,state=$3,token=$4,lease_until=$5,next_attempt_at=$6,attempt_limit=$7,error_code=$8,updated_at=$9 WHERE id=$1`, j.Message.JobID, j.Message.Attempt, j.State, j.Token, j.LeaseUntil, j.NextAttemptAt, j.AttemptLimit, j.ErrorCode, j.UpdatedAt) + return e +} +func (r *AttachmentsRepo) ExpiredJobs(ctx context.Context) ([]uuid.UUID, error) { + rows, e := r.Pool.Query(ctx, `SELECT id FROM attachment_jobs WHERE state='processing' AND lease_until>'attempt')::integer=j.attempt LIMIT 100`) + if e != nil { + return nil, e + } + defer rows.Close() + var ids []uuid.UUID + for rows.Next() { + var id uuid.UUID + if e = rows.Scan(&id); e != nil { + return nil, e + } + ids = append(ids, id) + } + return ids, rows.Err() +} +func (r *AttachmentsRepo) RetiredSources(ctx context.Context, before time.Time) ([]*dto.AttachmentRecord, error) { + rows, e := r.Pool.Query(ctx, `SELECT `+AttachmentColumns+` FROM attachments WHERE staging_key<>'' AND (status='ready' OR (status='failed' AND updated_at<$1)) LIMIT 100`, before) + if e != nil { + return nil, e + } + defer rows.Close() + var out []*dto.AttachmentRecord + for rows.Next() { + a, e := ScanAttachment(rows) + if e != nil { + return nil, e + } + out = append(out, a) + } + return out, rows.Err() +} + +func (r *AttachmentsRepo) SaveVariantTx(ctx context.Context, tx pgx.Tx, id uuid.UUID, version int, name, key, sha string, size int64) error { + _, e := tx.Exec(ctx, `INSERT INTO attachment_variants(attachment_id,pipeline_version,variant,object_key,sha256,size_bytes) VALUES($1,$2,$3,$4,$5,$6)`, id, version, name, key, sha, size) + return e +} + +func (r *AttachmentsRepo) PublicationDeadTx(ctx context.Context, tx pgx.Tx, id uuid.UUID, attempt int) (bool, error) { + var dead bool + e := tx.QueryRow(ctx, `SELECT EXISTS(SELECT 1 FROM outbox WHERE job_id=$1 AND status='dead_letter' AND (payload->>'attempt')::integer=$2)`, id, attempt).Scan(&dead) + return dead, e +} diff --git a/service/internal/adapter/repository/storage/messages.go b/service/internal/adapter/repository/storage/messages.go index 61c29ba8..0c1b6f90 100644 --- a/service/internal/adapter/repository/storage/messages.go +++ b/service/internal/adapter/repository/storage/messages.go @@ -1,10 +1,8 @@ package storage import ( - "bytes" "context" - "crypto/sha256" - "encoding/json" + "errors" "time" @@ -26,55 +24,49 @@ func NewMessagesRepo(pool *pgxpool.Pool) *MessagesRepo { return &MessagesRepo{pool: pool} } -// CreateIdempotentTx serializes only requests sharing an idempotency scope. -// The transaction commits both the message and its replay record, or neither. -func (r *MessagesRepo) CreateIdempotentTx(ctx context.Context, tx pgx.Tx, req dto.CreateMessage) (*dto.Message, error) { - scope := req.SenderID.String() + ":POST:" + req.ChatID.String() + ":" + req.IdempotencyKey - if _, err := tx.Exec(ctx, `SELECT pg_advisory_xact_lock(hashtextextended($1, 0))`, scope); err != nil { - return nil, apperr.WrapPostgres(err, "failed to lock message key") +// LockKeyTx serializes persistence for one idempotency scope. +func (r *MessagesRepo) LockKeyTx(ctx context.Context, tx pgx.Tx, scope string) error { + _, err := tx.Exec(ctx, `SELECT pg_advisory_xact_lock(hashtextextended($1,0))`, scope) + if err != nil { + return apperr.WrapPostgres(err, "failed to lock message key") } - hash := messageHash(req) - var hashVersion int - var storedHash []byte - var messageID uuid.UUID - err := tx.QueryRow(ctx, `SELECT request_hash, message_id, hash_version FROM message_idempotency - WHERE user_id=$1 AND method='POST' AND chat_id=$2 AND key=$3 AND expires_at>clock_timestamp()`, - req.SenderID, req.ChatID, req.IdempotencyKey).Scan(&storedHash, &messageID, &hashVersion) - if err == nil { - if hashVersion == 1 && len(req.AttachmentIDs) == 0 { - hash = sha256.Sum256([]byte(req.Text)) - } else if hashVersion != 2 { - return nil, apperr.New(apperr.CodeConflict, "idempotency key used with different content") - } - if !bytes.Equal(hash[:], storedHash) { - return nil, apperr.New(apperr.CodeConflict, "idempotency key used with different content") - } - return scanMessage(tx.QueryRow(ctx, `SELECT `+messageColumns+` FROM messages WHERE id=$1`, messageID)) + return nil +} +func (r *MessagesRepo) ReplayTx(ctx context.Context, tx pgx.Tx, req dto.CreateMessage) (*dto.MessageReplay, error) { + v := new(dto.MessageReplay) + err := tx.QueryRow(ctx, `SELECT request_hash,message_id,hash_version FROM message_idempotency WHERE user_id=$1 AND method='POST' AND chat_id=$2 AND key=$3 AND expires_at>clock_timestamp()`, req.SenderID, req.ChatID, req.IdempotencyKey).Scan(&v.Hash, &v.MessageID, &v.HashVersion) + if errors.Is(err, pgx.ErrNoRows) { + return nil, nil } - if !errors.Is(err, pgx.ErrNoRows) { + if err != nil { return nil, apperr.WrapPostgres(err, "failed to read message key") } - message := dto.Message{ID: uuid.New(), ChatID: req.ChatID, SenderID: req.SenderID, Text: req.Text, AttachmentIDs: append([]uuid.UUID{}, req.AttachmentIDs...), CreatedAt: time.Now().UTC().Truncate(time.Microsecond)} - if _, err = tx.Exec(ctx, `INSERT INTO messages(id,chat_id,sender_id,text,created_at) VALUES($1,$2,$3,$4,$5)`, - message.ID, message.ChatID, message.SenderID, message.Text, message.CreatedAt); err != nil { + return v, nil +} +func (r *MessagesRepo) MessageTx(ctx context.Context, tx pgx.Tx, id uuid.UUID) (*dto.Message, error) { + return scanMessage(tx.QueryRow(ctx, `SELECT `+messageColumns+` FROM messages WHERE id=$1`, id)) +} +func (r *MessagesRepo) InsertMessageTx(ctx context.Context, tx pgx.Tx, message dto.Message) error { + _, err := tx.Exec(ctx, `INSERT INTO messages(id,chat_id,sender_id,text,created_at) VALUES($1,$2,$3,$4,$5)`, message.ID, message.ChatID, message.SenderID, message.Text, message.CreatedAt) + if err != nil { if pgErr, ok := errors.AsType[*pgconn.PgError](err); ok && pgErr.Code == pgerrcode.ForeignKeyViolation { - return nil, apperr.Wrap(err, apperr.CodeNotFound, "chat not found") + return apperr.Wrap(err, apperr.CodeNotFound, "chat not found") } - return nil, apperr.WrapPostgres(err, "failed to create message") + return apperr.WrapPostgres(err, "failed to create message") } - for position, attachmentID := range req.AttachmentIDs { - if _, err = tx.Exec(ctx, `INSERT INTO message_attachments(message_id,chat_id,attachment_id,position) VALUES($1,$2,$3,$4)`, message.ID, message.ChatID, attachmentID, position); err != nil { - return nil, apperr.WrapPostgres(err, "failed to link attachment") + for position, id := range message.AttachmentIDs { + if _, err = tx.Exec(ctx, `INSERT INTO message_attachments(message_id,chat_id,attachment_id,position) VALUES($1,$2,$3,$4)`, message.ID, message.ChatID, id, position); err != nil { + return apperr.WrapPostgres(err, "failed to link attachment") } } - - if _, err = tx.Exec(ctx, `INSERT INTO message_idempotency(user_id,method,chat_id,key,request_hash,message_id,expires_at,hash_version) - VALUES($1,'POST',$2,$3,$4,$5,clock_timestamp()+interval '24 hours',2) - ON CONFLICT(user_id,method,chat_id,key) DO UPDATE SET request_hash=EXCLUDED.request_hash, - message_id=EXCLUDED.message_id,expires_at=EXCLUDED.expires_at,hash_version=EXCLUDED.hash_version`, req.SenderID, req.ChatID, req.IdempotencyKey, hash[:], message.ID); err != nil { - return nil, apperr.WrapPostgres(err, "failed to save message key") + return nil +} +func (r *MessagesRepo) SaveKeyTx(ctx context.Context, tx pgx.Tx, req dto.CreateMessage, hash []byte, messageID uuid.UUID, expires time.Time) error { + _, err := tx.Exec(ctx, `INSERT INTO message_idempotency(user_id,method,chat_id,key,request_hash,message_id,expires_at,hash_version) VALUES($1,'POST',$2,$3,$4,$5,$6,2) ON CONFLICT(user_id,method,chat_id,key) DO UPDATE SET request_hash=EXCLUDED.request_hash,message_id=EXCLUDED.message_id,expires_at=EXCLUDED.expires_at,hash_version=EXCLUDED.hash_version`, req.SenderID, req.ChatID, req.IdempotencyKey, hash, messageID, expires) + if err != nil { + return apperr.WrapPostgres(err, "failed to save message key") } - return &message, nil + return nil } func (r *MessagesRepo) GetByIDForChat(ctx context.Context, chatID, messageID uuid.UUID) (*dto.Message, error) { @@ -120,20 +112,10 @@ func scanMessage(row pgx.Row) (*dto.Message, error) { const messageColumns = `id, chat_id, sender_id, text, created_at, ARRAY(SELECT attachment_id FROM message_attachments WHERE message_id=messages.id ORDER BY position)` -func messageHash(req dto.CreateMessage) [32]byte { - // Field order and a non-nil list define the canonical v2 representation. - canonical := struct { - Text string `json:"text"` - AttachmentIDs []uuid.UUID `json:"attachment_ids"` - }{req.Text, append([]uuid.UUID{}, req.AttachmentIDs...)} - encoded, _ := json.Marshal(canonical) // strings and UUIDs have no encoding failures. - return sha256.Sum256(encoded) -} - func (r *MessagesRepo) AttachmentsForMessageTx(ctx context.Context, tx pgx.Tx, ids []uuid.UUID) ([]dto.MessageAttachment, error) { // Prevent a concurrent integrity update from invalidating the checked rows // until the message and its links have committed together. - rows, err := tx.Query(ctx, `SELECT id, chat_id, integrity_missing FROM attachments WHERE id=ANY($1) ORDER BY id FOR SHARE`, ids) + rows, err := tx.Query(ctx, `SELECT id, chat_id, integrity_missing,status FROM attachments WHERE id=ANY($1) ORDER BY id FOR SHARE`, ids) if err != nil { return nil, apperr.WrapPostgres(err, "failed to read attachments") } @@ -141,7 +123,7 @@ func (r *MessagesRepo) AttachmentsForMessageTx(ctx context.Context, tx pgx.Tx, i result := make([]dto.MessageAttachment, 0, len(ids)) for rows.Next() { var a dto.MessageAttachment - if err := rows.Scan(&a.ID, &a.ChatID, &a.Missing); err != nil { + if err := rows.Scan(&a.ID, &a.ChatID, &a.Missing, &a.Status); err != nil { return nil, apperr.WrapPostgres(err, "failed to read attachment") } result = append(result, a) diff --git a/service/internal/adapter/repository/storage/outbox.go b/service/internal/adapter/repository/storage/outbox.go new file mode 100644 index 00000000..ffc31913 --- /dev/null +++ b/service/internal/adapter/repository/storage/outbox.go @@ -0,0 +1,52 @@ +package storage + +import ( + "context" + "errors" + "gitea.privcloud.tech/mtusi2026/internal/service/outbox" + "github.com/google/uuid" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" + "time" +) + +type OutboxRepo struct{ Pool *pgxpool.Pool } + +func (r *OutboxRepo) CreateTx(ctx context.Context, tx pgx.Tx, q outbox.CreateMessageReq) (outbox.CreateMessageRes, error) { + _, e := tx.Exec(ctx, `INSERT INTO outbox(id,job_id,payload,status,max_attempts,token,next_attempt_at,created_at,updated_at) VALUES($1,$2,$3,$4,$5,$6,$7,$7,$7)`, q.ID, q.AggregateID, q.Payload, q.Status, q.MaxAttempts, uuid.New(), q.CreatedAt) + return outbox.CreateMessageRes{ID: q.ID, CreatedAt: q.CreatedAt}, e +} +func scanOutbox(rows pgx.Rows) (out []outbox.AcquiredMessage, err error) { + defer rows.Close() + for rows.Next() { + var m outbox.AcquiredMessage + if err = rows.Scan(&m.ID, &m.AggregateID, &m.Payload, &m.Attempts, &m.MaxAttempts, &m.Token); err != nil { + return + } + out = append(out, m) + } + err = rows.Err() + return +} +func (r *OutboxRepo) AcquireManyForProcessing(ctx context.Context, q outbox.AcquireMessagesForProcessingReq) ([]outbox.AcquiredMessage, error) { + rows, e := r.Pool.Query(ctx, `WITH picked AS (SELECT id FROM outbox WHERE status IN ('pending','failed') AND next_attempt_at<=clock_timestamp() ORDER BY next_attempt_at,id LIMIT $1 FOR UPDATE SKIP LOCKED) + UPDATE outbox o SET status='processing',attempts=attempts+1,token=$2,locked_until=clock_timestamp()+$3::interval,updated_at=clock_timestamp() FROM picked p WHERE o.id=p.id RETURNING o.id,o.job_id,o.payload,o.attempts,o.max_attempts,o.token`, q.Limit, uuid.New(), q.LockDuration.String()) + if e != nil { + return nil, e + } + return scanOutbox(rows) +} +func (r *OutboxRepo) Expired(ctx context.Context) ([]outbox.AcquiredMessage, error) { + rows, e := r.Pool.Query(ctx, `SELECT id,job_id,payload,attempts,max_attempts,token FROM outbox WHERE status='processing' AND locked_until 64 { + return nil, fmt.Errorf("invalid %s", k) + } + *p = n + } + } + for k, p := range map[string]*time.Duration{"JOB_TIMEOUT": &c.Timeout, "JOB_LEASE": &c.Lease, "STAGING_RETENTION": &c.Retention} { + if v := os.Getenv(k); v != "" { + d, e := time.ParseDuration(v) + if e != nil || d <= 0 { + return nil, fmt.Errorf("invalid %s", k) + } + *p = d + } + } + if c.Lease <= c.Timeout || c.Prefetch > c.Concurrency { + return nil, fmt.Errorf("lease must exceed timeout; prefetch must not exceed concurrency") + } + if p := os.Getenv("BROKER_URL_FILE"); p != "" { + b, e := os.ReadFile(p) + if e != nil { + return nil, fmt.Errorf("read broker secret: %w", e) + } + c.BrokerURL = strings.TrimSpace(string(b)) + } + return c, nil +} diff --git a/service/internal/dto/attachment.go b/service/internal/dto/attachment.go index 76d3a172..1469e14c 100644 --- a/service/internal/dto/attachment.go +++ b/service/internal/dto/attachment.go @@ -1 +1,48 @@ package dto + +import ( + "gitea.privcloud.tech/mtusi2026/internal/gen/contracts" + "github.com/google/uuid" + "io" + "time" +) + +type AttachmentRecord struct { + Public contracts.Attachment + Key string + StagingKey string + PipelineVersion int + VariantKeys map[string]string + Missing bool +} +type UploadAttachment struct { + UserID, ChatID uuid.UUID + Filename, ContentType string + Body io.Reader +} +type AttachmentContent struct { + Body io.ReadCloser + ContentType string + Size int64 +} +type JobMessage struct { + Traceparent string `json:"traceparent,omitempty"` + Tracestate string `json:"tracestate,omitempty"` + SchemaVersion int `json:"schema_version"` + JobID uuid.UUID `json:"job_id"` + Type string `json:"type"` + AttachmentID uuid.UUID `json:"attachment_id"` + PipelineVersion int `json:"pipeline_version"` + Attempt int `json:"attempt"` + CreatedAt time.Time `json:"created_at"` +} +type AttachmentJob struct { + Message JobMessage + State string + Token uuid.UUID + LeaseUntil *time.Time + NextAttemptAt time.Time + AttemptLimit int + ErrorCode *string + UpdatedAt time.Time +} diff --git a/service/internal/dto/message.go b/service/internal/dto/message.go index b85296da..48f33b64 100644 --- a/service/internal/dto/message.go +++ b/service/internal/dto/message.go @@ -35,4 +35,11 @@ type MessageAttachment struct { ID uuid.UUID ChatID uuid.UUID Missing bool + Status string +} + +type MessageReplay struct { + Hash []byte + HashVersion int + MessageID uuid.UUID } diff --git a/service/internal/gen/contracts/types.gen.go b/service/internal/gen/contracts/types.gen.go index 28d12dbf..977fa8c7 100644 --- a/service/internal/gen/contracts/types.gen.go +++ b/service/internal/gen/contracts/types.gen.go @@ -22,7 +22,10 @@ const ( // Defines values for AttachmentStatus. const ( - Ready AttachmentStatus = "ready" + Failed AttachmentStatus = "failed" + Processing AttachmentStatus = "processing" + Queued AttachmentStatus = "queued" + Ready AttachmentStatus = "ready" ) // Defines values for ErrorErrorCode. @@ -52,14 +55,17 @@ const ( // Defines values for VariantName. const ( - Crop VariantName = "crop" - Optimized VariantName = "optimized" - Thumbnail VariantName = "thumbnail" + VariantNameCrop VariantName = "crop" + VariantNameOptimized VariantName = "optimized" + VariantNameThumbnail VariantName = "thumbnail" ) // Defines values for DownloadAttachmentParamsVariant. const ( - Original DownloadAttachmentParamsVariant = "original" + DownloadAttachmentParamsVariantCrop DownloadAttachmentParamsVariant = "crop" + DownloadAttachmentParamsVariantOptimized DownloadAttachmentParamsVariant = "optimized" + DownloadAttachmentParamsVariantOriginal DownloadAttachmentParamsVariant = "original" + DownloadAttachmentParamsVariantThumbnail DownloadAttachmentParamsVariant = "thumbnail" ) // Attachment defines model for Attachment. diff --git a/service/internal/pkg/fault/fault.go b/service/internal/pkg/fault/fault.go new file mode 100644 index 00000000..0d5af152 --- /dev/null +++ b/service/internal/pkg/fault/fault.go @@ -0,0 +1,64 @@ +// Package fault implements explicit, file-controlled barriers for the isolated lab. +package fault + +import ( + "context" + "errors" + "github.com/google/uuid" + "os" + "path/filepath" + "strings" + "time" +) + +func Hit(ctx context.Context, point string, id uuid.UUID) { + if os.Getenv("LAB5_FAULTS") != "1" { + return + } + p := filepath.Join("/faults", point+"."+id.String()) + b, e := os.ReadFile(p) + if e != nil { + p = filepath.Join("/faults", point+".all") + b, e = os.ReadFile(p) + if e != nil { + return + } + } + hit, e := os.OpenFile(p+".hit", os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0666) + if e != nil { + return + } + _, _ = hit.WriteString(id.String() + "\n") + _ = hit.Close() + switch strings.TrimSpace(string(b)) { + case "crash": + _ = os.Remove(p) + os.Exit(86) + case "pause": + for { + if _, e = os.Stat(p); os.IsNotExist(e) { + return + } + select { + case <-ctx.Done(): + return + case <-time.After(50 * time.Millisecond): + } + } + } +} + +func Error(point string, id uuid.UUID) error { + if os.Getenv("LAB5_FAULTS") != "1" { + return nil + } + b, e := os.ReadFile(filepath.Join("/faults", point+"."+id.String())) + if e != nil { + return nil + } + v := strings.TrimSpace(string(b)) + if v == "temporary" || v == "permanent" { + return errors.New(v) + } + return nil +} diff --git a/service/internal/pkg/imageprocessor/image.go b/service/internal/pkg/imageprocessor/image.go new file mode 100644 index 00000000..b87678c7 --- /dev/null +++ b/service/internal/pkg/imageprocessor/image.go @@ -0,0 +1,83 @@ +package imageprocessor + +import ( + "context" + "fmt" + "image" + _ "image/jpeg" + _ "image/png" + "os" + "os/exec" + "path/filepath" + "time" +) + +func run(ctx context.Context, args ...string) error { + cmd := exec.CommandContext(ctx, "magick", args...) + cmd.Env = append(os.Environ(), "MAGICK_MEMORY_LIMIT=256MiB", "MAGICK_MAP_LIMIT=0", "MAGICK_DISK_LIMIT=256MiB", "MAGICK_THREAD_LIMIT=1", "MAGICK_TIME_LIMIT=60") + cmd.WaitDelay = time.Second + if err := cmd.Run(); err != nil { + if ctx.Err() != nil { + return ctx.Err() + } + return fmt.Errorf("image conversion failed: %w", err) + } + return nil +} +func Validate(ctx context.Context, path string) error { + return run(ctx, "-regard-warnings", path, "null:") +} + +type Result struct { + Name, Path, MIME string + Width, Height int +} + +func fit(w, h, limit int) (int, int) { + if w <= limit && h <= limit { + return w, h + } + if w >= h { + return limit, max(1, h*limit/w) + } + return max(1, w*limit/h), limit +} +func Process(ctx context.Context, source, dir string, version int) ([]Result, error) { + normalized := filepath.Join(dir, "normalized.png") + if e := run(ctx, source, "-auto-orient", "-background", "white", "-alpha", "remove", "-alpha", "off", "-strip", "png:"+normalized); e != nil { + return nil, e + } + f, e := os.Open(normalized) + if e != nil { + return nil, e + } + cfg, _, e := image.DecodeConfig(f) + f.Close() + if e != nil { + return nil, e + } + tw, th := fit(cfg.Width, cfg.Height, 256) + ow, oh := fit(cfg.Width, cfg.Height, 1280) + out := []Result{{"thumbnail", filepath.Join(dir, "thumbnail.jpg"), "image/jpeg", tw, th}, {"crop", filepath.Join(dir, "crop.jpg"), "image/jpeg", 128, 128}, {"optimized", filepath.Join(dir, "optimized.webp"), "image/webp", ow, oh}} + for _, v := range out { + args := []string{normalized} + if v.Name == "crop" { + side := min(cfg.Width, cfg.Height) + args = append(args, "-gravity", "center", "-crop", fmt.Sprintf("%dx%d+0+0", side, side), "+repage") + } + args = append(args, "-resize", fmt.Sprintf("%dx%d!", v.Width, v.Height), "-strip", "-quality") + quality := "85" + if v.Name == "optimized" && version == 1 { + quality = "80" + } + coder := "jpeg:" + if v.Name == "optimized" { + coder = "webp:" + } + args = append(args, quality, coder+v.Path) + if e = run(ctx, args...); e != nil { + return nil, e + } + } + return out, nil +} diff --git a/service/internal/pkg/objectstore/s3.go b/service/internal/pkg/objectstore/s3.go index 5f90f281..78822da3 100644 --- a/service/internal/pkg/objectstore/s3.go +++ b/service/internal/pkg/objectstore/s3.go @@ -67,3 +67,11 @@ func Missing(err error) bool { var api smithy.APIError return errors.As(err, &api) && (api.ErrorCode() == "NoSuchKey" || api.ErrorCode() == "NotFound" || api.ErrorCode() == "404") } + +func (s *Store) List(ctx context.Context, token string) (*s3.ListObjectsV2Output, error) { + q := &s3.ListObjectsV2Input{Bucket: &s.Bucket, MaxKeys: aws.Int32(100)} + if token != "" { + q.ContinuationToken = &token + } + return s.Client.ListObjectsV2(ctx, q) +} diff --git a/service/internal/service/attachment/interface.go b/service/internal/service/attachment/interface.go new file mode 100644 index 00000000..71599bf0 --- /dev/null +++ b/service/internal/service/attachment/interface.go @@ -0,0 +1,51 @@ +package attachment + +import ( + "context" + "gitea.privcloud.tech/mtusi2026/internal/dto" + "gitea.privcloud.tech/mtusi2026/internal/service/outbox" + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/google/uuid" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" + "io" + "time" +) + +type TransactionManager interface { + RunInTransaction(context.Context, *pgx.TxOptions, func(pgx.Tx) error) error +} +type Outbox interface { + CreateTx(context.Context, pgx.Tx, outbox.CreateMessageReq) (outbox.CreateMessageRes, error) +} +type ObjectStore interface { + Put(context.Context, string, string, io.Reader, int64) error + Get(context.Context, string) (*s3.GetObjectOutput, error) + Head(context.Context, string) (*s3.HeadObjectOutput, error) + Delete(context.Context, string) error + Ping(context.Context) error + List(context.Context, string) (*s3.ListObjectsV2Output, error) +} +type Repository interface { + SaveVariantTx(context.Context, pgx.Tx, uuid.UUID, int, string, string, string, int64) error + + Ping(context.Context) error + Get(context.Context, uuid.UUID) (*dto.AttachmentRecord, error) + GetTx(context.Context, pgx.Tx, uuid.UUID) (*dto.AttachmentRecord, error) + InsertTx(context.Context, pgx.Tx, *dto.AttachmentRecord) error + UpdateTx(context.Context, pgx.Tx, *dto.AttachmentRecord) error + JobTx(context.Context, pgx.Tx, uuid.UUID) (*dto.AttachmentJob, error) + InsertJobTx(context.Context, pgx.Tx, *dto.AttachmentJob) error + SaveJobTx(context.Context, pgx.Tx, *dto.AttachmentJob) error + ExpiredJobs(context.Context) ([]uuid.UUID, error) + PublicationDeadTx(context.Context, pgx.Tx, uuid.UUID, int) (bool, error) + FailedOutbox(context.Context) ([]uuid.UUID, error) + ListJobs(context.Context) ([]dto.AttachmentJob, error) + RetiredSources(context.Context, time.Time) ([]*dto.AttachmentRecord, error) + AuditTx(context.Context, pgx.Tx, uuid.UUID, string) error + Lock(context.Context, string) (*pgxpool.Conn, func(), error) + Exists(context.Context, *pgxpool.Conn, string) (bool, error) + Page(context.Context, uuid.UUID) ([]*dto.AttachmentRecord, error) + MarkMissing(context.Context, *pgxpool.Conn, uuid.UUID, bool) (bool, error) + CheckConn(context.Context, *pgxpool.Conn) error +} diff --git a/service/internal/service/attachment/pipeline.go b/service/internal/service/attachment/pipeline.go new file mode 100644 index 00000000..fcdb6cc8 --- /dev/null +++ b/service/internal/service/attachment/pipeline.go @@ -0,0 +1,486 @@ +package attachment + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/json" + "errors" + "fmt" + "gitea.privcloud.tech/mtusi2026/internal/dto" + "gitea.privcloud.tech/mtusi2026/internal/gen/contracts" + "gitea.privcloud.tech/mtusi2026/internal/pkg/apperr" + "gitea.privcloud.tech/mtusi2026/internal/pkg/fault" + "gitea.privcloud.tech/mtusi2026/internal/pkg/imageprocessor" + "gitea.privcloud.tech/mtusi2026/internal/pkg/objectstore" + "gitea.privcloud.tech/mtusi2026/internal/service/outbox" + "github.com/google/uuid" + "github.com/jackc/pgx/v5" + "io" + "os" + "path/filepath" + "strings" + "time" +) + +var ErrBusy = errors.New("job lease held") + +func validateImage(ctx context.Context, path string) error { return imageprocessor.Validate(ctx, path) } +func (s *Service) Fault(ctx context.Context, point string, id uuid.UUID) { fault.Hit(ctx, point, id) } +func (s *Service) newJob(ctx context.Context, tx pgx.Tx, a *dto.AttachmentRecord) error { + now := time.Now().UTC() + j := &dto.AttachmentJob{Message: dto.JobMessage{SchemaVersion: 1, JobID: uuid.New(), Type: "attachment.process.v1", AttachmentID: a.Public.Id, PipelineVersion: a.PipelineVersion, Attempt: 1, CreatedAt: now}, State: "queued", Token: uuid.New(), NextAttemptAt: now, AttemptLimit: s.Jobs.MaxAttempts, UpdatedAt: now} + if e := s.Repo.InsertJobTx(ctx, tx, j); e != nil { + return e + } + return s.enqueue(ctx, tx, j) +} +func (s *Service) enqueue(ctx context.Context, tx pgx.Tx, j *dto.AttachmentJob) error { + b, e := json.Marshal(j.Message) + if e != nil { + return e + } + _, e = s.outbox.CreateTx(ctx, tx, outbox.CreateMessageReq{ID: uuid.New(), AggregateID: j.Message.JobID, Transport: outbox.TransportAMQP, Payload: b, CreatedAt: j.NextAttemptAt, MaxAttempts: s.Jobs.MaxAttempts}) + return e +} +func (s *Service) Content(ctx context.Context, user, id uuid.UUID, variant string) (*dto.AttachmentContent, error) { + a, e := s.Get(ctx, user, id) + if e != nil { + return nil, e + } + if a.Public.Status != "ready" { + return nil, apperr.New(apperr.CodeConflict, "attachment not ready") + } + key, mime, size := a.Key, string(a.Public.ContentType), int64(a.Public.SizeBytes) + if variant != "original" { + key = a.VariantKeys[variant] + if key == "" { + return nil, apperr.New(apperr.CodeNotFound, "variant not found") + } + for _, v := range a.Public.Variants { + if string(v.Name) == variant { + mime = string(v.ContentType) + size = int64(v.SizeBytes) + } + } + } + obj, e := s.Store.Get(ctx, key) + if e != nil { + return nil, Unavailable() + } + return &dto.AttachmentContent{Body: obj.Body, ContentType: mime, Size: size}, nil +} +func (s *Service) Process(ctx context.Context, m dto.JobMessage) error { + var j *dto.AttachmentJob + var a *dto.AttachmentRecord + claimed := false + err := s.tm.RunInTransaction(ctx, nil, func(tx pgx.Tx) error { + var e error + j, e = s.Repo.JobTx(ctx, tx, m.JobID) + if errors.Is(e, pgx.ErrNoRows) { + return nil + } + if e != nil { + return e + } + // Broker data is a notification. Trusted metadata and attempt live in PostgreSQL. + if j.Message.AttachmentID != m.AttachmentID || j.Message.PipelineVersion != m.PipelineVersion || j.Message.Attempt != m.Attempt { + return nil + } + a, e = s.Repo.GetTx(ctx, tx, m.AttachmentID) + if e != nil { + return e + } + if a.PipelineVersion != m.PipelineVersion || j.State == "ready" || j.State == "failed" { + return nil + } + if j.State == "processing" { + return ErrBusy + } + if time.Now().Before(j.NextAttemptAt) { + return ErrBusy + } + j.State = "processing" + j.Token = uuid.New() + lease := time.Now().Add(s.Jobs.Lease) + j.LeaseUntil = &lease + j.UpdatedAt = time.Now().UTC() + a.Public.Status = "processing" + a.Public.UpdatedAt = j.UpdatedAt + if e = s.Repo.SaveJobTx(ctx, tx, j); e != nil { + return e + } + if e = s.Repo.UpdateTx(ctx, tx, a); e != nil { + return e + } + claimed = true + return nil + }) + if err != nil || !claimed { + return err + } + s.Fault(ctx, "after_claim", m.JobID) + task, cancel := context.WithTimeout(ctx, s.Jobs.Timeout) + defer cancel() + result, e := s.effects(task, j, a) + if e != nil { + if ctx.Err() != nil { + return ctx.Err() + } + return s.finish(ctx, j, nil, e) + } + s.Fault(ctx, "after_objects", m.JobID) + if err = s.finish(ctx, j, result, nil); err != nil { + return err + } + s.Fault(ctx, "after_result_commit", m.JobID) + return nil +} + +type processed struct { + Key string + Variants []contracts.Variant + Keys map[string]string +} +type permanentError struct{ code string } + +func (e permanentError) Error() string { return e.code } +func (s *Service) effects(ctx context.Context, j *dto.AttachmentJob, a *dto.AttachmentRecord) (*processed, error) { + dir, e := os.MkdirTemp("", "pipeline-*") + if e != nil { + return nil, e + } + defer os.RemoveAll(dir) + key := a.StagingKey + if key == "" { + key = a.Key + } + obj, e := s.Store.Get(ctx, key) + if e != nil { + if objectstore.Missing(e) { + return nil, permanentError{"source_missing"} + } + return nil, e + } + defer obj.Body.Close() + path := filepath.Join(dir, "source") + f, e := os.Create(path) + if e != nil { + return nil, e + } + hash := sha256.New() + n, e := io.Copy(io.MultiWriter(f, hash), io.LimitReader(obj.Body, s.Config.MaxBytes+1)) + f.Close() + if e != nil { + return nil, e + } + if n != int64(a.Public.SizeBytes) || fmt.Sprintf("%x", hash.Sum(nil)) != a.Public.Sha256 { + return nil, permanentError{"source_corrupt"} + } + f, e = os.Open(path) + if e != nil { + return nil, e + } + e = s.Validate(ctx, f, string(a.Public.ContentType)) + f.Close() + if e != nil { + if ctx.Err() != nil { + return nil, ctx.Err() + } + return nil, permanentError{"invalid_image"} + } + s.Fault(ctx, "before_objects", j.Message.JobID) + if e := fault.Error("before_objects", j.Message.JobID); e != nil { + if e.Error() == "permanent" { + return nil, permanentError{"invalid_image"} + } + return nil, e + } + converted, e := imageprocessor.Process(ctx, path, dir, j.Message.PipelineVersion) + if e != nil { + return nil, e + } + prefix := fmt.Sprintf("results/%s/v%d/", a.Public.Id, j.Message.PipelineVersion) + result := &processed{Key: prefix + "original/" + a.Public.Sha256, Variants: []contracts.Variant{}, Keys: map[string]string{}} + f, e = os.Open(path) + if e != nil { + return nil, e + } + e = s.Store.Put(ctx, result.Key, string(a.Public.ContentType), f, n) + f.Close() + if e != nil { + return nil, e + } + s.Fault(ctx, "after_original_put", j.Message.JobID) + for _, v := range converted { + b, e := os.ReadFile(v.Path) + if e != nil { + return nil, e + } + sum := fmt.Sprintf("%x", sha256.Sum256(b)) + key := prefix + v.Name + "/" + sum + if e = s.Store.Put(ctx, key, v.MIME, bytes.NewReader(b), int64(len(b))); e != nil { + return nil, e + } + result.Keys[v.Name] = key + result.Variants = append(result.Variants, contracts.Variant{Name: contracts.VariantName(v.Name), ContentType: contracts.VariantContentType(v.MIME), Width: v.Width, Height: v.Height, SizeBytes: len(b), Sha256: sum}) + } + return result, nil +} +func (s *Service) finish(ctx context.Context, owned *dto.AttachmentJob, result *processed, cause error) error { + return s.tm.RunInTransaction(ctx, nil, func(tx pgx.Tx) error { + j, e := s.Repo.JobTx(ctx, tx, owned.Message.JobID) + if e != nil { + return e + } + a, e := s.Repo.GetTx(ctx, tx, j.Message.AttachmentID) + if e != nil { + return e + } + if j.Token != owned.Token || j.State != "processing" || j.LeaseUntil == nil || !j.LeaseUntil.After(time.Now()) || a.PipelineVersion != j.Message.PipelineVersion { + return nil + } + if cause != nil { + return s.retry(ctx, tx, j, a, cause) + } + + if e = s.Repo.SaveVariantTx(ctx, tx, a.Public.Id, j.Message.PipelineVersion, "original", result.Key, a.Public.Sha256, int64(a.Public.SizeBytes)); e != nil { + return e + } + for _, v := range result.Variants { + if e = s.Repo.SaveVariantTx(ctx, tx, a.Public.Id, j.Message.PipelineVersion, string(v.Name), result.Keys[string(v.Name)], v.Sha256, int64(v.SizeBytes)); e != nil { + return e + } + } + j.State = "ready" + j.ErrorCode = nil + j.LeaseUntil = nil + j.UpdatedAt = time.Now().UTC() + a.Missing = false + a.Key = result.Key + a.VariantKeys = result.Keys + a.Public.Variants = result.Variants + a.Public.Status = "ready" + a.Public.ErrorCode = nil + a.Public.UpdatedAt = j.UpdatedAt + if e = s.Repo.SaveJobTx(ctx, tx, j); e != nil { + return e + } + return s.Repo.UpdateTx(ctx, tx, a) + }) +} +func (s *Service) retry(ctx context.Context, tx pgx.Tx, j *dto.AttachmentJob, a *dto.AttachmentRecord, cause error) error { + code := "processing_failed" + var permanent permanentError + terminal := errors.As(cause, &permanent) || j.Message.Attempt >= j.AttemptLimit + if permanent.code != "" { + code = permanent.code + } + if terminal && permanent.code == "" { + code = "retry_exhausted" + } + j.LeaseUntil = nil + j.Token = uuid.New() + j.UpdatedAt = time.Now().UTC() + j.ErrorCode = &code + if terminal { + j.State = "failed" + a.Public.Status = "failed" + a.Public.ErrorCode = &code + } else { + j.State = "queued" + j.Message.Attempt++ + j.NextAttemptAt = j.UpdatedAt.Add(outbox.Backoff(j.Message.Attempt - 1)) + a.Public.Status = "queued" + a.Public.ErrorCode = nil + if e := s.enqueue(ctx, tx, j); e != nil { + return e + } + } + a.Public.UpdatedAt = j.UpdatedAt + if e := s.Repo.SaveJobTx(ctx, tx, j); e != nil { + return e + } + return s.Repo.UpdateTx(ctx, tx, a) +} +func (s *Service) Recover(ctx context.Context) error { + ids, e := s.Repo.ExpiredJobs(ctx) + if e != nil { + return e + } + for _, id := range ids { + e = s.tm.RunInTransaction(ctx, nil, func(tx pgx.Tx) error { + j, e := s.Repo.JobTx(ctx, tx, id) + if e != nil { + return e + } + if j.State != "processing" || j.LeaseUntil == nil || j.LeaseUntil.After(time.Now()) { + return nil + } + a, e := s.Repo.GetTx(ctx, tx, j.Message.AttachmentID) + if e != nil { + return e + } + if a.PipelineVersion != j.Message.PipelineVersion { + return nil + } + return s.retry(ctx, tx, j, a, errors.New("lease expired")) + }) + if e != nil { + return e + } + } + dead, e := s.Repo.FailedOutbox(ctx) + if e != nil { + return e + } + for _, id := range dead { + if e = s.PublishFailed(ctx, id); e != nil { + return e + } + } + return nil +} +func (s *Service) PublishFailed(ctx context.Context, id uuid.UUID) error { + return s.tm.RunInTransaction(ctx, nil, func(tx pgx.Tx) error { + j, e := s.Repo.JobTx(ctx, tx, id) + if e != nil { + return e + } + if j.State != "queued" { + return nil + } + dead, e := s.Repo.PublicationDeadTx(ctx, tx, id, j.Message.Attempt) + if e != nil { + return e + } + if !dead { + return nil + } + a, e := s.Repo.GetTx(ctx, tx, j.Message.AttachmentID) + if e != nil { + return e + } + if a.PipelineVersion != j.Message.PipelineVersion { + return nil + } + code := "publish_failed" + j.State = "failed" + j.ErrorCode = &code + j.UpdatedAt = time.Now().UTC() + a.Public.Status = "failed" + a.Public.ErrorCode = &code + a.Public.UpdatedAt = j.UpdatedAt + if e = s.Repo.SaveJobTx(ctx, tx, j); e != nil { + return e + } + return s.Repo.UpdateTx(ctx, tx, a) + }) +} +func (s *Service) Replay(ctx context.Context, id uuid.UUID) error { + return s.tm.RunInTransaction(ctx, nil, func(tx pgx.Tx) error { + j, e := s.Repo.JobTx(ctx, tx, id) + if e != nil { + return e + } + a, e := s.Repo.GetTx(ctx, tx, j.Message.AttachmentID) + if e != nil { + return e + } + if j.State != "failed" || a.PipelineVersion != j.Message.PipelineVersion { + return apperr.New(apperr.CodeConflict, "only current failed job can be replayed") + } + source := a.StagingKey + if source == "" { + return errors.New("source unavailable; restore original first") + } + if _, e = s.Store.Head(ctx, source); e != nil { + return errors.New("source unavailable") + } + j.State = "queued" + j.Message.Attempt++ + j.AttemptLimit = j.Message.Attempt + s.Jobs.MaxAttempts - 1 + j.NextAttemptAt = time.Now().UTC() + j.UpdatedAt = j.NextAttemptAt + j.Token = uuid.New() + j.ErrorCode = nil + j.LeaseUntil = nil + a.Public.Status = "queued" + a.Public.ErrorCode = nil + a.Public.UpdatedAt = j.UpdatedAt + if e = s.Repo.SaveJobTx(ctx, tx, j); e != nil { + return e + } + if e = s.Repo.UpdateTx(ctx, tx, a); e != nil { + return e + } + if e = s.Repo.AuditTx(ctx, tx, id, "replay"); e != nil { + return e + } + return s.enqueue(ctx, tx, j) + }) +} +func (s *Service) Reprocess(ctx context.Context, id uuid.UUID, version int) error { + if version < 1 || version > 2 { + return errors.New("supported pipeline versions: 1, 2") + } + return s.tm.RunInTransaction(ctx, nil, func(tx pgx.Tx) error { + a, e := s.Repo.GetTx(ctx, tx, id) + if e != nil { + return e + } + if version <= a.PipelineVersion || a.Public.Status != "ready" { + return errors.New("requires ready attachment and newer version") + } + a.PipelineVersion = version + a.StagingKey = a.Key + a.Public.Status = "queued" + a.Public.Variants = []contracts.Variant{} + a.VariantKeys = map[string]string{} + a.Public.UpdatedAt = time.Now().UTC() + if e = s.Repo.UpdateTx(ctx, tx, a); e != nil { + return e + } + return s.newJob(ctx, tx, a) + }) +} +func (s *Service) Cleanup(ctx context.Context) error { + records, e := s.Repo.RetiredSources(ctx, time.Now().Add(-s.Jobs.Retention)) + if e != nil { + return e + } + for _, record := range records { + // Serialize deletion with operator Restore and the initial upload handoff. + _, unlock, lockErr := s.Repo.Lock(ctx, record.StagingKey) + if lockErr != nil { + return lockErr + } + e = s.tm.RunInTransaction(ctx, nil, func(tx pgx.Tx) error { + a, e := s.Repo.GetTx(ctx, tx, record.Public.Id) + if e != nil { + return e + } + if a.StagingKey == "" || a.StagingKey == a.Key && a.Public.Status == "ready" { + return nil + } + if a.Public.Status != "ready" && (a.Public.Status != "failed" || time.Since(a.Public.UpdatedAt) < s.Jobs.Retention) { + return nil + } + if !strings.HasPrefix(a.StagingKey, "results/") { + if e = s.Store.Delete(ctx, a.StagingKey); e != nil { + return e + } + } + a.StagingKey = "" + return s.Repo.UpdateTx(ctx, tx, a) + }) + unlock() + if e != nil { + return e + } + } + return nil +} +func managedPipelineKey(key string) bool { + return strings.HasPrefix(key, "staging/") || strings.HasPrefix(key, "results/") +} diff --git a/service/internal/service/attachment/reconcile.go b/service/internal/service/attachment/reconcile.go index 0a11258a..0ad54ad7 100644 --- a/service/internal/service/attachment/reconcile.go +++ b/service/internal/service/attachment/reconcile.go @@ -10,8 +10,9 @@ import ( "time" "github.com/aws/aws-sdk-go-v2/aws" - "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/google/uuid" + "github.com/jackc/pgx/v5" "gitea.privcloud.tech/mtusi2026/internal/pkg/objectstore" ) @@ -38,17 +39,17 @@ func (s *Service) Reconcile(ctx context.Context, apply bool) (result ScanResult, result.Apply = apply result.GraceSeconds = s.Config.Grace.Seconds() defer func() { result.DurationMS = time.Since(start).Milliseconds() }() - if err = s.Repo.Pool.Ping(ctx); err != nil { + if err = s.Repo.Ping(ctx); err != nil { return } result.S3Requests++ if err = s.Store.Ping(ctx); err != nil { return } - pager := s3.NewListObjectsV2Paginator(s.Store.Client, &s3.ListObjectsV2Input{Bucket: &s.Store.Bucket, Prefix: aws.String(Prefix), MaxKeys: aws.Int32(100)}) - for pager.HasMorePages() { + token := "" + for { result.S3Requests++ - page, e := pager.NextPage(ctx) + page, e := s.Store.List(ctx, token) if e != nil { return result, e } @@ -92,7 +93,7 @@ func (s *Service) Reconcile(ctx context.Context, apply bool) (result ScanResult, return nil } result.DBChecks++ - if _, e = conn.Exec(ctx, "SELECT 1"); e != nil { + if e = s.Repo.CheckConn(ctx, conn); e != nil { return e } result.S3Requests++ @@ -106,7 +107,12 @@ func (s *Service) Reconcile(ctx context.Context, apply bool) (result ScanResult, return result, e } } + if !aws.ToBool(page.IsTruncated) { + break + } + token = aws.ToString(page.NextContinuationToken) } + after := uuid.Nil for { result.DBChecks++ @@ -120,6 +126,9 @@ func (s *Service) Reconcile(ctx context.Context, apply bool) (result ScanResult, for _, a := range page { result.Metadata++ after = a.Public.Id + if a.Public.Status != "ready" { + continue + } result.LockAttempts++ conn, unlock, e := s.Repo.Lock(ctx, a.Key) if e != nil { @@ -201,5 +210,19 @@ func (s *Service) Restore(ctx context.Context, id uuid.UUID, path string) error return fmt.Errorf("stored original verification failed") } _, e = s.Repo.MarkMissing(ctx, conn, id, false) - return e + if e != nil { + return e + } + return s.tm.RunInTransaction(ctx, nil, func(tx pgx.Tx) error { + current, e := s.Repo.GetTx(ctx, tx, id) + if e != nil { + return e + } + if current.Public.Status == "failed" { + current.StagingKey = current.Key + current.Public.UpdatedAt = time.Now().UTC() + return s.Repo.UpdateTx(ctx, tx, current) + } + return nil + }) } diff --git a/service/internal/service/attachment/service.go b/service/internal/service/attachment/service.go index 36c5d713..686bc13b 100644 --- a/service/internal/service/attachment/service.go +++ b/service/internal/service/attachment/service.go @@ -8,8 +8,7 @@ import ( _ "image/jpeg" _ "image/png" "io" - "mime" - "net/http" + "os" "strings" "time" @@ -18,28 +17,31 @@ import ( "github.com/google/uuid" - "gitea.privcloud.tech/mtusi2026/internal/adapter/repository/storage" "gitea.privcloud.tech/mtusi2026/internal/config" "gitea.privcloud.tech/mtusi2026/internal/dto" "gitea.privcloud.tech/mtusi2026/internal/gen/contracts" "gitea.privcloud.tech/mtusi2026/internal/pkg/apperr" - "gitea.privcloud.tech/mtusi2026/internal/pkg/objectstore" + "github.com/jackc/pgx/v5" ) const Prefix = "originals/" +const StagingPrefix = "staging/" type Membership interface { GetForMember(context.Context, dto.ChatMember) (*dto.Chat, error) } type Service struct { - Repo *storage.AttachmentsRepo - Store *objectstore.Store + Repo Repository + Store ObjectStore Config *config.Attachment chats Membership + tm TransactionManager + outbox Outbox + Jobs *config.Pipeline uploads, decoders chan struct{} } -func New(repo *storage.AttachmentsRepo, store *objectstore.Store, c *config.Attachment, chats Membership) *Service { +func New(repo Repository, store ObjectStore, c *config.Attachment, chats Membership) *Service { return &Service{Repo: repo, Store: store, Config: c, chats: chats, uploads: make(chan struct{}, c.Uploads), decoders: make(chan struct{}, c.Decoders)} } func ValidFilename(s string) bool { @@ -53,21 +55,15 @@ func ValidFilename(s string) bool { } return true } -func (s *Service) Upload(w http.ResponseWriter, r *http.Request, user, chat uuid.UUID, filename string) (*contracts.Attachment, error) { - ctx := r.Context() - if _, e := s.chats.GetForMember(ctx, dto.ChatMember{UserID: user, ChatID: chat}); e != nil { +func (s *Service) Configure(tm TransactionManager, outbox Outbox, jobs *config.Pipeline) { + s.tm = tm + s.outbox = outbox + s.Jobs = jobs +} +func (s *Service) Upload(ctx context.Context, req dto.UploadAttachment) (*contracts.Attachment, error) { + if _, e := s.chats.GetForMember(ctx, dto.ChatMember{UserID: req.UserID, ChatID: req.ChatID}); e != nil { return nil, e } - if !ValidFilename(filename) { - return nil, apperr.New(apperr.CodeBadRequest, "invalid filename") - } - mt, _, e := mime.ParseMediaType(r.Header.Get("Content-Type")) - if e != nil || (mt != "image/png" && mt != "image/jpeg") { - return nil, apperr.New(apperr.CodeUnsupportedMediaType, "expected PNG or JPEG") - } - if r.ContentLength > s.Config.MaxBytes { - return nil, apperr.New(apperr.CodePayloadTooLarge, "upload too large") - } select { case s.uploads <- struct{}{}: defer func() { <-s.uploads }() @@ -80,23 +76,22 @@ func (s *Service) Upload(w http.ResponseWriter, r *http.Request, user, chat uuid } defer os.Remove(f.Name()) defer f.Close() - r.Body = http.MaxBytesReader(w, r.Body, s.Config.MaxBytes) hash := sha256.New() - size, e := io.Copy(io.MultiWriter(f, hash), r.Body) + size, e := io.Copy(io.MultiWriter(f, hash), io.LimitReader(req.Body, s.Config.MaxBytes+1)) if e != nil { - if _, ok := e.(*http.MaxBytesError); ok { - return nil, apperr.New(apperr.CodePayloadTooLarge, "upload too large") - } return nil, apperr.New(apperr.CodeBadRequest, "incomplete upload") } - if e = s.Validate(ctx, f, mt); e != nil { + if size > s.Config.MaxBytes { + return nil, apperr.New(apperr.CodePayloadTooLarge, "upload too large") + } + if e = s.Validate(ctx, f, req.ContentType); e != nil { return nil, e } now := time.Now().UTC().Truncate(time.Microsecond) id := uuid.New() - key := Prefix + id.String() - a := &storage.AttachmentRecord{Key: key, Public: contracts.Attachment{Id: id, ChatId: chat, OwnerId: user, Filename: filename, ContentType: contracts.AttachmentContentType(mt), SizeBytes: int(size), Sha256: fmt.Sprintf("%x", hash.Sum(nil)), Status: "ready", Variants: []contracts.Variant{}, CreatedAt: now, UpdatedAt: now}} - conn, unlock, e := s.Repo.Lock(ctx, key) + key := StagingPrefix + id.String() + a := &dto.AttachmentRecord{Key: key, StagingKey: key, PipelineVersion: 1, VariantKeys: map[string]string{}, Public: contracts.Attachment{Id: id, ChatId: req.ChatID, OwnerId: req.UserID, Filename: req.Filename, ContentType: contracts.AttachmentContentType(req.ContentType), SizeBytes: int(size), Sha256: fmt.Sprintf("%x", hash.Sum(nil)), Status: "queued", Variants: []contracts.Variant{}, CreatedAt: now, UpdatedAt: now}} + _, unlock, e := s.Repo.Lock(ctx, key) if e != nil { return nil, e } @@ -104,14 +99,19 @@ func (s *Service) Upload(w http.ResponseWriter, r *http.Request, user, chat uuid if _, e = f.Seek(0, 0); e != nil { return nil, e } - if e = s.Store.Put(ctx, key, mt, f, size); e != nil { + if e = s.Store.Put(ctx, key, req.ContentType, f, size); e != nil { return nil, Unavailable() } - // Never blindly compensate a failed/ambiguous COMMIT with DELETE: a committed - // row may already reference this key. Reconciliation decides after grace. - if e = s.Repo.Save(ctx, conn, a); e != nil { + s.Fault(ctx, "after_staging", id) + if e = s.tm.RunInTransaction(ctx, nil, func(tx pgx.Tx) error { + if e := s.Repo.InsertTx(ctx, tx, a); e != nil { + return e + } + return s.newJob(ctx, tx, a) + }); e != nil { return nil, e } + s.Fault(ctx, "after_upload_commit", id) return &a.Public, nil } func (s *Service) Validate(ctx context.Context, f *os.File, mt string) error { @@ -134,12 +134,12 @@ func (s *Service) Validate(ctx context.Context, f *os.File, mt string) error { if _, e = f.Seek(0, 0); e != nil { return e } - if _, _, e = image.Decode(f); e != nil { + if e = validateImage(ctx, f.Name()); e != nil { return apperr.New(apperr.CodeUnsupportedMediaType, "corrupt image") } return ctx.Err() } -func (s *Service) Get(ctx context.Context, user, id uuid.UUID) (*storage.AttachmentRecord, error) { +func (s *Service) Get(ctx context.Context, user, id uuid.UUID) (*dto.AttachmentRecord, error) { a, e := s.Repo.Get(ctx, id) if e != nil { return nil, e @@ -154,6 +154,9 @@ func (s *Service) Get(ctx context.Context, user, id uuid.UUID) (*storage.Attachm } func Unavailable() error { return apperr.New(apperr.CodeUnavailable, "attachment storage unavailable") } func ManagedKey(key string) bool { + if managedPipelineKey(key) { + return true + } if !strings.HasPrefix(key, Prefix) { return false } diff --git a/service/internal/service/chat/idempotency.go b/service/internal/service/chat/idempotency.go new file mode 100644 index 00000000..13fa9620 --- /dev/null +++ b/service/internal/service/chat/idempotency.go @@ -0,0 +1,54 @@ +package chat + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/json" + "gitea.privcloud.tech/mtusi2026/internal/dto" + "gitea.privcloud.tech/mtusi2026/internal/pkg/apperr" + "github.com/google/uuid" + "github.com/jackc/pgx/v5" + "time" +) + +// CreateIdempotentTx performs the message use case inside an existing transaction. +func CreateIdempotentTx(ctx context.Context, tx pgx.Tx, repo messagesRepo, req dto.CreateMessage) (*dto.Message, error) { + scope := req.SenderID.String() + ":POST:" + req.ChatID.String() + ":" + req.IdempotencyKey + if err := repo.LockKeyTx(ctx, tx, scope); err != nil { + return nil, err + } + canonical := struct { + Text string `json:"text"` + AttachmentIDs []uuid.UUID `json:"attachment_ids"` + }{req.Text, append([]uuid.UUID{}, req.AttachmentIDs...)} + encoded, err := json.Marshal(canonical) + if err != nil { + return nil, err + } + hash := sha256.Sum256(encoded) + replay, err := repo.ReplayTx(ctx, tx, req) + if err != nil { + return nil, err + } + if replay != nil { + if replay.HashVersion == 1 && len(req.AttachmentIDs) == 0 { + hash = sha256.Sum256([]byte(req.Text)) + } else if replay.HashVersion != 2 { + return nil, apperr.New(apperr.CodeConflict, "idempotency key used with different content") + } + if !bytes.Equal(hash[:], replay.Hash) { + return nil, apperr.New(apperr.CodeConflict, "idempotency key used with different content") + } + return repo.MessageTx(ctx, tx, replay.MessageID) + } + now := time.Now().UTC().Truncate(time.Microsecond) + message := dto.Message{ID: uuid.New(), ChatID: req.ChatID, SenderID: req.SenderID, Text: req.Text, AttachmentIDs: append([]uuid.UUID{}, req.AttachmentIDs...), CreatedAt: now} + if err = repo.InsertMessageTx(ctx, tx, message); err != nil { + return nil, err + } + if err = repo.SaveKeyTx(ctx, tx, req, hash[:], message.ID, now.Add(24*time.Hour)); err != nil { + return nil, err + } + return &message, nil +} diff --git a/service/internal/service/chat/interface.go b/service/internal/service/chat/interface.go index dbe71578..17c7742a 100644 --- a/service/internal/service/chat/interface.go +++ b/service/internal/service/chat/interface.go @@ -2,6 +2,7 @@ package chat import ( "context" + "time" "github.com/google/uuid" "github.com/jackc/pgx/v5" @@ -17,7 +18,11 @@ type chatsRepo interface { type messagesRepo interface { AttachmentsForMessageTx(ctx context.Context, tx pgx.Tx, ids []uuid.UUID) ([]dto.MessageAttachment, error) - CreateIdempotentTx(ctx context.Context, tx pgx.Tx, req dto.CreateMessage) (*dto.Message, error) + LockKeyTx(context.Context, pgx.Tx, string) error + ReplayTx(context.Context, pgx.Tx, dto.CreateMessage) (*dto.MessageReplay, error) + MessageTx(context.Context, pgx.Tx, uuid.UUID) (*dto.Message, error) + InsertMessageTx(context.Context, pgx.Tx, dto.Message) error + SaveKeyTx(context.Context, pgx.Tx, dto.CreateMessage, []byte, uuid.UUID, time.Time) error GetByIDForChat(ctx context.Context, chatID, messageID uuid.UUID) (*dto.Message, error) ListForChat(ctx context.Context, chatID uuid.UUID, page dto.PageQuery) ([]dto.Message, error) } diff --git a/service/internal/service/chat/svc.go b/service/internal/service/chat/svc.go index 6dacf961..700f1683 100644 --- a/service/internal/service/chat/svc.go +++ b/service/internal/service/chat/svc.go @@ -190,6 +190,9 @@ func (c *Chat) CreateMessage(ctx context.Context, req dto.CreateMessage) (*dto.M if attachment.ChatID != req.ChatID { return apperr.New(apperr.CodeNotFound, "attachment not found") } + if attachment.Status == "failed" { + return apperr.New(apperr.CodeConflict, "failed attachment") + } if attachment.Missing { return apperr.New(apperr.CodeUnavailable, "attachment storage unavailable") } @@ -197,7 +200,7 @@ func (c *Chat) CreateMessage(ctx context.Context, req dto.CreateMessage) (*dto.M } var err error - message, err = c.messagesRepo.CreateIdempotentTx(ctx, tx, req) + message, err = CreateIdempotentTx(ctx, tx, c.messagesRepo, req) return err }) return message, err diff --git a/service/internal/service/outbox/api.go b/service/internal/service/outbox/api.go new file mode 100644 index 00000000..aaac1a7d --- /dev/null +++ b/service/internal/service/outbox/api.go @@ -0,0 +1,12 @@ +package outbox + +import ( + "context" + + "github.com/jackc/pgx/v5" +) + +type API interface { + CreateTx(context.Context, pgx.Tx, CreateMessageReq) (CreateMessageRes, error) + Create(context.Context, CreateMessageReq) (CreateMessageRes, error) +} diff --git a/service/internal/service/outbox/models.go b/service/internal/service/outbox/models.go new file mode 100644 index 00000000..fc90e0ea --- /dev/null +++ b/service/internal/service/outbox/models.go @@ -0,0 +1,115 @@ +package outbox + +import ( + "time" + + "github.com/google/uuid" +) + +type CreateMessageReq struct { + ID uuid.UUID + CreatedAt time.Time + Transport Transport + AggregateType string + AggregateID uuid.UUID + MessageType string + IdempotencyKey *string + SchemaVersion int + CausationID *uuid.UUID + Metadata map[string]string + Payload []byte + MaxAttempts int + Status Status +} + +type CreateMessageRes struct { + ID uuid.UUID + CreatedAt time.Time +} + +type MarkPublishedMessageReq struct { + ID uuid.UUID + UpdatedAt time.Time + PublishedAt time.Time +} + +type MarkFailedMessageReq struct { + ID uuid.UUID + UpdatedAt time.Time + NextAttemptAt time.Time + LastError string +} + +type MarkDeadLetterMessageReq struct { + ID uuid.UUID + UpdatedAt time.Time + DeadLetterAt time.Time + LastError string +} + +type AcquireMessagesForProcessingReq struct { + Limit uint + LockDuration time.Duration +} + +type AcquiredMessage struct { + Token uuid.UUID + ID uuid.UUID + Transport Transport + AggregateType string + AggregateID uuid.UUID + MessageType string + SchemaVersion int + CorrelationID uuid.UUID + CausationID *uuid.UUID + Status Status + Metadata map[string]string + Payload []byte + Attempts int + MaxAttempts int + LastError *string + LockedAt *time.Time + LockedUntil *time.Time + NextAttemptAt *time.Time + PublishedAt *time.Time + DeadLetterAt *time.Time + CreatedAt time.Time + UpdatedAt *time.Time +} + +type RecoverStuckMessagesAsFailedReq struct { + UpdatedAt time.Time + NextAttemptAt time.Time + LastError string + Limit uint +} + +type RecoverStuckMessagesAsDeadLetterReq struct { + UpdatedAt time.Time + DeadLetterAt time.Time + LastError string + Limit uint +} + +type DeliveryMessage struct { + ID uuid.UUID + Transport Transport + AggregateType string + AggregateID uuid.UUID + MessageType string + SchemaVersion int + CorrelationID uuid.UUID + CausationID *uuid.UUID + Payload []byte + Metadata map[string]string +} + +type ProcessOnceResult struct { + RecoveredAsFailed int64 + RecoveredAsDeadLetter int64 + + Acquired int + Published int + Failed int + DeadLettered int +} diff --git a/service/internal/service/outbox/status.go b/service/internal/service/outbox/status.go new file mode 100644 index 00000000..c7a1d3a3 --- /dev/null +++ b/service/internal/service/outbox/status.go @@ -0,0 +1,16 @@ +package outbox + +const ( + StatusCreated Status = "created" + StatusPending Status = "pending" + StatusProcessing Status = "processing" + StatusPublished Status = "published" + StatusFailed Status = "failed" + StatusDeadLetter Status = "dead_letter" +) + +type Status string + +func (s Status) String() string { + return string(s) +} diff --git a/service/internal/service/outbox/svc.go b/service/internal/service/outbox/svc.go new file mode 100644 index 00000000..7dd7faaa --- /dev/null +++ b/service/internal/service/outbox/svc.go @@ -0,0 +1,114 @@ +// Adapted from mamont-optics/application/internal/service/outbox. +// Retains transactional creation, batch acquisition, bounded retries and lease recovery. +package outbox + +import ( + "context" + "encoding/json" + "errors" + "github.com/google/uuid" + "github.com/jackc/pgx/v5" + "math/rand/v2" + "time" +) + +type Repository interface { + CreateTx(context.Context, pgx.Tx, CreateMessageReq) (CreateMessageRes, error) + AcquireManyForProcessing(context.Context, AcquireMessagesForProcessingReq) ([]AcquiredMessage, error) + Expired(context.Context) ([]AcquiredMessage, error) + Update(context.Context, AcquiredMessage, Status, time.Time, string) error +} +type TransportPublisher interface { + Publish(context.Context, DeliveryMessage) error +} +type TransactionManager interface { + RunInTransaction(context.Context, *pgx.TxOptions, func(pgx.Tx) error) error +} +type Svc struct { + repo Repository + tm TransactionManager + publisher TransportPublisher + OnDead func(context.Context, uuid.UUID) error +} + +func New(repo Repository, tm TransactionManager, publisher TransportPublisher) *Svc { + return &Svc{repo: repo, tm: tm, publisher: publisher} +} +func (s *Svc) CreateTx(ctx context.Context, tx pgx.Tx, r CreateMessageReq) (CreateMessageRes, error) { + if r.ID == uuid.Nil { + r.ID = uuid.New() + } + if r.CreatedAt.IsZero() { + r.CreatedAt = time.Now().UTC() + } + if r.Status == "" { + r.Status = StatusPending + } + if r.MaxAttempts == 0 { + r.MaxAttempts = 5 + } + return s.repo.CreateTx(ctx, tx, r) +} +func (s *Svc) Create(ctx context.Context, r CreateMessageReq) (res CreateMessageRes, err error) { + err = s.tm.RunInTransaction(ctx, nil, func(tx pgx.Tx) error { var e error; res, e = s.CreateTx(ctx, tx, r); return e }) + return +} +func Backoff(attempt int) time.Duration { + n := min(max(attempt-1, 0), 5) + cap := min(time.Second*time.Duration(1<= m.MaxAttempts { + status = StatusDeadLetter + } + if err := s.repo.Update(ctx, m, status, time.Now().UTC().Add(Backoff(m.Attempts)), code); err != nil { + return err + } + if status == StatusDeadLetter && s.OnDead != nil { + var j struct { + JobID uuid.UUID `json:"job_id"` + } + if err := json.Unmarshal(m.Payload, &j); err != nil { + return err + } + return s.OnDead(ctx, j.JobID) + } + return nil +} diff --git a/service/internal/service/outbox/transport.go b/service/internal/service/outbox/transport.go new file mode 100644 index 00000000..a92608cc --- /dev/null +++ b/service/internal/service/outbox/transport.go @@ -0,0 +1,7 @@ +package outbox + +type Transport string + +const TransportAMQP Transport = "amqp" + +func (t Transport) String() string { return string(t) } diff --git a/service/migrations/008_async_pipeline.sql b/service/migrations/008_async_pipeline.sql new file mode 100644 index 00000000..4607e201 --- /dev/null +++ b/service/migrations/008_async_pipeline.sql @@ -0,0 +1,41 @@ +ALTER TABLE attachments + ADD COLUMN status text NOT NULL DEFAULT 'queued' CHECK(status IN ('queued','processing','ready','failed')), + ADD COLUMN error_code text, + ADD COLUMN pipeline_version integer NOT NULL DEFAULT 1, + ADD COLUMN staging_key text NOT NULL DEFAULT '', + ADD COLUMN variants jsonb NOT NULL DEFAULT '[]', + ADD COLUMN variant_keys jsonb NOT NULL DEFAULT '{}'; +UPDATE attachments SET staging_key=object_key; +CREATE TABLE attachment_jobs ( + id uuid PRIMARY KEY, attachment_id uuid NOT NULL REFERENCES attachments(id), + pipeline_version integer NOT NULL, attempt integer NOT NULL CHECK(attempt>0), + attempt_limit integer NOT NULL, state text NOT NULL, + token uuid NOT NULL, lease_until timestamptz, next_attempt_at timestamptz NOT NULL, + error_code text, created_at timestamptz NOT NULL, updated_at timestamptz NOT NULL, + UNIQUE(attachment_id,pipeline_version) +); +CREATE INDEX jobs_recovery ON attachment_jobs(state,lease_until); +CREATE TABLE outbox ( + id uuid PRIMARY KEY, job_id uuid NOT NULL REFERENCES attachment_jobs(id), + payload jsonb NOT NULL, status text NOT NULL, + attempts integer NOT NULL DEFAULT 0, max_attempts integer NOT NULL, + token uuid NOT NULL, locked_until timestamptz, next_attempt_at timestamptz NOT NULL, + last_error text, created_at timestamptz NOT NULL, updated_at timestamptz NOT NULL +); +CREATE INDEX outbox_delivery ON outbox(status,next_attempt_at); +CREATE TABLE job_audit ( + id bigserial PRIMARY KEY, job_id uuid NOT NULL REFERENCES attachment_jobs(id), + action text NOT NULL, created_at timestamptz NOT NULL DEFAULT clock_timestamp() +); +-- Existing lab4 originals become durable sources for the first asynchronous run. +INSERT INTO attachment_jobs + SELECT gen_random_uuid(),id,1,1,5,'queued',gen_random_uuid(),NULL,clock_timestamp(),NULL,created_at,clock_timestamp() FROM attachments; +INSERT INTO outbox + SELECT gen_random_uuid(),id,jsonb_build_object('schema_version',1,'job_id',id,'type','attachment.process.v1','attachment_id',attachment_id,'pipeline_version',1,'attempt',1,'created_at',created_at), + 'pending',0,5,gen_random_uuid(),NULL,clock_timestamp(),NULL,clock_timestamp(),clock_timestamp() FROM attachment_jobs; + +---- create above / drop below ---- +DROP TABLE job_audit; +DROP TABLE outbox; +DROP TABLE attachment_jobs; +ALTER TABLE attachments DROP COLUMN status,DROP COLUMN error_code,DROP COLUMN pipeline_version,DROP COLUMN staging_key,DROP COLUMN variants,DROP COLUMN variant_keys; diff --git a/service/migrations/009_variant_manifest.sql b/service/migrations/009_variant_manifest.sql new file mode 100644 index 00000000..84413c85 --- /dev/null +++ b/service/migrations/009_variant_manifest.sql @@ -0,0 +1,8 @@ +CREATE TABLE attachment_variants ( + attachment_id uuid NOT NULL REFERENCES attachments(id), pipeline_version integer NOT NULL, + variant text NOT NULL CHECK(variant IN ('original','thumbnail','crop','optimized')), + object_key text NOT NULL UNIQUE, sha256 text NOT NULL, size_bytes bigint NOT NULL, + PRIMARY KEY(attachment_id,pipeline_version,variant) +); +---- create above / drop below ---- +DROP TABLE attachment_variants; diff --git a/service/tests/attachments_test.go b/service/tests/attachments_test.go index e9a5860f..1a3e6bf3 100644 --- a/service/tests/attachments_test.go +++ b/service/tests/attachments_test.go @@ -69,11 +69,12 @@ func TestAttachmentContractAndErrors(t *testing.T) { Sha string `json:"sha256"` Size int `json:"size_bytes"` } - decodeOK(t, api.upload(t, chat, alice, "Файл.png", "image/png", data, false), 201, &att) + decodeOK(t, api.upload(t, chat, alice, "Файл.png", "image/png", data, false), 202, &att) hash := sha256.Sum256(data) if att.Sha != hex.EncodeToString(hash[:]) || att.Size != len(data) { t.Fatal("original metadata mismatch") } + waitAttachmentReady(t, api, alice, att.ID) content := "/api/v1/attachments/" + att.ID + "/content" got := api.request(t, "GET", content, bob, "") if got.status != 200 || !bytes.Equal(got.body, data) { @@ -115,7 +116,7 @@ func TestAttachmentContractAndErrors(t *testing.T) { boundary = binary.BigEndian.AppendUint32(boundary, crc32.ChecksumIEEE(chunk)) boundary = append(boundary, data[len(data)-12:]...) var boundaryAtt map[string]any - decodeOK(t, api.upload(t, chat, alice, "boundary.png", "image/png", boundary, false), 201, &boundaryAtt) + decodeOK(t, api.upload(t, chat, alice, "boundary.png", "image/png", boundary, false), 202, &boundaryAtt) oversized := make([]byte, 10485761) copy(oversized, data) @@ -127,7 +128,7 @@ func TestAttachmentContractAndErrors(t *testing.T) { t.Fatal(e) } var jpegAtt map[string]any - decodeOK(t, api.upload(t, chat, bob, "file.jpg", "image/jpeg", jpg.Bytes(), false), 201, &jpegAtt) + decodeOK(t, api.upload(t, chat, bob, "file.jpg", "image/jpeg", jpg.Bytes(), false), 202, &jpegAtt) imageOnly := `{"text":"","attachment_ids":["` + att.ID + `"]}` var message map[string]any key := uuid.NewString() @@ -145,7 +146,9 @@ func TestAttachmentContractAndErrors(t *testing.T) { for _, body := range []string{`{"text":""}`, `{"text":" "}`, `{"attachment_ids":["` + att.ID + `"]}`, `{"text":"ok","attachment_ids":null}`, `{"text":"ok","attachment_ids":"bad"}`, `{"text":"ok","attachment_ids":["` + att.ID + `","` + att.ID + `"]}`} { assertError(t, api.request(t, "POST", "/api/v1/chats/"+chat+"/messages", alice, body), 400, "invalid_request") } - assertError(t, api.request(t, "GET", content+"?variant=thumbnail", bob, ""), 400, "invalid_request") + if got := api.request(t, "GET", content+"?variant=thumbnail", bob, ""); got.status != 200 { + t.Fatal("thumbnail unavailable") + } } func TestConcurrentAttachmentMessage(t *testing.T) { api := newAPI(t) @@ -154,7 +157,7 @@ func TestConcurrentAttachmentMessage(t *testing.T) { var att struct { ID string `json:"id"` } - decodeOK(t, api.upload(t, chat, user, "sample.png", "image/png", fixturePNG(t), false), 201, &att) + decodeOK(t, api.upload(t, chat, user, "sample.png", "image/png", fixturePNG(t), false), 202, &att) key := uuid.NewString() body := `{"text":"","attachment_ids":["` + att.ID + `"]}` type result struct { @@ -202,3 +205,22 @@ func TestConcurrentAttachmentMessage(t *testing.T) { t.Fatal("duplicate history") } } + +func waitAttachmentReady(t *testing.T, api *apiClient, user, id string) { + t.Helper() + deadline := time.Now().Add(60 * time.Second) + for time.Now().Before(deadline) { + var att struct { + Status string `json:"status"` + } + decodeOK(t, api.request(t, "GET", "/api/v1/attachments/"+id, user, ""), 200, &att) + if att.Status == "ready" { + return + } + if att.Status == "failed" { + t.Fatal("attachment failed") + } + time.Sleep(100 * time.Millisecond) + } + t.Fatal("attachment readiness deadline") +} diff --git a/student-tests/README.md b/student-tests/README.md new file mode 100644 index 00000000..6e922fbb --- /dev/null +++ b/student-tests/README.md @@ -0,0 +1,14 @@ +# Проверки решения lab5 + +Публичные `tests/` не изменены. Собственные проверки используют их HTTP-клиент и создают только синтетические данные в локальном стенде. + +- `make test-images`: реальные форматы и размеры вариантов, floor/no-upscale, EXIF orientation, белый фон alpha, центральный crop, hashes, ACL и отрицательные входы. +- `make test-staging`: защита queued-source, удаление staging после ready, terminal source_missing, 24h retention и восстановление с явным replay. Для retention меняется только возраст одной синтетической записи на 25h; реальный день не ожидается и конфигурация retention не сокращается. +- `make test-broker-persistence`: заменяет контейнер брокера при остановленных worker и dispatcher; проверяет, что уже опубликованное сообщение осталось в очереди с тем же volume. +- `make test-pipeline`: 13 сценариев × 3 повтора, включая crash, дубли, lease recovery, reorder, устаревший worker и версию pipeline. На время выполнения подключает `compose.faults.yaml`, перезапускает процессы и stores с сохранением volumes. В конце восстанавливает обычный Compose без fault injection. + +Перед запуском нужен `make up`. Проверки, которые останавливают/перезапускают процессы (`test-staging`, `test-pipeline`, persistence/failover), запускаются последовательно. Результаты — `evidence/*.json`; секреты и токены туда не записываются. Директивы fault injection лежат в игнорируемой `.lab5-faults/`. + +`jobs redeliver --id JOB_UUID` — только инструмент экспериментального профиля с `LAB5_FAULTS=1`, публикующий дополнительную копию сохранённой job. Обычный оператор использует `jobs replay` для failed или `jobs reprocess` для новой версии. + +Исследование и ограничения описаны в [REPORT.md](../REPORT.md). diff --git a/student-tests/broker_persistence.py b/student-tests/broker_persistence.py new file mode 100644 index 00000000..6d63bc98 --- /dev/null +++ b/student-tests/broker_persistence.py @@ -0,0 +1,47 @@ +#!/usr/bin/env python3 +"""Prove a confirmed queued message survives broker container replacement, not only restart.""" +from pathlib import Path +import json +import subprocess +import sys +import time +ROOT=Path(__file__).resolve().parents[1] +sys.path.insert(0,str(ROOT/'tests')) +from client import Client,check + +def compose(*args): + r=subprocess.run([str(ROOT/'scripts/compose.sh'),*args],stdout=subprocess.PIPE,stderr=subprocess.PIPE,text=True,timeout=150) + check(r.returncode==0,'broker persistence infrastructure command failed: '+args[0]);return r.stdout.strip() + +def queued(): + r=subprocess.run([str(ROOT/'scripts/compose.sh'),'exec','-T','broker','rabbitmqctl','-q','list_queues','name','messages'],stdout=subprocess.PIPE,stderr=subprocess.PIPE,text=True,timeout=30) + if r.returncode:return 0 + out=r.stdout + for line in out.splitlines(): + fields=line.split() + if len(fields)==2 and fields[0]=='attachments.v1':return int(fields[1]) + return 0 + +def main(): + c=Client('http://localhost:18085');alice,bob,eve,chat=c.bootstrap() + try: + compose('stop','worker');att,data=c.attachment(alice,chat) + deadline=time.monotonic()+20 + while queued()<1 and time.monotonic()=1,'message not durably queued before replacement') + published=compose('exec','-T','postgres','psql','-U','messenger','-d','messenger','-At','-c',"SELECT bool_and(status='published') FROM outbox WHERE job_id=(SELECT id FROM attachment_jobs WHERE attachment_id='%s')"%att['id']) + check(published=='t','outbox not marked published before replacement') + compose('stop','dispatcher') + old=compose('ps','-q','broker') + compose('up','-d','--no-deps','--force-recreate','--wait','--wait-timeout','120','broker') + new=compose('ps','-q','broker');check(new!=old,'broker container was not replaced') + deadline=time.monotonic()+30 + while queued()<1 and time.monotonic()=1,'confirmed queue message lost after replacement') + compose('start','worker');c.wait_ready(bob,att) + check(c.call('GET','/api/v1/attachments/'+att['id']+'/content',actor=bob)['bytes']==data,'original changed after broker replacement') + result={'result':'PASS','attachment_id':att['id'],'broker_hostname':'broker','old_container_id':old,'new_container_id':new,'message_present_before_and_after':True,'outbox_published_before_replacement':True,'dispatcher_stopped_during_replacement':True} + (ROOT/'evidence/broker-persistence.json').write_text(json.dumps(result,indent=2)+'\n') + print('PASS: confirmed message survived broker container replacement with the same volume and stable node identity') + finally:compose('start','worker','dispatcher') +if __name__=='__main__':main() diff --git a/student-tests/images.py b/student-tests/images.py new file mode 100644 index 00000000..4b32463d --- /dev/null +++ b/student-tests/images.py @@ -0,0 +1,53 @@ +#!/usr/bin/env python3 +"""Image contract checks using synthetic fixtures and real decoding in ImageMagick.""" +import hashlib +import json +from pathlib import Path +import struct +import subprocess +import sys +import urllib.parse +import zlib +ROOT=Path(__file__).resolve().parents[1] +sys.path.insert(0,str(ROOT/'tests')) +from client import Client,check +from image_probe import jpeg_size + +def magick(args,data=None): + r=subprocess.run([str(ROOT/'scripts/compose.sh'),'run','--rm','--no-deps','-T','--entrypoint','magick','operator']+args,input=data,stdout=subprocess.PIPE,stderr=subprocess.PIPE,timeout=30) + check(r.returncode==0,'fixture decoder failed') + return r.stdout + +def png(w,h,rgba): + def chunk(t,b):return struct.pack('>I',len(b))+t+b+struct.pack('>I',zlib.crc32(t+b)) + return b'\x89PNG\r\n\x1a\n'+chunk(b'IHDR',struct.pack('>IIBBBBB',w,h,8,6,0,0,0))+chunk(b'IDAT',zlib.compress(b''.join(b'\0'+b''.join(bytes(rgba(x,y)) for x in range(w)) for y in range(h)) if callable(rgba) else (b'\0'+bytes(rgba)*w)*h))+chunk(b'IEND',b'') + +def main(): + c=Client('http://localhost:18085');alice,bob,eve,chat=c.bootstrap();results=[] + jpeg=magick(['png:-','jpeg:-'],png(300,150,(255,0,0,255))) + # EXIF TIFF orientation 6: display rotates the stored 300x150 pixels clockwise. + tiff=b'II'+struct.pack('H',len(exif)+2)+exif+jpeg[2:] + for name,data,mime,dimensions in [('small',png(16,8,(255,0,0,255)),'image/png',(16,8)),('floor',png(333,100,(255,0,0,255)),'image/png',(256,76)),('alpha',png(40,20,(0,0,0,0)),'image/png',(40,20)),('orientation',oriented,'image/jpeg',(128,256)),('thin',png(1,300,(0,0,0,255)),'image/png',(1,256)),('center_crop',png(300,100,lambda x,y:(255,0,0,255) if 100<=x<200 else (0,0,255,255)),'image/png',(256,85))]: + att=c.call('POST','/api/v1/chats/'+chat['id']+'/attachments?filename='+name,202,actor=alice,raw=data,headers={'Content-Type':mime},schema='Attachment')['json'];ready=c.wait_ready(bob,att) + original=c.call('GET','/api/v1/attachments/'+att['id']+'/content',actor=bob)['bytes'];check(original==data,'original changed') + for v in ready['variants']: + b=c.call('GET','/api/v1/attachments/'+att['id']+'/content?variant='+v['name'],actor=bob)['bytes'];check(hashlib.sha256(b).hexdigest()==v['sha256'],'checksum mismatch') + actual=tuple(map(int,magick(['identify','-format','%w %h','-'],b).split())) + check(actual==(v['width'],v['height']),'actual dimensions differ') + if v['name']=='thumbnail': + check(jpeg_size(b)==dimensions,'thumbnail geometry differs') + if name=='alpha':check(magick(['identify','-format','%[pixel:p{0,0}]','jpeg:-'],b).strip() in (b'gray(255)',b'srgb(255,255,255)'),'alpha was not composited on white') + if v['name']=='crop': + check(actual==(128,128),'crop geometry differs') + if name=='center_crop': + import re + color=tuple(map(int,re.findall(rb'\d+',magick(['identify','-format','%[pixel:p{0,0}]','jpeg:-'],b)))) + check(len(color)==3 and color[0]>=250 and color[1]<=5 and color[2]<=5,'crop is not centered') + check(b'Exif\0\0' not in b,'EXIF retained in derivative') + c.call('GET','/api/v1/attachments/'+att['id']+'/content',404,actor=eve) + results.append({'case':name,'result':'PASS','thumbnail':dimensions}) + for data,mime,status in [(b'not image','image/png',415),(png(2,2,(0,0,0,255)),'image/jpeg',415),(png(2,2,(0,0,0,255))[:40],'image/png',415),(b'x'*10485761,'image/png',413)]: + c.call('POST','/api/v1/chats/'+chat['id']+'/attachments?filename=bad',status,actor=alice,raw=data,headers={'Content-Type':mime}) + (ROOT/'evidence/images.json').write_text(json.dumps({'results':results,'negative_cases':4},indent=2)+'\n');print('PASS: image geometry, EXIF, alpha, real formats, checksums, ACL and negative inputs') +if __name__=='__main__':main() diff --git a/student-tests/pipeline.py b/student-tests/pipeline.py new file mode 100644 index 00000000..6e88ea61 --- /dev/null +++ b/student-tests/pipeline.py @@ -0,0 +1,179 @@ +#!/usr/bin/env python3 +"""Repeatable lab5 experiment on the local synthetic Compose stand. Never prints tokens.""" +import argparse +import concurrent.futures +import datetime +import hashlib +import json +import os +from pathlib import Path +import subprocess +import sys +import time +import uuid + +ROOT=Path(__file__).resolve().parents[1] +sys.path.insert(0,str(ROOT/'tests')) +from client import Client, Failure, check +from image_probe import jpeg_size +FAULTS=ROOT/'.lab5-faults' +COMPOSE=[str(ROOT/'scripts/compose.sh'),'-f',str(ROOT/'compose.yaml'),'-f',str(ROOT/'compose.faults.yaml')] +RESULTS=[] +SELECTED=set() + +def compose(*args, allow_failure=False): + p=subprocess.run(COMPOSE+list(args),cwd=ROOT,stdout=subprocess.PIPE,stderr=subprocess.PIPE,text=True,timeout=180) + if p.returncode and not allow_failure: + # Infrastructure logs may contain settings; report only action and exit status. + raise Failure('Compose '+args[0]+' failed, exit '+str(p.returncode)) + return p + +def sql(query): + return compose('exec','-T','postgres','psql','-U','messenger','-d','messenger','-At','-v','ON_ERROR_STOP=1','-c',query).stdout.strip() + +def uid(value): + return str(uuid.UUID(value)) + +def job(att,version=1): + return sql("SELECT id FROM attachment_jobs WHERE attachment_id='%s' AND pipeline_version=%d"%(uid(att['id']),version)) + +def operator(action,id=None,version=None, allow_failure=False): + args=['run','--rm','--no-deps','-T','operator','jobs',action] + if id: args+=['--id',uid(id)] + if version:args+=['--pipeline-version',str(version)] + return compose(*args,allow_failure=allow_failure) + +def wait(predicate,timeout=65): + deadline=time.monotonic()+timeout + while time.monotonic()=2,'lease not recovered') + operator('reprocess',att['id'],2);second=ready(c,bob,att,data) + check(first['variants']!=second['variants'],'pipeline versions are indistinguishable') + operator('redeliver',jid);marker.unlink();time.sleep(1) + check(metadata(c,bob,att)==second,'stale worker/job overwrote version 2') + check(sql("SELECT pipeline_version FROM attachments WHERE id='%s'"%uid(att['id']))=='2','version rolled back') + ready(c,bob,att,data) + return {'attachment_id':att['id'],'old_job_id':jid,'pipeline_version':2,'manifest_rows':8} + record('lease_reorder_and_stale_version',run,version_fence) + def retries(): + att,data,jid=upload_stopped();marker=directive('before_objects',jid,'temporary');compose('start','worker');failed=state(c,bob,att,'failed',65) + check(failed['error_code']=='retry_exhausted','wrong terminal code');check(sql("SELECT attempt FROM attachment_jobs WHERE id='%s'"%uid(jid))=='5','attempt limit not honored') + c.call('POST','/api/v1/chats/'+chat['id']+'/messages',409,actor=alice,body={'text':'','attachment_ids':[att['id']]},headers={'Idempotency-Key':uuid.uuid4().hex}) + marker.unlink();operator('replay',jid);ready(c,bob,att,data);check(operator('replay',jid,allow_failure=True).returncode!=0,'successful job replay allowed') + return {'job_id':jid,'failed_attempt':5,'replayed_attempt':6} + record('bounded_retry_and_explicit_replay',run,retries) + def permanent(): + att,data,jid=upload_stopped();marker=directive('before_objects',jid,'permanent');compose('start','worker');failed=state(c,bob,att,'failed');check(failed['error_code']=='invalid_image','wrong permanent code');check(sql("SELECT attempt FROM attachment_jobs WHERE id='%s'"%uid(jid))=='1','permanent error was retried');marker.unlink();operator('replay',jid);ready(c,bob,att,data);return {'job_id':jid} + record('permanent_error',run,permanent) + for point in ('after_staging','after_upload_commit'): + def api_crash(point=point): + if point=='after_upload_commit':compose('stop','dispatcher') + marker=directive(point,'all','crash') + try:c.attachment(alice,chat) + except Failure:pass + aid=hit(marker);att={'id':aid} + if point=='after_staging':check(sql("SELECT count(*) FROM attachments WHERE id='%s'"%uid(aid))=='0','precommit row exists') + else: + check(sql("SELECT status FROM outbox WHERE job_id=(SELECT id FROM attachment_jobs WHERE attachment_id='%s')"%uid(aid))=='pending','COMMIT/publish window not reached') + compose('start','dispatcher');ready(c,bob,att,(ROOT/'tests/fixtures/sample.png').read_bytes()) + wait(lambda:health(c));return {'attachment_id':aid,'accepted_commit':point=='after_upload_commit'} + record('api_crash_'+point,run,api_crash) + payload={'date_utc':datetime.datetime.now(datetime.timezone.utc).isoformat(),'commit':subprocess.check_output(['git','rev-parse','HEAD'],cwd=ROOT,text=True).strip(),'working_tree':'implementation changes uncommitted','repeats':args.repeats,'worker':{'concurrency':2,'prefetch':2,'timeout_seconds':5,'lease_seconds':8,'memory':'1GiB','cpus':1},'results':RESULTS} + check(bool(RESULTS),'no scenarios selected') + payload['image_id']=subprocess.check_output(['docker','image','inspect','mtusi-messenger:lab5','--format','{{.Id}}'],text=True).strip() + (ROOT/args.output).write_text(json.dumps(payload,ensure_ascii=False,indent=2)+'\n') + finally: + for f in FAULTS.iterdir():f.unlink() + compose('stop','worker','dispatcher',allow_failure=True) + # Restore standard configuration, disabling fault injection and short experiment leases. + subprocess.run([str(ROOT/'scripts/compose.sh'),'up','-d','--wait','--wait-timeout','120'],cwd=ROOT,stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL,timeout=180) + +def health(c): + try:c.call('GET','/health/ready',schema='Health');return True + except Failure:return False + +if __name__=='__main__':main() diff --git a/student-tests/storage.py b/student-tests/storage.py new file mode 100644 index 00000000..e652d8d7 --- /dev/null +++ b/student-tests/storage.py @@ -0,0 +1,62 @@ +#!/usr/bin/env python3 +"""Staging protection, retirement, retention and explicit restoration on synthetic data.""" +import json +from pathlib import Path +import sys +import time +ROOT=Path(__file__).resolve().parents[1] +sys.path.insert(0,str(ROOT/'tests')) +from client import Client,check +BASE=[str(ROOT/'scripts/compose.sh')] +import subprocess + +def command(*args,fail=False): + r=subprocess.run(BASE+list(args),stdout=subprocess.PIPE,stderr=subprocess.PIPE,text=True,timeout=120) + if not fail:check(r.returncode==0,'storage scenario infrastructure command failed: '+args[0]) + return r + +def sql(q):return command('exec','-T','postgres','psql','-U','messenger','-d','messenger','-At','-v','ON_ERROR_STOP=1','-c',q).stdout.strip() +def op(*args,fail=False):return command('run','--rm','--no-deps','-T','operator',*args,fail=fail) +def wait(fn): + deadline=time.monotonic()+20 + while time.monotonic()0 and exists(key),'reconciliation removed active source') + results.append({'scenario':'queued_staging_protected','result':'PASS','attachment_id':att['id']}) + command('start','worker');c.wait_ready(bob,att) + wait(lambda:sql("SELECT staging_key FROM attachments WHERE id='%s'"%att['id'])=='') + check(not exists(key),'ready source staging not retired') + results.append({'scenario':'staging_deleted_after_ready','result':'PASS','attachment_id':att['id']}) + command('stop','worker') + failed,data=c.attachment(alice,chat);key='staging/'+failed['id'];jid=sql("SELECT id FROM attachment_jobs WHERE attachment_id='%s'"%failed['id']) + op('storage','remove-object','--id',failed['id']);command('start','worker') + def is_failed():return c.call('GET','/api/v1/attachments/'+failed['id'],actor=bob,schema='Attachment')['json']['status']=='failed' + wait(is_failed) + check(sql("SELECT error_code FROM attachment_jobs WHERE id='%s'"%jid)=='source_missing','missing source not permanent') + check(op('jobs','replay','--id',jid,fail=True).returncode!=0,'replay accepted missing source') + op('storage','restore','--id',failed['id'],'--source','/fixtures/sample.png') + check(exists(key),'restored source missing') + # Advance only the synthetic record's age; do not wait a real day or change production retention. + sql("UPDATE attachments SET updated_at=clock_timestamp()-interval '25 hours' WHERE id='%s'"%failed['id']) + wait(lambda:sql("SELECT staging_key FROM attachments WHERE id='%s'"%failed['id'])=='') + check(not exists(key),'expired failed source not retired') + check(op('jobs','replay','--id',jid,fail=True).returncode!=0,'replay accepted retired source') + results.append({'scenario':'failed_retention_and_missing_source_replay','result':'PASS','age_simulated_hours':25}) + op('storage','restore','--id',failed['id'],'--source','/fixtures/sample.png');op('jobs','replay','--id',jid);c.wait_ready(bob,failed) + check(c.call('GET','/api/v1/attachments/'+failed['id']+'/content',actor=bob)['bytes']==data,'restored original changed') + results.append({'scenario':'verified_restore_then_replay','result':'PASS','job_id':jid}) + (ROOT/'evidence/storage.json').write_text(json.dumps({'results':results},indent=2)+'\n') + print('PASS: active source protected, ready staging removed, failed retention enforced, restore/replay verified') + finally:command('start','worker') +if __name__=='__main__':main() -- 2.54.0